@staticmethod<EOL><INDENT>def _with_error_handling(resp, error, mode, response_format):<DEDENT>

def safe_parse(r):<EOL><INDENT>try:<EOL><INDENT>return APIWrapper._parse_resp(r, response_format)<EOL><DEDENT>except (ValueError, SyntaxError) as ex:<EOL><INDENT>log.error(ex)<EOL>r.parsed = None<EOL>return r<EOL><DEDENT><DEDENT>if isinstance(error, requests.HTTPError):<EOL><INDENT>if resp.status_code == <NUM_LIT>:<EOL><INDENT>resp = safe_parse(resp)<EOL>if resp.parsed is not None:<EOL><INDENT>parsed_resp = resp.parsed<EOL>messages = []<EOL>if response_format == '<STR_LIT>' andparsed_resp.find('<STR_LIT>') is not None:<EOL><INDENT>messages = [e.find('<STR_LIT>').text<EOL>for e in parsed_resp.findall('<STR_LIT>')]<EOL><DEDENT>elif response_format == '<STR_LIT>' and '<STR_LIT>' in parsed_resp:<EOL><INDENT>messages = [e['<STR_LIT>']<EOL>for e in parsed_resp['<STR_LIT>']]<EOL><DEDENT>error = requests.HTTPError(<EOL>'<STR_LIT>' % (error, '<STR_LIT>'.join(messages)), response=resp)<EOL><DEDENT><DEDENT>elif resp.status_code == <NUM_LIT>:<EOL><INDENT>error = requests.HTTPError('<STR_LIT>' % error,<EOL>response=resp)<EOL><DEDENT><DEDENT>if STRICT == mode:<EOL><INDENT>raise error<EOL><DEDENT>elif GRACEFUL == mode:<EOL><INDENT>if isinstance(error, EmptyResponse):<EOL><INDENT>log.warning(error)<EOL>resp.parsed = None<EOL>return resp<EOL><DEDENT>elif isinstance(error, requests.HTTPError):<EOL><INDENT>if resp.status_code == <NUM_LIT>:    <EOL><INDENT>log.warning(error)<EOL>return safe_parse(resp)<EOL><DEDENT>else:<EOL><INDENT>raise error<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise error<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.error(error)<EOL>return safe_parse(resp)<EOL><DEDENT>

Static method for error handling.

:param resp - API response
:param error - Error thrown
:param mode - Error mode
:param response_format - XML or json


def poll(self, url, initial_delay=<NUM_LIT:2>, delay=<NUM_LIT:1>, tries=<NUM_LIT:20>, errors=STRICT, is_complete_callback=None, **params):

time.sleep(initial_delay)<EOL>poll_response = None<EOL>if is_complete_callback == None:<EOL><INDENT>is_complete_callback = self._default_poll_callback<EOL><DEDENT>for n in range(tries):<EOL><INDENT>poll_response = self.make_request(url, headers=self._headers(),<EOL>errors=errors, **params)<EOL>if is_complete_callback(poll_response):<EOL><INDENT>return poll_response<EOL><DEDENT>else:<EOL><INDENT>time.sleep(delay)<EOL><DEDENT><DEDENT>if STRICT == errors:<EOL><INDENT>raise ExceededRetries(<EOL>"<STR_LIT>".format(tries))<EOL><DEDENT>else:<EOL><INDENT>return poll_response<EOL><DEDENT>

Poll the URL
:param url - URL to poll, should be returned by 'create_session' call
:param initial_delay - specifies how many seconds to wait before the first poll
:param delay - specifies how many seconds to wait between the polls
:param tries - number of polls to perform
:param errors - errors handling mode, see corresponding parameter in 'make_request' method
:param params - additional query params for each poll request


if poll_resp.parsed is None:<EOL><INDENT>return False<EOL><DEDENT>success_list = ['<STR_LIT>', True, '<STR_LIT>']<EOL>status = None<EOL>if self.response_format == '<STR_LIT>':<EOL><INDENT>status = poll_resp.parsed.find('<STR_LIT>').text<EOL><DEDENT>elif self.response_format == '<STR_LIT>':<EOL><INDENT>status = poll_resp.parsed.get(<EOL>'<STR_LIT>', poll_resp.parsed.get('<STR_LIT:status>'))<EOL><DEDENT>if status is None:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>return status in success_list<EOL>

Checks the condition in poll response to determine if it is complete
and no subsequent poll requests should be done.


class Foo(MetaTest):<EOL><INDENT>"""<STR_LIT>"""<EOL>pass<EOL><DEDENT>Foo.__name__ = cname<EOL>return Foo<EOL>

Constructs a class named `cname` inheriting from `MetaTest`.


classes = {}<EOL>suite_fname = "<STR_LIT>" % suite_name<EOL>suite_path = os.sep.join([CIJ_ROOT, "<STR_LIT>", suite_fname])<EOL>suite = open(suite_path).read().splitlines() if suite_name else []<EOL>listing = tindex(search_path)<EOL>if listing is None:<EOL><INDENT>return {}<EOL><DEDENT>tests = suite if suite else listing<EOL>if not tests:<EOL><INDENT>return {}<EOL><DEDENT>cname = "<STR_LIT>"<EOL>if suite:<EOL><INDENT>cname += suite_name.capitalize()<EOL><DEDENT>cls = cls_factory(cname)<EOL>for test in tests:              <EOL><INDENT>tname = "<STR_LIT>" % test.replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>setattr(cls, tname, cls.meta_test)<EOL>classes[cname] = cls<EOL><DEDENT>return classes<EOL>

Construct a class with a name picked up by test-collection


if version == "<STR_LIT>":<EOL><INDENT>return DescriptorTableDenali<EOL><DEDENT>elif version == "<STR_LIT>":<EOL><INDENT>return DescriptorTableSpec20<EOL><DEDENT>elif version == "<STR_LIT>":<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>

Get descriptor table by version(denali, spec20)


if version == "<STR_LIT>":<EOL><INDENT>return sizeof(DescriptorTableDenali)<EOL><DEDENT>elif version == "<STR_LIT>":<EOL><INDENT>return sizeof(DescriptorTableSpec20)<EOL><DEDENT>elif version == "<STR_LIT>":<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>


cmd = ["<STR_LIT>", path]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True, echo=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % path)<EOL><DEDENT>return stdout.strip()<EOL>


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict(PREFIX, REQUIRED)<EOL>nvme["<STR_LIT>"] = os.path.join("<STR_LIT>", nvme["<STR_LIT>"])<EOL>try:<EOL><INDENT>sysfs = os.path.join("<STR_LIT>", nvme["<STR_LIT>"], "<STR_LIT>")<EOL>nvme["<STR_LIT>"] = cat_file(os.path.join(sysfs, "<STR_LIT:version>"))<EOL>if nvme["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>luns = "<STR_LIT>"<EOL>chs = "<STR_LIT>"<EOL><DEDENT>elif nvme["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>luns = "<STR_LIT>"<EOL>chs = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % nvme["<STR_LIT>"])<EOL><DEDENT>nvme["<STR_LIT>"] = cat_file(os.path.join(sysfs, "<STR_LIT>"))<EOL>nvme["<STR_LIT>"] = cat_file(os.path.join(sysfs, luns))<EOL>nvme["<STR_LIT>"] = cat_file(os.path.join(sysfs, chs))<EOL>nvme["<STR_LIT>"] = str(int(nvme["<STR_LIT>"]) * int(nvme["<STR_LIT>"]))<EOL>nvme["<STR_LIT>"] = str(int(nvme["<STR_LIT>"]) * int(nvme["<STR_LIT>"]))<EOL>if nvme["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>cmd = ["<STR_LIT>", "<STR_LIT>", nvme["<STR_LIT>"], "<STR_LIT>"]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>buff = cij.bin.Buffer(types=IdentifyCDS, length=<NUM_LIT:1>)<EOL>buff.memcopy(stdout)<EOL>if buff[<NUM_LIT:0>].VS[<NUM_LIT>] == <NUM_LIT>:<EOL><INDENT>nvme["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>nvme["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT><DEDENT>else:<EOL><INDENT>nvme["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>nvme["<STR_LIT>"] = str(get_sizeof_descriptor_table(nvme["<STR_LIT>"]))<EOL>nvme["<STR_LIT>"] = str(int(nvme["<STR_LIT>"]) *<EOL>int(nvme["<STR_LIT>"]))<EOL><DEDENT>except StandardError:<EOL><INDENT>traceback.print_exc()<EOL>return <NUM_LIT:1><EOL><DEDENT>cij.env_export(PREFIX, EXPORTED, nvme)<EOL>return <NUM_LIT:0><EOL>

Verify NVME variables and construct exported variables


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ["<STR_LIT>", "<STR_LIT>", nvme["<STR_LIT>"], "<STR_LIT>", str(lbaf)]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return rcode<EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ["<STR_LIT>", "<STR_LIT>", nvme["<STR_LIT>"], "<STR_LIT>"]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>if rcode:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Verify that the ENV defined NVMe device exists


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>max_size = <NUM_LIT><EOL>with open(output, "<STR_LIT:wb>") as fout:<EOL><INDENT>for off in range(offset, length, max_size):<EOL><INDENT>size = min(length - off, max_size)<EOL>cmd = ["<STR_LIT>",<EOL>nvme["<STR_LIT>"],<EOL>"<STR_LIT>",<EOL>"<STR_LIT>" % off,<EOL>"<STR_LIT>" % size,<EOL>"<STR_LIT>"]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>fout.write(stdout)<EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


def comp_meta(file_bef, file_aft, mode="<STR_LIT>"):

if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>num_chk = int(nvme["<STR_LIT>"])<EOL>meta_bef = cij.bin.Buffer(types=get_descriptor_table(nvme['<STR_LIT>']), length=num_chk)<EOL>meta_aft = cij.bin.Buffer(types=get_descriptor_table(nvme['<STR_LIT>']), length=num_chk)<EOL>meta_bef.read(file_bef)<EOL>meta_aft.read(file_aft)<EOL>for chk in range(num_chk):<EOL><INDENT>ignore = ["<STR_LIT>", "<STR_LIT>"]<EOL>if mode == "<STR_LIT>" and meta_bef[chk].CS == <NUM_LIT:4>:<EOL><INDENT>ignore.append("<STR_LIT>")<EOL><DEDENT>if meta_bef.compare(meta_aft, chk, ignore=ignore):<EOL><INDENT>cij.warn("<STR_LIT>" % chk)<EOL>meta_bef.dump(chk)<EOL>cij.warn("<STR_LIT>" % chk)<EOL>meta_aft.dump(chk)<EOL>cij.err("<STR_LIT>" % chk)<EOL>return <NUM_LIT:1><EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>

Compare chunk meta, mode=[pfail, power, reboot]


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvm = cij.env_to_dict(PREFIX, REQUIRED)<EOL>if "<STR_LIT>" in nvm["<STR_LIT>"]:<EOL><INDENT>nvm["<STR_LIT>"] = "<STR_LIT>" % nvm["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>nvm["<STR_LIT>"] = "<STR_LIT>" % nvm["<STR_LIT>"]<EOL><DEDENT>cij.env_export(PREFIX, EXPORTED, nvm)<EOL>return <NUM_LIT:0><EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ['<STR_LIT>' % nvm["<STR_LIT>"]]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>return rcode<EOL>


def dev_get_rprt(dev_name, pugrp=None, punit=None):

cmd = ["<STR_LIT>", "<STR_LIT>", dev_name]<EOL>if not (pugrp is None and punit is None):<EOL><INDENT>cmd = ["<STR_LIT>", "<STR_LIT>", dev_name, str(pugrp), str(punit)]<EOL><DEDENT>_, _, _, struct = cij.test.command_to_struct(cmd)<EOL>if not struct:<EOL><INDENT>return None<EOL><DEDENT>return struct["<STR_LIT>"]<EOL>

Get-log-page chunk information

If the pugrp and punit is set, then provide report only for that pugrp/punit

@returns the first chunk in the given state if one exists, None otherwise


def dev_get_chunk(dev_name, state, pugrp=None, punit=None):

rprt = dev_get_rprt(dev_name, pugrp, punit)<EOL>if not rprt:<EOL><INDENT>return None<EOL><DEDENT>return next((d for d in rprt if d["<STR_LIT>"] == state), None)<EOL>

Get a chunk-descriptor for the first chunk in the given state.

If the pugrp and punit is set, then search only that pugrp/punit

@returns the first chunk in the given state if one exists, None otherwise


cs = yml['<STR_LIT>'][chk]['<STR_LIT>']<EOL>return cs<EOL>


def get_chunk_information(self, chk, lun, chunk_name):

cmd = ["<STR_LIT>", self.envs,<EOL>"<STR_LIT>" % (chk, lun, chunk_name)]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


cs = self.get_chunk_status(chk, yml)<EOL>if cs >= <NUM_LIT:8>:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>


cs = self.get_chunk_status(chk)<EOL>if cs & <NUM_LIT> != <NUM_LIT:0>:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>


def s20_to_gen(self, pugrp, punit, chunk, sectr):

cmd = ["<STR_LIT>", self.envs,<EOL>"<STR_LIT>" % (pugrp, punit, chunk, sectr)]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>return int(re.findall(r"<STR_LIT>", stdout)[<NUM_LIT:0>], <NUM_LIT:16>)<EOL>


cmd = ["<STR_LIT>", self.envs, "<STR_LIT>".format(address)]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>return int(re.findall(r"<STR_LIT>", stdout)[<NUM_LIT:0>], <NUM_LIT:16>)<EOL>


cmd = ["<STR_LIT>", self.envs, "<STR_LIT>" % address]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vblk_write(self, address, file_name=None):

cmd = ["<STR_LIT>", self.envs, "<STR_LIT>" % address]<EOL>if file_name:<EOL><INDENT>cmd += ["<STR_LIT>".format(file_name)]<EOL><DEDENT>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vblk_read(self, address, file_name=None):


def vector_write(self, address_list, file_name):

address = ["<STR_LIT>".format(i) for i in address_list]<EOL>cmd = ["<STR_LIT>", self.envs, "<STR_LIT:U+0020>".join(address), "<STR_LIT>".format(file_name)]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vector_read(self, address_list, file_name):

address = ["<STR_LIT>".format(i) for i in address_list]<EOL>cmd = ["<STR_LIT>", self.envs, "<STR_LIT:U+0020>".join(address),<EOL>"<STR_LIT>".format(file_name)]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def scalar_write(self, address, block_count, data_file, meta_file):

cmd = ["<STR_LIT>", "<STR_LIT>", self.envs, "<STR_LIT>".format(address),<EOL>"<STR_LIT>".format(block_count-<NUM_LIT:1>), "<STR_LIT>".format(data_file), "<STR_LIT>".format(meta_file),<EOL>"<STR_LIT>".format(block_count * self.get_env("<STR_LIT>", "<STR_LIT>")),<EOL>"<STR_LIT>".format(block_count * self.get_env("<STR_LIT>", "<STR_LIT>"))]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def scalar_read(self, address, block_count, data_file, meta_file):

cmd = ["<STR_LIT>", "<STR_LIT>", self.envs, "<STR_LIT>".format(address),<EOL>"<STR_LIT>".format(block_count - <NUM_LIT:1>), "<STR_LIT>".format(data_file),<EOL>"<STR_LIT>".format(meta_file),<EOL>"<STR_LIT>".format(block_count * self.get_env("<STR_LIT>", "<STR_LIT>")),<EOL>"<STR_LIT>".format(block_count * self.get_env("<STR_LIT>", "<STR_LIT>"))]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def slc_erase(self, address, BE_ID=<NUM_LIT>, PMODE=<NUM_LIT>):

cmd = ["<STR_LIT>" % BE_ID, "<STR_LIT>" % PMODE, "<STR_LIT>", self.envs, "<STR_LIT>" % address]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


return self.envs[key]<EOL>


chunks = self.envs["<STR_LIT>"]<EOL>if cij.nvme.get_meta(<NUM_LIT:0>, chunks * self.envs["<STR_LIT>"], meta_file):<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>chunk_meta = cij.bin.Buffer(types=self.envs["<STR_LIT>"], length=chunks)<EOL>chunk_meta.read(meta_file)<EOL>return chunk_meta<EOL>


def get_chunk_meta_item(self, chunk_meta, grp, pug, chk):

num_chk = self.envs["<STR_LIT>"]<EOL>num_pu = self.envs["<STR_LIT>"]<EOL>index = grp * num_pu * num_chk + pug * num_chk + chk<EOL>return chunk_meta[index]<EOL>


def is_free_chunk(self, chunk_meta, grp, pug, chk):

meta = self.get_chunk_meta_item(chunk_meta, grp, pug, chk)<EOL>if meta.CS & <NUM_LIT> != <NUM_LIT:0>:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>


def is_closed_chunk(self, chunk_meta, grp, pug, chk):


def is_open_chunk(self, chunk_meta, grp, pug, chk):


def is_bad_chunk(self, chunk_meta, grp, pug, chk):


cmd = ["<STR_LIT>", self.envs["<STR_LIT>"],<EOL>"<STR_LIT>" % (pugrp, punit, chunk, sectr)]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>return int(re.findall(r"<STR_LIT>", stdout)[<NUM_LIT:0>], <NUM_LIT:16>)<EOL>


cmd = ["<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>".format(address)]<EOL>status, stdout, _ = cij.ssh.command(cmd, shell=True)<EOL>if status:<EOL><INDENT>raise RuntimeError("<STR_LIT>")<EOL><DEDENT>return int(re.findall(r"<STR_LIT>", stdout)[<NUM_LIT:0>], <NUM_LIT:16>)<EOL>


cmd = ["<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>" % address]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vblk_write(self, address, meta=False):

cmd = list()<EOL>if meta:<EOL><INDENT>cmd.append("<STR_LIT>")<EOL><DEDENT>cmd += ["<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>" % address]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vblk_read(self, address, meta=False):


cmd = ["<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>".format(address)]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vector_write(self, address_list, file_name=None):

address = ["<STR_LIT>".format(i) for i in address_list]<EOL>cmd = ["<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT:U+0020>".join(address)]<EOL>if file_name:<EOL><INDENT>cmd += ["<STR_LIT>".format(file_name)]<EOL><DEDENT>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


def vector_read(self, address_list, file_name=None):


cmd = ["<STR_LIT>", "<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>".format(address),<EOL>"<STR_LIT>".format(block_count-<NUM_LIT:1>), "<STR_LIT>".format(data_file), "<STR_LIT>".format(meta_file),<EOL>"<STR_LIT>".format(block_count * self.envs["<STR_LIT>"]),<EOL>"<STR_LIT>".format(block_count * self.envs["<STR_LIT>"])]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


cmd = ["<STR_LIT>", "<STR_LIT>", self.envs["<STR_LIT>"], "<STR_LIT>".format(address),<EOL>"<STR_LIT>".format(block_count - <NUM_LIT:1>), "<STR_LIT>".format(data_file),<EOL>"<STR_LIT>".format(meta_file),<EOL>"<STR_LIT>".format(block_count * self.envs["<STR_LIT>"]),<EOL>"<STR_LIT>".format(block_count * self.envs["<STR_LIT>"])]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True)<EOL>return status<EOL>


here = os.path.abspath(os.path.dirname(__file__))<EOL>with codecs.open(os.path.join(here, *parts), '<STR_LIT:r>') as pfp:<EOL><INDENT>return pfp.read()<EOL><DEDENT>

Read parts to use a e.g. long_description


return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))<EOL>

Expands variables from the given path and turns it into absolute path


find = re.findall(pattern, content)<EOL>if not find:<EOL><INDENT>cij.err("<STR_LIT>" % pattern)<EOL>cij.err("<STR_LIT>" % content)<EOL>return '<STR_LIT>'<EOL><DEDENT>if len(find) >= <NUM_LIT:2>:<EOL><INDENT>cij.err("<STR_LIT>" % pattern)<EOL>cij.err("<STR_LIT>" % content)<EOL>return '<STR_LIT>'<EOL><DEDENT>return find[<NUM_LIT:0>]<EOL>


def execute(cmd=None, shell=True, echo=True):

if echo:<EOL><INDENT>cij.emph("<STR_LIT>" % (shell, cmd))<EOL><DEDENT>rcode = <NUM_LIT:1><EOL>stdout, stderr = ("<STR_LIT>", "<STR_LIT>")<EOL>if cmd:<EOL><INDENT>if shell:<EOL><INDENT>cmd = "<STR_LIT:U+0020>".join(cmd)<EOL><DEDENT>proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=shell, close_fds=True)<EOL>stdout, stderr = proc.communicate()<EOL>rcode = proc.returncode<EOL><DEDENT>if rcode and echo:<EOL><INDENT>cij.warn("<STR_LIT>" % stdout)<EOL>cij.err("<STR_LIT>" % stderr)<EOL>cij.err("<STR_LIT>" % rcode)<EOL><DEDENT>return rcode, stdout, stderr<EOL>

Execute the given 'cmd'

@returns (rcode, stdout, stderr)


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>board = cij.env_to_dict(PREFIX, REQUIRED)   <EOL>if board is None:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>board["<STR_LIT>"] = "<STR_LIT:_>".join([board[r] for r in REQUIRED[:-<NUM_LIT:1>]])<EOL>board["<STR_LIT>"] = "<STR_LIT:->".join([board["<STR_LIT>"], board["<STR_LIT>"]])<EOL>cij.env_export(PREFIX, EXPORTED, board)     <EOL>return <NUM_LIT:0><EOL>

Verify BOARD variables and construct exported variables


def generate_rt_pic(process_data, para_meter, scale):

pic_path = para_meter['<STR_LIT:filename>'] + '<STR_LIT>'<EOL>plt.figure(figsize=(<NUM_LIT> * scale, <NUM_LIT> * scale))<EOL>for key in process_data.keys():<EOL><INDENT>plt.plot(process_data[key][:, <NUM_LIT:0>], process_data[key][:, <NUM_LIT:1>], label=str(key))<EOL><DEDENT>plt.title(para_meter['<STR_LIT:title>'])<EOL>plt.xlabel(para_meter['<STR_LIT>'])<EOL>plt.ylabel(para_meter['<STR_LIT>'])<EOL>plt.legend(loc='<STR_LIT>')<EOL>plt.savefig(pic_path)<EOL>return pic_path<EOL>


def generate_steady_rt_pic(process_data, para_meter, scale, steady_time):

pic_path_steady = para_meter['<STR_LIT:filename>'] + '<STR_LIT>'<EOL>plt.figure(figsize=(<NUM_LIT:4> * scale, <NUM_LIT> * scale))<EOL>for key in process_data.keys():<EOL><INDENT>if len(process_data[key]) < steady_time:<EOL><INDENT>steady_time = len(process_data[key])<EOL><DEDENT>plt.scatter(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>],<EOL>process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:1>], label=str(key), s=<NUM_LIT:10>)<EOL>steady_value = np.mean(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:1>])<EOL>steady_value_5 = steady_value * (<NUM_LIT:1> + <NUM_LIT>)<EOL>steady_value_10 = steady_value * (<NUM_LIT:1> + <NUM_LIT:0.1>)<EOL>steady_value_ng_5 = steady_value * (<NUM_LIT:1> - <NUM_LIT>)<EOL>steady_value_ng_10 = steady_value * (<NUM_LIT:1> - <NUM_LIT:0.1>)<EOL>plt.plot(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>], [steady_value] * steady_time, '<STR_LIT:b>')<EOL>plt.plot(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>], [steady_value_5] * steady_time, '<STR_LIT:g>')<EOL>plt.plot(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>],<EOL>[steady_value_ng_5] * steady_time, '<STR_LIT:g>')<EOL>plt.plot(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>], [steady_value_10] * steady_time, '<STR_LIT:r>')<EOL>plt.plot(process_data[key][-<NUM_LIT:1> * steady_time:, <NUM_LIT:0>],<EOL>[steady_value_ng_10] * steady_time, '<STR_LIT:r>')<EOL><DEDENT>plt.title(para_meter['<STR_LIT:title>'] + '<STR_LIT>')<EOL>plt.xlabel(para_meter['<STR_LIT>'] + '<STR_LIT>')<EOL>plt.ylabel(para_meter['<STR_LIT>'] + '<STR_LIT>')<EOL>plt.legend(loc='<STR_LIT>')<EOL>plt.savefig(pic_path_steady)<EOL>return pic_path_steady<EOL>


print("<STR_LIT>", len(source_data))<EOL>filter_data = {}<EOL>for index in range(<NUM_LIT:2>):<EOL><INDENT>filter_mask = source_data[:, <NUM_LIT:2>] == index<EOL>if np.any(filter_mask):<EOL><INDENT>filter_data[index] = sum_data(round_data(source_data[filter_mask]), is_bw)<EOL><DEDENT><DEDENT>return filter_data<EOL>


for index in range(len(filter_data) - <NUM_LIT:1>):<EOL><INDENT>if filter_data[index][<NUM_LIT:0>] > filter_data[index + <NUM_LIT:1>][<NUM_LIT:0>]:<EOL><INDENT>max_index = index + <NUM_LIT:1><EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>max_index = len(filter_data)<EOL><DEDENT>print("<STR_LIT>", max_index + <NUM_LIT:1>)<EOL>num_jobs = int(round(len(filter_data) * <NUM_LIT:1.0> / max_index))<EOL>print("<STR_LIT>", num_jobs)<EOL>dict_time = Counter(filter_data[:, <NUM_LIT:0>])<EOL>list_sum = []<EOL>for time_index in range(<NUM_LIT:1>, max_index + <NUM_LIT:1>):<EOL><INDENT>if dict_time.get(time_index * <NUM_LIT:1000>, <NUM_LIT:0>) != num_jobs:<EOL><INDENT>print("<STR_LIT>" % (<EOL>time_index * <NUM_LIT:1000>, dict_time.get(time_index * <NUM_LIT:1000>, <NUM_LIT:0>), num_jobs<EOL>))<EOL>continue<EOL><DEDENT>filter_mask = (filter_data[:, <NUM_LIT:0>] == time_index * <NUM_LIT:1000>)<EOL>sum_rst = np.sum(filter_data[filter_mask][:, <NUM_LIT:1>])<EOL>if is_bw:<EOL><INDENT>sum_rst = sum_rst / <NUM_LIT><EOL><DEDENT>list_sum.append([time_index, sum_rst])<EOL><DEDENT>return np.array(list_sum)<EOL>


for index, _ in enumerate(filter_data):<EOL><INDENT>filter_data[index][<NUM_LIT:0>] = round(filter_data[index][<NUM_LIT:0>] / <NUM_LIT>) * <NUM_LIT><EOL><DEDENT>return filter_data<EOL>


def import_source(self, sheet, source, delimiter="<STR_LIT:U+002C>"):

<EOL>if '<STR_LIT:U+0020>' in sheet:<EOL><INDENT>raise RuntimeError("<STR_LIT>" % sheet)<EOL><DEDENT>if not source.endswith("<STR_LIT>") and not source.endswith("<STR_LIT>"):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % source)<EOL><DEDENT>self.source_sheet = sheet<EOL>source_data = np.loadtxt(source, dtype=str, delimiter=delimiter)<EOL>self.source_data = {"<STR_LIT:title>": source_data[<NUM_LIT:0>].tolist(),<EOL>"<STR_LIT:data>": source_data[<NUM_LIT:1>:]}<EOL>cell_format_title = self.workbook.add_format({'<STR_LIT>': True,<EOL>'<STR_LIT>': u'<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT>})<EOL>cell_format = self.workbook.add_format({'<STR_LIT>': False,<EOL>'<STR_LIT>': u'<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT:0>})<EOL>worksheet = self.workbook.add_worksheet(sheet)<EOL>worksheet.write_row('<STR_LIT>', self.source_data['<STR_LIT:title>'], cell_format_title)<EOL>_, col_num = self.source_data['<STR_LIT:data>'].shape<EOL>for i in range(col_num):<EOL><INDENT>try:<EOL><INDENT>data_array = self.source_data['<STR_LIT:data>'][:, i].astype(float)<EOL><DEDENT>except ValueError:<EOL><INDENT>data_array = self.source_data['<STR_LIT:data>'][:, i]<EOL><DEDENT>worksheet.write_column(<NUM_LIT:1>, i, data_array.tolist(), cell_format)<EOL><DEDENT>

Function:
    Save original data into specific sheet, and try to translate data to float type
Input:
    sheet: Must be a non exists sheet
    source: File path of source


<EOL>if not {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}.issubset(set(properties.keys())):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % properties.keys())<EOL><DEDENT>mask = self.__filter_data(properties['<STR_LIT>'])<EOL>chart = self.__generate_chart(mask, properties)<EOL>sheet = properties['<STR_LIT>']<EOL>if sheet in self.sheet_dict.keys():<EOL><INDENT>self.sheet_dict[sheet] += <NUM_LIT:1><EOL>worksheet = self.workbook.get_worksheet_by_name(sheet)<EOL><DEDENT>else:<EOL><INDENT>self.sheet_dict[sheet] = <NUM_LIT:1><EOL>worksheet = self.workbook.add_worksheet(sheet)<EOL><DEDENT>worksheet.insert_chart('<STR_LIT>' % (<NUM_LIT:5> + (self.sheet_dict[sheet] - <NUM_LIT:1>) * <NUM_LIT>), chart)<EOL>

Function:
    Generate and save chart to specific sheet.
Input:
    sheet: If already exists, new chart will be added below.
           Otherwise, it would create a new sheet;
    x_axis: Specify x axis;
    y_axis: Specify y axis;
    series: Specify series;
    filters: dict type, use to filter useful data from original data;
    title: if None, the chart will create without title;
    x_axis_name: if None, use x_axis instead;
    y_axis_name: if None, use y_axis instead;


self.workbook.close()<EOL>


def gen_config_sheet(self, sheetname, plist):

worksheet_cfg = self.workbook.add_worksheet(sheetname)<EOL>cell_format = self.workbook.add_format({'<STR_LIT>': False, '<STR_LIT>': u'<STR_LIT>'})<EOL>cell_format_title = self.workbook.add_format({'<STR_LIT>': <NUM_LIT:0>, '<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': <NUM_LIT:12>,<EOL>'<STR_LIT>': u'<STR_LIT>', '<STR_LIT>': False})<EOL>worksheet_cfg.set_column('<STR_LIT>', <NUM_LIT>, cell_format)<EOL>worksheet_cfg.write_row('<STR_LIT>', plist[<NUM_LIT:0>], cell_format_title)<EOL>for i in range(<NUM_LIT:1>, len(plist)):<EOL><INDENT>worksheet_cfg.write_row('<STR_LIT>' % (i+<NUM_LIT:1>), plist[i], cell_format)<EOL><DEDENT>


def gen_data_sheet(self, datafile, para_meter, scale=<NUM_LIT>, steady_time=<NUM_LIT>):

filename = os.path.splitext(os.path.split(datafile)[<NUM_LIT:1>])[<NUM_LIT:0>][:-<NUM_LIT:5>]<EOL>para_meter['<STR_LIT:filename>'] = filename<EOL>source_data = np.loadtxt(datafile, dtype=int, delimiter='<STR_LIT:U+002C>')[:, :<NUM_LIT:3>]<EOL>is_bw = '<STR_LIT>'in para_meter['<STR_LIT:title>'].lower()<EOL>file_data = process_rt_data(source_data, is_bw)<EOL>pic_path = generate_rt_pic(file_data, para_meter, scale)<EOL>pic_path_steady = generate_steady_rt_pic(file_data, para_meter, scale, steady_time)<EOL>if para_meter['<STR_LIT>'] in self.sheetname_dict.keys():<EOL><INDENT>self.sheetname_dict[para_meter['<STR_LIT>']] =self.sheetname_dict[para_meter['<STR_LIT>']] + <NUM_LIT:1><EOL>chart_sheet = self.workbook.get_worksheet_by_name(para_meter['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>self.sheetname_dict[para_meter['<STR_LIT>']] = <NUM_LIT:1><EOL>chart_sheet = self.workbook.add_worksheet(para_meter['<STR_LIT>'])<EOL><DEDENT>chart_sheet.insert_image('<STR_LIT>' %<EOL>(<NUM_LIT:5> + (self.sheetname_dict[para_meter['<STR_LIT>']] - <NUM_LIT:1>) * <NUM_LIT:30>),<EOL>pic_path)<EOL>chart_sheet.insert_image('<STR_LIT>' %<EOL>(<NUM_LIT:5> + (self.sheetname_dict[para_meter['<STR_LIT>']] - <NUM_LIT:1>) * <NUM_LIT:30>),<EOL>pic_path_steady)<EOL>self.__insert_value(chart_sheet, file_data,<EOL><NUM_LIT:5> + (self.sheetname_dict[para_meter['<STR_LIT>']] - <NUM_LIT:1>) * <NUM_LIT:30>,<EOL>steady_time)<EOL>self.pic_list.append(pic_path)<EOL>self.pic_list.append(pic_path_steady)<EOL>

datafile, sheetname, x_axis_name, y_axis_name, title,
Function:
    Turn realtime bw data into picture, and save into specific sheet
Input:
    sheetname: If already exists, new chart will be added continuely.
               Otherwise, it would create new sheet;
    x_axis_name: x_axis name;
    y_axis_name: y_axis name;
    title: picure name;
    scale； size of picture.


hnames = []<EOL>for hook in ent["<STR_LIT>"]["<STR_LIT>"] + ent["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>hname = os.path.basename(hook["<STR_LIT>"])<EOL>hname = os.path.splitext(hname)[<NUM_LIT:0>]<EOL>hname = hname.strip()<EOL>hname = hname.replace("<STR_LIT>", "<STR_LIT>")<EOL>hname = hname.replace("<STR_LIT>", "<STR_LIT>")<EOL>if hname in hnames:<EOL><INDENT>continue<EOL><DEDENT>hnames.append(hname)<EOL><DEDENT>hnames.sort()<EOL>return hnames<EOL>

Extract hook names from the given entity


src = open(tcase["<STR_LIT>"]).read()<EOL>if len(src) < <NUM_LIT:3>:<EOL><INDENT>cij.err("<STR_LIT>" % tcase["<STR_LIT:name>"])<EOL>return None<EOL><DEDENT>ext = os.path.splitext(tcase["<STR_LIT>"])[-<NUM_LIT:1>]<EOL>if ext not in ["<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>cij.err("<STR_LIT>" % (<EOL>ext, tcase["<STR_LIT:name>"]<EOL>))<EOL>return None<EOL><DEDENT>comment = []<EOL>for line in src.splitlines()[<NUM_LIT:2>:]:<EOL><INDENT>if ext == "<STR_LIT>" and not line.startswith("<STR_LIT:#>"):<EOL><INDENT>break<EOL><DEDENT>elif ext == "<STR_LIT>" and not '<STR_LIT>' in line:<EOL><INDENT>break<EOL><DEDENT>comment.append(line)<EOL><DEDENT>return comment<EOL>

Extract testcase comment section / testcase description

@returns the testcase-comment from the tcase["fpath"] as a list of strings


descr_short = "<STR_LIT>"<EOL>descr_long = "<STR_LIT>"<EOL>try:<EOL><INDENT>comment = tcase_comment(tcase)<EOL><DEDENT>except (IOError, OSError, ValueError) as exc:<EOL><INDENT>comment = []<EOL>cij.err("<STR_LIT>" % (exc, tcase))<EOL><DEDENT>comment = [l for l in comment if l.strip()]     <EOL>for line_number, line in enumerate(comment):<EOL><INDENT>if line.startswith("<STR_LIT:#>"):<EOL><INDENT>comment[line_number] = line[<NUM_LIT:1>:]<EOL><DEDENT><DEDENT>if comment:<EOL><INDENT>descr_short = comment[<NUM_LIT:0>]<EOL><DEDENT>if len(comment) > <NUM_LIT:1>:<EOL><INDENT>descr_long = "<STR_LIT:\n>".join(comment[<NUM_LIT:1>:])<EOL><DEDENT>return descr_short, descr_long<EOL>

Parse descriptions from the the given tcase


if not os.path.isdir(run_root):<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>hook_enter = []<EOL>hook_exit = []<EOL>tcase = []<EOL>for fpath in glob.glob(os.sep.join([run_root, "<STR_LIT>"])):<EOL><INDENT>if "<STR_LIT>" in fpath:<EOL><INDENT>hook_exit.append(fpath)<EOL>continue<EOL><DEDENT>if "<STR_LIT>" in fpath:<EOL><INDENT>hook_enter.append(fpath)<EOL>continue<EOL><DEDENT>tcase.append(fpath)<EOL><DEDENT>content = "<STR_LIT>"<EOL>for fpath in hook_enter + tcase + hook_exit:<EOL><INDENT>content += "<STR_LIT>" % fpath<EOL>content += open(fpath, "<STR_LIT:r>").read()<EOL>content += "<STR_LIT>" % fpath<EOL><DEDENT>return content<EOL>

Returns content of the given 'fpath' with HTML annotations, currently simply
a conversion of ANSI color codes to HTML elements


if not os.path.exists(fpath):<EOL><INDENT>return "<STR_LIT>" % fpath<EOL><DEDENT>return open(fpath, "<STR_LIT:r>").read()<EOL>

Returns content of the given 'fpath' with HTML annotations for syntax
highlighting


listing = []<EOL>for root, _, fnames in os.walk(aux_root):<EOL><INDENT>count = len(aux_root.split(os.sep))<EOL>prefix = root.split(os.sep)[count:]<EOL>for fname in fnames:<EOL><INDENT>listing.append(os.sep.join(prefix + [fname]))<EOL><DEDENT><DEDENT>return listing<EOL>


<EOL>tsuite["<STR_LIT>"] = runlogs_to_html(tsuite["<STR_LIT>"])<EOL>tsuite["<STR_LIT>"] = aux_listing(tsuite["<STR_LIT>"])<EOL>tsuite["<STR_LIT>"] = extract_hook_names(tsuite)<EOL>return True<EOL>

Goes through the tsuite and processes "*.log


tcase["<STR_LIT>"] = src_to_html(tcase["<STR_LIT>"])<EOL>tcase["<STR_LIT>"] = runlogs_to_html(tcase["<STR_LIT>"])<EOL>tcase["<STR_LIT>"] = aux_listing(tcase["<STR_LIT>"])<EOL>tcase["<STR_LIT>"], tcase["<STR_LIT>"] = tcase_parse_descr(tcase)<EOL>tcase["<STR_LIT>"] = extract_hook_names(tcase)<EOL>return True<EOL>

Goes through the trun and processes "run.log


trun["<STR_LIT>"] = runlogs_to_html(trun["<STR_LIT>"])<EOL>trun["<STR_LIT>"] = aux_listing(trun["<STR_LIT>"])<EOL>trun["<STR_LIT>"] = extract_hook_names(trun)<EOL>return True<EOL>


plog = []<EOL>plog.append(("<STR_LIT>", process_trun(trun)))<EOL>for tsuite in trun["<STR_LIT>"]:<EOL><INDENT>plog.append(("<STR_LIT>", process_tsuite(tsuite)))<EOL>for tcase in tsuite["<STR_LIT>"]:<EOL><INDENT>plog.append(("<STR_LIT>", process_tcase(tcase)))<EOL><DEDENT><DEDENT>for task, success in plog:<EOL><INDENT>if not success:<EOL><INDENT>cij.err("<STR_LIT>" % task)<EOL><DEDENT><DEDENT>return sum((success for task, success in plog))<EOL>

Perform postprocessing of the given test run


def stamp_to_datetime(stamp):<EOL><INDENT>"""<STR_LIT>"""<EOL>return datetime.datetime.fromtimestamp(int(stamp))<EOL><DEDENT>def strftime(dtime, fmt):<EOL><INDENT>"""<STR_LIT>"""<EOL>return dtime.strftime(fmt)<EOL><DEDENT>def ansi_to_html(ansi):<EOL><INDENT>"""<STR_LIT>"""<EOL>conv = ansi2html.Ansi2HTMLConverter(<EOL>scheme="<STR_LIT>",<EOL>inline=True<EOL>)<EOL>html = conv.convert(ansi, full=False)<EOL>with open("<STR_LIT>", "<STR_LIT:w>") as html_file:<EOL><INDENT>html_file.write(html)<EOL><DEDENT>return html<EOL><DEDENT>tmpl_dpath = os.path.dirname(tmpl_fpath)<EOL>tmpl_fname = os.path.basename(tmpl_fpath)<EOL>env = jinja2.Environment(<EOL>autoescape=True,<EOL>loader=jinja2.FileSystemLoader(tmpl_dpath)<EOL>)<EOL>env.filters['<STR_LIT>'] = stamp_to_datetime<EOL>env.filters['<STR_LIT>'] = strftime<EOL>env.filters['<STR_LIT>'] = ansi_to_html<EOL>tmpl = env.get_template(tmpl_fname)<EOL>return tmpl.render(dset=dset)<EOL>

@returns A HTML representation of the given 'dset' using the template at
'tmpl_fpath'


if old == new:<EOL><INDENT>return<EOL><DEDENT>if isinstance(struct, list):<EOL><INDENT>for item in struct:<EOL><INDENT>rehome(old, new, item)<EOL><DEDENT><DEDENT>elif isinstance(struct, dict):<EOL><INDENT>for key, val in struct.iteritems():<EOL><INDENT>if isinstance(val, (dict, list)):<EOL><INDENT>rehome(old, new, val)<EOL><DEDENT>elif "<STR_LIT>" in key:<EOL><INDENT>continue<EOL><DEDENT>elif "<STR_LIT>" in key:<EOL><INDENT>continue<EOL><DEDENT>elif "<STR_LIT:root>" in key or "<STR_LIT:path>" in key:<EOL><INDENT>struct[key] = struct[key].replace(old, new)<EOL><DEDENT><DEDENT><DEDENT>

Replace all absolute paths to "re-home" it


trun = cij.runner.trun_from_file(args.trun_fpath)<EOL>rehome(trun["<STR_LIT>"]["<STR_LIT>"], args.output, trun)<EOL>postprocess(trun)<EOL>cij.emph("<STR_LIT>" % args.tmpl_fpath)<EOL>cij.emph("<STR_LIT>" % args.output)<EOL>html_fpath = os.sep.join([args.output, "<STR_LIT>" % args.tmpl_name])<EOL>cij.emph("<STR_LIT>" % html_fpath)<EOL>try:                                    <EOL><INDENT>with open(html_fpath, '<STR_LIT:w>') as html_file:<EOL><INDENT>html_file.write(dset_to_html(trun, args.tmpl_fpath))<EOL><DEDENT><DEDENT>except (IOError, OSError, ValueError) as exc:<EOL><INDENT>import traceback<EOL>traceback.print_exc()<EOL>cij.err("<STR_LIT>" % exc)<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>


spath = spath if spath else os.environ.get("<STR_LIT>", None)<EOL>if spath is None:<EOL><INDENT>return None<EOL><DEDENT>tests = []                          <EOL>for root, _, files in os.walk(spath):<EOL><INDENT>if root != spath:<EOL><INDENT>continue<EOL><DEDENT>tests += [f for f in files if f[-<NUM_LIT:3>:] in ["<STR_LIT>", "<STR_LIT>"]]<EOL><DEDENT>return tests<EOL>

Lists tindex in CIJ_TESTCASES

@Returns On success, a list of filenames is returned. On error, None is
returned


variables = {}<EOL>for req in REQS:<EOL><INDENT>prefix = req.upper()<EOL>variables[prefix] = cij.env_to_dict(<EOL>prefix, getattr(cij, req).REQUIRED + getattr(cij, req).EXPORTED<EOL>)<EOL><DEDENT>return variables<EOL>

Return variables defined by modules required by test


REQS.append(req)<EOL>


if cij.ssh.env():<EOL><INDENT>tfail("<STR_LIT>")<EOL><DEDENT>for req in REQS:<EOL><INDENT>if getattr(cij, req).env():<EOL><INDENT>tfail()<EOL><DEDENT><DEDENT>cij.emph("<STR_LIT>")<EOL>

Enter the test, check requirements and setup aux. environment


msg = "<STR_LIT>" % msg if msg else "<STR_LIT>"<EOL>if rcode:<EOL><INDENT>cij.err("<STR_LIT>" % msg)<EOL><DEDENT>else:<EOL><INDENT>cij.good("<STR_LIT>" % msg)<EOL><DEDENT>sys.exit(rcode)<EOL>


texit(msg, <NUM_LIT:0>)<EOL>

Testing: Exit test indicating test passed


texit(msg, <NUM_LIT:1>)<EOL>

Testing: Exit test indicating test failed


def command(cmd, ssh=True, shell=True, echo=True):

if ssh:<EOL><INDENT>return cij.ssh.command(cmd, shell, echo)<EOL><DEDENT>return cij.util.execute(cmd, shell, echo)<EOL>

Execute the given 'cmd'

@returns (rcode, stdout, stderr)


struct = None<EOL>rcode, stdout, stderr = command(cmd)<EOL>try:<EOL><INDENT>lines = []<EOL>for line in stdout.splitlines():<EOL><INDENT>if line.strip().startswith("<STR_LIT:#>"):<EOL><INDENT>continue<EOL><DEDENT>lines.append(line)<EOL><DEDENT>struct = yaml.safe_load("<STR_LIT:\n>".join(lines))<EOL><DEDENT>except (yaml.YAMLError) as exc:<EOL><INDENT>cij.err("<STR_LIT>" % exc)<EOL><DEDENT>return rcode, stdout, stderr, struct<EOL>

Same as `command` except it tries to convert stdout to struct

@returns (rcode, struct, stderr, struct)


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>lnvm = cij.env_to_dict(PREFIX, REQUIRED)<EOL>nvme = cij.env_to_dict("<STR_LIT>", ["<STR_LIT>"])<EOL>if "<STR_LIT>" not in lnvm.keys():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>if "<STR_LIT>" not in lnvm.keys():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>if "<STR_LIT>" not in lnvm.keys():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>lnvm["<STR_LIT>"] = "<STR_LIT>" % (nvme["<STR_LIT>"], int(lnvm["<STR_LIT>"]), int(lnvm["<STR_LIT>"]))<EOL>lnvm["<STR_LIT>"] = "<STR_LIT>" % lnvm["<STR_LIT>"]<EOL>cij.env_export(PREFIX, EXPORTED, lnvm)<EOL>return <NUM_LIT:0><EOL>

Verify LNVM variables and construct exported variables


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>nvme = cij.env_to_dict("<STR_LIT>", ["<STR_LIT>"])<EOL>lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cij.emph("<STR_LIT>" % lnvm["<STR_LIT>"])<EOL>cmd = ["<STR_LIT>" % (<EOL>nvme["<STR_LIT>"], lnvm["<STR_LIT>"], lnvm["<STR_LIT>"], lnvm["<STR_LIT>"], lnvm["<STR_LIT>"])]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True)<EOL>if rcode:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cij.emph("<STR_LIT>" % lnvm["<STR_LIT>"])<EOL>cmd = ["<STR_LIT>" % (lnvm["<STR_LIT>"])]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True)<EOL>if rcode:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>lnvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ['<STR_LIT>' % lnvm["<STR_LIT>"]]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>if rcode:<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Verify that the ENV defined LNVM device exists


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>

Verify FIO variables and construct exported variables


if env():<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>cij.emph("<STR_LIT>" % (shell, self.__prefix + self.__suffix))<EOL>return cij.ssh.command(self.__prefix, shell, echo, self.__suffix)<EOL>


self.__thread = Thread(target=self.__run, args=(True, False))<EOL>self.__thread.setDaemon(True)<EOL>self.__thread.start()<EOL>


if self.__thread:<EOL><INDENT>cmd = ["<STR_LIT>"]<EOL>status, output, _ = cij.util.execute(cmd, shell=True, echo=True)<EOL>if status:<EOL><INDENT>cij.warn("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>tty = output.split()[<NUM_LIT:1>]<EOL>cmd = ["<STR_LIT>".format("<STR_LIT:U+0020>".join(self.__prefix), tty)]<EOL>status, _, _ = cij.util.execute(cmd, shell=True, echo=True)<EOL>if status:<EOL><INDENT>cij.warn("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>self.__thread.join()<EOL>self.__thread = None<EOL><DEDENT>return <NUM_LIT:0><EOL>


if ext is None:<EOL><INDENT>ext = "<STR_LIT>"<EOL><DEDENT>fnames = set([])<EOL>for _, _, files in os.walk(search_path):<EOL><INDENT>for fname in files:<EOL><INDENT>if os.path.splitext(fname)[-<NUM_LIT:1>] in EXTS[ext]:<EOL><INDENT>fnames.add(fname)<EOL><DEDENT><DEDENT><DEDENT>return fnames<EOL>

@returns a set of filenames with extension 'ext' in 'search_path


if CIJ_ECHO_TIME_STAMP == "<STR_LIT:1>":<EOL><INDENT>return time.strftime('<STR_LIT>', time.localtime(time.time()))<EOL><DEDENT>return "<STR_LIT>"<EOL>

Get time stampe if CIJ_ECHO_TIME_STAMP is 1


print("<STR_LIT>" % (PR_EMPH_CC, get_time_stamp(), txt, PR_NC))<EOL>sys.stdout.flush()<EOL>

Print, emphasized 'neutral', the given 'txt' message


print("<STR_LIT>" % (PR_GOOD_CC, get_time_stamp(), txt, PR_NC))<EOL>sys.stdout.flush()<EOL>

Print, emphasized 'good', the given 'txt' message


print("<STR_LIT>" % (PR_WARN_CC, get_time_stamp(), txt, PR_NC))<EOL>sys.stdout.flush()<EOL>

Print, emphasized 'warning', the given 'txt' message


print("<STR_LIT>" % (PR_ERR_CC, get_time_stamp(), txt, PR_NC))<EOL>sys.stdout.flush()<EOL>

Print, emphasized 'error', the given 'txt' message


if rval is None:    <EOL><INDENT>info(txt)<EOL><DEDENT>elif rval == <NUM_LIT:0>:     <EOL><INDENT>good(txt)<EOL><DEDENT>else:               <EOL><INDENT>err(txt)<EOL><DEDENT>


def expand_path(path):<EOL><INDENT>"""<STR_LIT>"""<EOL>return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))<EOL><DEDENT>if prefix is None:<EOL><INDENT>prefix = "<STR_LIT>"<EOL><DEDENT>if names is None:<EOL><INDENT>names = [<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>"<EOL>]<EOL><DEDENT>conf = {v: os.environ.get("<STR_LIT:_>".join([prefix, v])) for v in names}<EOL>for env in (e for e in conf.keys() if e[:len(prefix)] in names and conf[e]):<EOL><INDENT>conf[env] = expand_path(conf[env])<EOL>if not os.path.exists(conf[env]):<EOL><INDENT>err("<STR_LIT>" % (prefix, env, conf[env]))<EOL><DEDENT><DEDENT>return conf<EOL>

Construct dict of paths from environment variables


env = {}<EOL>for name in names:<EOL><INDENT>env[name] = ENV.get("<STR_LIT:_>".join([prefix, name]))<EOL>if env[name] is None:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>return env<EOL>

Construct dict from environment variables named: PREFIX_NAME

@returns dict of names


for exp in exported:<EOL><INDENT>ENV["<STR_LIT:_>".join([prefix, exp])] = env[exp]<EOL><DEDENT>

Define the list of 'exported' variables with 'prefix' with values from 'env'


def dump(buf, indent=<NUM_LIT:0>, skip="<STR_LIT>"):

if not isinstance(type(buf), (type(Union), type(Structure))):<EOL><INDENT>raise RuntimeError("<STR_LIT>" % type(buf))<EOL><DEDENT>for field in getattr(buf, '<STR_LIT>'):<EOL><INDENT>name, types = field[<NUM_LIT:0>], field[<NUM_LIT:1>]<EOL>if name in skip:<EOL><INDENT>return<EOL><DEDENT>value = getattr(buf, name)<EOL>if isinstance(types, (type(Union), type(Structure))):<EOL><INDENT>cij.info("<STR_LIT>" % ("<STR_LIT:U+0020>" * indent, name))<EOL>dump(value, indent+<NUM_LIT:2>, skip)<EOL><DEDENT>elif isinstance(types, type(Array)):<EOL><INDENT>for i, item in enumerate(value):<EOL><INDENT>name_index = "<STR_LIT>" % (name, i)<EOL>if isinstance(types, (type(Union), type(Structure))):<EOL><INDENT>cij.info("<STR_LIT>" % ("<STR_LIT:U+0020>" * indent, name_index))<EOL>dump(item, indent + <NUM_LIT:2>, skip)<EOL><DEDENT>else:<EOL><INDENT>cij.info("<STR_LIT>" % ("<STR_LIT:U+0020>" * indent, name_index, item))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>cij.info("<STR_LIT>" % ("<STR_LIT:U+0020>" * indent, name, value))<EOL><DEDENT><DEDENT>


for field in getattr(buf_a, '<STR_LIT>'):<EOL><INDENT>name, types = field[<NUM_LIT:0>], field[<NUM_LIT:1>]<EOL>if name in ignore:<EOL><INDENT>continue<EOL><DEDENT>val_a = getattr(buf_a, name)<EOL>val_b = getattr(buf_b, name)<EOL>if isinstance(types, (type(Union), type(Structure))):<EOL><INDENT>if compare(val_a, val_b, ignore):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>elif isinstance(types, type(Array)):<EOL><INDENT>for i, _ in enumerate(val_a):<EOL><INDENT>if isinstance(types, (type(Union), type(Structure))):<EOL><INDENT>if compare(val_a[i], val_b[i], ignore):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>if val_a[i] != val_b[i]:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if val_a != val_b:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


return self.m_len<EOL>


return self.m_size<EOL>


return self.m_types<EOL>


def memcopy(self, stream, offset=<NUM_LIT:0>, length=float("<STR_LIT>")):

data = [ord(i) for i in list(stream)]<EOL>size = min(length, len(data), self.m_size)<EOL>buff = cast(self.m_buf, POINTER(c_uint8))<EOL>for i in range(size):<EOL><INDENT>buff[offset + i] = data[i]<EOL><DEDENT>


with open(path, "<STR_LIT:wb>") as fout:<EOL><INDENT>fout.write(self.m_buf)<EOL><DEDENT>


with open(path, "<STR_LIT:rb>") as fout:<EOL><INDENT>memmove(self.m_buf, fout.read(self.m_size), self.m_size)<EOL><DEDENT>


def dump(self, offset=<NUM_LIT:0>, length=<NUM_LIT:1>):

for i in range(offset, offset + length):<EOL><INDENT>if "<STR_LIT>" in str(self.m_types):<EOL><INDENT>cij.info("<STR_LIT>" % (i, self.m_buf[i]))<EOL><DEDENT>else:<EOL><INDENT>cij.info("<STR_LIT>" % i)<EOL>dump(self.m_buf[i], <NUM_LIT:2>)<EOL><DEDENT><DEDENT>


def compare(self, buf, offset=<NUM_LIT:0>, length=<NUM_LIT:1>, ignore="<STR_LIT>"):

for i in range(offset, offset + length):<EOL><INDENT>if isinstance(self.m_types, (type(Union), type(Structure))):<EOL><INDENT>if compare(self.m_buf[i], buf[i], ignore=ignore):<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>elif self.m_buf[i] != buf[i]:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


if self.__power_on_port is None:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return self.__press(self.__power_on_port, interval=interval)<EOL>


if self.__power_off_port is None:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return self.__press(self.__power_off_port, interval=interval)<EOL>


if self.__power_btn_port is None:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return self.__press(self.__power_btn_port, interval=interval)<EOL>


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>block = cij.env_to_dict(PREFIX, REQUIRED)<EOL>block["<STR_LIT>"] = "<STR_LIT>" % block["<STR_LIT>"]<EOL>cij.env_export(PREFIX, EXPORTED, block)<EOL>return <NUM_LIT:0><EOL>

Verify BLOCK variables and construct exported variables


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>block = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ['<STR_LIT>' % block["<STR_LIT>"]]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>return rcode<EOL>

Verify that the ENV defined BLOCK device exists


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>return <NUM_LIT:0><EOL>


if env():<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>cmd = ["<STR_LIT>"]<EOL>status, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>if not status:<EOL><INDENT>status, _, _ = cij.ssh.command(["<STR_LIT>"], shell=True)<EOL>if status:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


self.output = self.target(*self.args)<EOL>


return self.output<EOL>


args = list()<EOL>for key, val in self.__parm.items():<EOL><INDENT>key = key.replace("<STR_LIT>", "<STR_LIT>").lower()<EOL>if key == "<STR_LIT>":<EOL><INDENT>args.append("<STR_LIT>")<EOL><DEDENT>if val is None:<EOL><INDENT>args.append("<STR_LIT>" % key)<EOL><DEDENT>else:<EOL><INDENT>args.append("<STR_LIT>" % (key, val))<EOL><DEDENT><DEDENT>return args<EOL>


for key, val in args.items():<EOL><INDENT>self.set_parm(key, val)<EOL><DEDENT>


self.__parm[key] = val<EOL>

Set parameter of FIO
If val is None, the parameter is "--key"
If val is not None, the parameter is "--key=val"


if key in self.__parm.keys():<EOL><INDENT>return self.__parm[key]<EOL><DEDENT>return None<EOL>


self.__thread = Threads(target=self.run, args=(True, True, False))<EOL>self.__thread.setDaemon(True)<EOL>self.__thread.start()<EOL>


if self.__thread:<EOL><INDENT>self.__thread.join(timeout)<EOL><DEDENT>


if self.__thread:<EOL><INDENT>return self.__thread.result()<EOL><DEDENT>return None<EOL>


def run(self, shell=True, cmdline=False, echo=True):

if env():<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>cmd = ["<STR_LIT>"] + self.__parse_parms()<EOL>if cmdline:<EOL><INDENT>cij.emph("<STR_LIT>" % (shell, cmd))<EOL><DEDENT>return cij.ssh.command(cmd, shell, echo)<EOL>


ipmi = cij.env_to_dict(PREFIX, REQUIRED)<EOL>if ipmi is None:<EOL><INDENT>ipmi["<STR_LIT>"] = "<STR_LIT>"<EOL>ipmi["<STR_LIT>"] = "<STR_LIT>"<EOL>ipmi["<STR_LIT>"] = "<STR_LIT:localhost>"<EOL>ipmi["<STR_LIT>"] = "<STR_LIT>"<EOL>cij.info("<STR_LIT>" % (<EOL>ipmi["<STR_LIT>"], ipmi["<STR_LIT>"], ipmi["<STR_LIT>"], ipmi["<STR_LIT>"]<EOL>))<EOL><DEDENT>cij.env_export(PREFIX, EXPORTED, ipmi)<EOL>return <NUM_LIT:0><EOL>


env()<EOL>ipmi = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>command = "<STR_LIT>" % (<EOL>ipmi["<STR_LIT>"], ipmi["<STR_LIT>"], ipmi["<STR_LIT>"], ipmi["<STR_LIT>"], command)<EOL>cij.info("<STR_LIT>" % command)<EOL>return cij.util.execute(command, shell=True, echo=True)<EOL>


if cij.ssh.env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>pci = cij.env_to_dict(PREFIX, REQUIRED)<EOL>pci["<STR_LIT>"] = "<STR_LIT>"<EOL>pci["<STR_LIT>"] = os.sep.join([pci["<STR_LIT>"], "<STR_LIT>", pci["<STR_LIT>"]])<EOL>cij.env_export(PREFIX, EXPORTED, pci)<EOL>return <NUM_LIT:0><EOL>

Verify PCI variables and construct exported variables


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>pci = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)<EOL>cmd = ['<STR_LIT>' % pci["<STR_LIT>"]]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>return rcode<EOL>

Verify that the ENV defined PCI device exists


return os.sep.join([output_path, "<STR_LIT>"])<EOL>


if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % script)<EOL>cij.emph("<STR_LIT>" % script["<STR_LIT>"])<EOL><DEDENT>launchers = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT:source>"<EOL>}<EOL>ext = os.path.splitext(script["<STR_LIT>"])[-<NUM_LIT:1>]<EOL>if not ext in launchers.keys():<EOL><INDENT>cij.err("<STR_LIT>" % script["<STR_LIT>"])<EOL>return <NUM_LIT:1><EOL><DEDENT>launch = launchers[ext]<EOL>with open(script["<STR_LIT>"], "<STR_LIT:a>") as log_fd:<EOL><INDENT>log_fd.write("<STR_LIT>" % script["<STR_LIT>"])<EOL>log_fd.flush()<EOL>bgn = time.time()<EOL>cmd = [<EOL>'<STR_LIT>', '<STR_LIT:-c>',<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (<EOL>trun["<STR_LIT>"]["<STR_LIT>"],<EOL>script["<STR_LIT>"],<EOL>launch,<EOL>script["<STR_LIT>"]<EOL>)<EOL>]<EOL>if trun["<STR_LIT>"]["<STR_LIT>"] > <NUM_LIT:1>:<EOL><INDENT>cij.emph("<STR_LIT>" % "<STR_LIT:U+0020>".join(cmd))<EOL><DEDENT>evars = os.environ.copy()<EOL>evars.update({k: str(script["<STR_LIT>"][k]) for k in script["<STR_LIT>"]})<EOL>process = Popen(<EOL>cmd,<EOL>stdout=log_fd,<EOL>stderr=STDOUT,<EOL>cwd=script["<STR_LIT>"],<EOL>env=evars<EOL>)<EOL>process.wait()<EOL>script["<STR_LIT>"] = process.returncode<EOL>script["<STR_LIT>"] = time.time() - bgn<EOL><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % script["<STR_LIT>"])<EOL>cij.emph(<EOL>"<STR_LIT>" % script["<STR_LIT>"],<EOL>script["<STR_LIT>"]<EOL>)<EOL><DEDENT>return script["<STR_LIT>"]<EOL>


hook = copy.deepcopy(HOOK)<EOL>hook["<STR_LIT:name>"] = os.path.splitext(os.path.basename(hook_fpath))[<NUM_LIT:0>]<EOL>hook["<STR_LIT:name>"] = hook["<STR_LIT:name>"].replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>hook["<STR_LIT>"] = parent["<STR_LIT>"]<EOL>hook["<STR_LIT>"] = hook_fpath<EOL>hook["<STR_LIT>"] = "<STR_LIT>" % os.path.basename(hook["<STR_LIT>"])<EOL>hook["<STR_LIT>"] = os.sep.join([hook["<STR_LIT>"], hook["<STR_LIT>"]])<EOL>hook["<STR_LIT>"] = os.sep.join([<EOL>hook["<STR_LIT>"],<EOL>"<STR_LIT>" % hook["<STR_LIT>"]<EOL>])<EOL>hook["<STR_LIT>"].update(copy.deepcopy(parent["<STR_LIT>"]))<EOL>shutil.copyfile(hook["<STR_LIT>"], hook["<STR_LIT>"])<EOL>return hook<EOL>


def hooks_setup(trun, parent, hnames=None):

hooks = {<EOL>"<STR_LIT>": [],<EOL>"<STR_LIT>": []<EOL>}<EOL>if hnames is None:       <EOL><INDENT>return hooks<EOL><DEDENT>for hname in hnames:      <EOL><INDENT>for med in HOOK_PATTERNS:<EOL><INDENT>for ptn in HOOK_PATTERNS[med]:<EOL><INDENT>fpath = os.sep.join([trun["<STR_LIT>"]["<STR_LIT>"], ptn % hname])<EOL>if not os.path.exists(fpath):<EOL><INDENT>continue<EOL><DEDENT>hook = hook_setup(parent, fpath)<EOL>if not hook:<EOL><INDENT>continue<EOL><DEDENT>hooks[med].append(hook)<EOL><DEDENT><DEDENT>if not hooks["<STR_LIT>"] + hooks["<STR_LIT>"]:<EOL><INDENT>cij.err("<STR_LIT>" % hname)<EOL>return None<EOL><DEDENT><DEDENT>return hooks<EOL>

Setup test-hooks
@returns dict of hook filepaths {"enter": [], "exit": []}


if fpath is None:<EOL><INDENT>fpath = yml_fpath(trun["<STR_LIT>"]["<STR_LIT>"])<EOL><DEDENT>with open(fpath, '<STR_LIT:w>') as yml_file:<EOL><INDENT>data = yaml.dump(trun, explicit_start=True, default_flow_style=False)<EOL>yml_file.write(data)<EOL><DEDENT>


with open(fpath, '<STR_LIT:r>') as yml_file:<EOL><INDENT>return yaml.safe_load(yml_file)<EOL><DEDENT>


if trun["<STR_LIT>"]["<STR_LIT>"] > <NUM_LIT:1>:               <EOL><INDENT>cij.emph("<STR_LIT>")<EOL>for cvar in sorted(trun["<STR_LIT>"].keys()):<EOL><INDENT>cij.emph("<STR_LIT>" % (cvar, trun["<STR_LIT>"][cvar]))<EOL><DEDENT>cij.emph("<STR_LIT:}>")<EOL><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>")<EOL>cij.emph("<STR_LIT>" % trun["<STR_LIT>"]["<STR_LIT>"])<EOL>cij.emph("<STR_LIT>" % yml_fpath(trun["<STR_LIT>"]["<STR_LIT>"]))<EOL>cij.emph("<STR_LIT:}>")<EOL><DEDENT>


def tcase_setup(trun, parent, tcase_fname):

<EOL>case = copy.deepcopy(TESTCASE)<EOL>case["<STR_LIT>"] = tcase_fname<EOL>case["<STR_LIT>"] = os.sep.join([trun["<STR_LIT>"]["<STR_LIT>"], case["<STR_LIT>"]])<EOL>if not os.path.exists(case["<STR_LIT>"]):<EOL><INDENT>cij.err('<STR_LIT>' % case["<STR_LIT>"])<EOL>return None<EOL><DEDENT>case["<STR_LIT:name>"] = os.path.splitext(case["<STR_LIT>"])[<NUM_LIT:0>]<EOL>case["<STR_LIT>"] = "<STR_LIT:/>".join([parent["<STR_LIT>"], case["<STR_LIT>"]])<EOL>case["<STR_LIT>"] = os.sep.join([parent["<STR_LIT>"], case["<STR_LIT>"]])<EOL>case["<STR_LIT>"] = os.sep.join([case["<STR_LIT>"], "<STR_LIT>"])<EOL>case["<STR_LIT>"] = os.sep.join([case["<STR_LIT>"], "<STR_LIT>"])<EOL>case["<STR_LIT>"] = os.sep.join([case["<STR_LIT>"], case["<STR_LIT>"]])<EOL>case["<STR_LIT>"].update(copy.deepcopy(parent["<STR_LIT>"]))<EOL>os.makedirs(case["<STR_LIT>"])                       <EOL>os.makedirs(case["<STR_LIT>"])<EOL>shutil.copyfile(case["<STR_LIT>"], case["<STR_LIT>"])  <EOL>case["<STR_LIT>"] = hooks_setup(trun, case, parent.get("<STR_LIT>"))<EOL>return case<EOL>


if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>")<EOL><DEDENT>rcode = <NUM_LIT:0><EOL>for hook in reversed(tsuite["<STR_LIT>"]["<STR_LIT>"]):      <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>

Triggers when exiting the given testsuite


if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % tsuite["<STR_LIT:name>"])<EOL><DEDENT>rcode = <NUM_LIT:0><EOL>for hook in tsuite["<STR_LIT>"]["<STR_LIT>"]:     <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>

Triggers when entering the given testsuite


suite = copy.deepcopy(TESTSUITE)  <EOL>suite["<STR_LIT:name>"] = declr.get("<STR_LIT:name>")<EOL>if suite["<STR_LIT:name>"] is None:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return None<EOL><DEDENT>suite["<STR_LIT>"] = declr.get("<STR_LIT>")<EOL>suite["<STR_LIT>"] = "<STR_LIT>" % (suite["<STR_LIT:name>"], enum)<EOL>suite["<STR_LIT>"] = os.sep.join([trun["<STR_LIT>"]["<STR_LIT>"], suite["<STR_LIT>"]])<EOL>suite["<STR_LIT>"] = os.sep.join([suite["<STR_LIT>"], "<STR_LIT>"])<EOL>suite["<STR_LIT>"].update(copy.deepcopy(trun["<STR_LIT>"]))<EOL>suite["<STR_LIT>"].update(copy.deepcopy(declr.get("<STR_LIT>", {})))<EOL>os.makedirs(suite["<STR_LIT>"])<EOL>os.makedirs(suite["<STR_LIT>"])<EOL>suite["<STR_LIT>"] = hooks_setup(trun, suite, declr.get("<STR_LIT>"))<EOL>suite["<STR_LIT>"] = declr.get("<STR_LIT>", [])<EOL>suite["<STR_LIT>"] = "<STR_LIT>" % suite["<STR_LIT:name>"]<EOL>suite["<STR_LIT>"] = os.sep.join([trun["<STR_LIT>"]["<STR_LIT>"], suite["<STR_LIT>"]])<EOL>tcase_fpaths = []                               <EOL>if os.path.exists(suite["<STR_LIT>"]):              <EOL><INDENT>suite_lines = (<EOL>l.strip() for l in open(suite["<STR_LIT>"]).read().splitlines()<EOL>)<EOL>tcase_fpaths.extend(<EOL>(l for l in suite_lines if len(l) > <NUM_LIT:1> and l[<NUM_LIT:0>] != "<STR_LIT:#>")<EOL>)<EOL><DEDENT>else:                                           <EOL><INDENT>tcase_fpaths.extend(declr.get("<STR_LIT>", []))<EOL><DEDENT>if len(set(tcase_fpaths)) != len(tcase_fpaths):<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return None<EOL><DEDENT>for tcase_fname in tcase_fpaths:                <EOL><INDENT>tcase = tcase_setup(trun, suite, tcase_fname)<EOL>if not tcase:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return None<EOL><DEDENT>suite["<STR_LIT>"].append(tcase)<EOL><DEDENT>return suite<EOL>

Creates and initialized a TESTSUITE struct and site-effects such as creating
output directories and forwarding initialization of testcases


<EOL>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % tcase["<STR_LIT>"])<EOL><DEDENT>rcode = <NUM_LIT:0><EOL>for hook in reversed(tcase["<STR_LIT>"]["<STR_LIT>"]):    <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>


<EOL>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>")<EOL>cij.emph("<STR_LIT>" % tcase["<STR_LIT>"])<EOL>cij.emph("<STR_LIT>" % tcase["<STR_LIT>"])<EOL><DEDENT>rcode = <NUM_LIT:0><EOL>for hook in tcase["<STR_LIT>"]["<STR_LIT>"]:    <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>

setup res_root and aux_root, log info and run tcase-enter-hooks

@returns 0 when all hooks succeed, some value othervise


if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>")<EOL><DEDENT>rcode = <NUM_LIT:0><EOL>for hook in reversed(trun["<STR_LIT>"]["<STR_LIT>"]):    <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>


if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>")<EOL><DEDENT>trun["<STR_LIT>"]["<STR_LIT>"] = int(time.time())     <EOL>rcode = <NUM_LIT:0><EOL>for hook in trun["<STR_LIT>"]["<STR_LIT>"]:     <EOL><INDENT>rcode = script_run(trun, hook)<EOL>if rcode:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if trun["<STR_LIT>"]["<STR_LIT>"]:<EOL><INDENT>cij.emph("<STR_LIT>" % rcode, rcode)<EOL><DEDENT>return rcode<EOL>

Triggers when entering the given testrun


declr = None<EOL>try:<EOL><INDENT>with open(conf["<STR_LIT>"]) as declr_fd:<EOL><INDENT>declr = yaml.safe_load(declr_fd)<EOL><DEDENT><DEDENT>except AttributeError as exc:<EOL><INDENT>cij.err("<STR_LIT>" % exc)<EOL><DEDENT>if not declr:<EOL><INDENT>return None<EOL><DEDENT>trun = copy.deepcopy(TRUN)<EOL>trun["<STR_LIT>"] = cij.VERSION<EOL>trun["<STR_LIT>"] = copy.deepcopy(conf)<EOL>trun["<STR_LIT>"] = conf["<STR_LIT>"]<EOL>trun["<STR_LIT>"] = os.sep.join([trun["<STR_LIT>"], "<STR_LIT>"])<EOL>trun["<STR_LIT>"].update(copy.deepcopy(declr.get("<STR_LIT>", {})))<EOL>os.makedirs(trun["<STR_LIT>"])<EOL>hook_names = declr.get("<STR_LIT>", [])<EOL>if "<STR_LIT>" not in hook_names:<EOL><INDENT>hook_names = ["<STR_LIT>"] + hook_names<EOL><DEDENT>if hook_names[<NUM_LIT:0>] != "<STR_LIT>":<EOL><INDENT>return None<EOL><DEDENT>trun["<STR_LIT>"] = hooks_setup(trun, trun, hook_names)<EOL>for enum, declr in enumerate(declr["<STR_LIT>"]):  <EOL><INDENT>tsuite = tsuite_setup(trun, declr, enum)<EOL>if tsuite is None:<EOL><INDENT>cij.err("<STR_LIT>" % tsuite)<EOL>return <NUM_LIT:1><EOL><DEDENT>trun["<STR_LIT>"].append(tsuite)<EOL>trun["<STR_LIT>"]["<STR_LIT>"] += len(tsuite["<STR_LIT>"])<EOL><DEDENT>return trun<EOL>

Setup the testrunner data-structure, embedding the parsed environment
variables and command-line arguments and continues with setup for testplans,
testsuites, and testcases


fpath = yml_fpath(conf["<STR_LIT>"])<EOL>if os.path.exists(fpath):   <EOL><INDENT>cij.err("<STR_LIT>" % fpath)<EOL>return <NUM_LIT:1><EOL><DEDENT>trun = trun_setup(conf)         <EOL>if not trun:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>trun_to_file(trun)              <EOL>trun_emph(trun)                 <EOL>tr_err = <NUM_LIT:0><EOL>tr_ent_err = trun_enter(trun)<EOL>for tsuite in (ts for ts in trun["<STR_LIT>"] if not tr_ent_err):<EOL><INDENT>ts_err = <NUM_LIT:0><EOL>ts_ent_err = tsuite_enter(trun, tsuite)<EOL>for tcase in (tc for tc in tsuite["<STR_LIT>"] if not ts_ent_err):<EOL><INDENT>tc_err = tcase_enter(trun, tsuite, tcase)<EOL>if not tc_err:<EOL><INDENT>tc_err += script_run(trun, tcase)<EOL>tc_err += tcase_exit(trun, tsuite, tcase)<EOL><DEDENT>tcase["<STR_LIT:status>"] = "<STR_LIT>" if tc_err else "<STR_LIT>"<EOL>trun["<STR_LIT>"][tcase["<STR_LIT:status>"]] += <NUM_LIT:1>  <EOL>trun["<STR_LIT>"]["<STR_LIT>"] -= <NUM_LIT:1><EOL>ts_err += tc_err                        <EOL>trun_to_file(trun)                      <EOL><DEDENT>if not ts_ent_err:<EOL><INDENT>ts_err += tsuite_exit(trun, tsuite)<EOL><DEDENT>ts_err += ts_ent_err                        <EOL>tr_err += ts_err<EOL>tsuite["<STR_LIT:status>"] = "<STR_LIT>" if ts_err else "<STR_LIT>"<EOL>cij.emph("<STR_LIT>" % tsuite["<STR_LIT:status>"], tsuite["<STR_LIT:status>"] != "<STR_LIT>")<EOL><DEDENT>if not tr_ent_err:<EOL><INDENT>trun_exit(trun)<EOL><DEDENT>tr_err += tr_ent_err<EOL>trun["<STR_LIT:status>"] = "<STR_LIT>" if tr_err else "<STR_LIT>"<EOL>trun["<STR_LIT>"]["<STR_LIT:end>"] = int(time.time()) + <NUM_LIT:1>         <EOL>trun_to_file(trun)                                  <EOL>cij.emph("<STR_LIT>" % trun["<STR_LIT>"])<EOL>cij.emph("<STR_LIT>" % trun["<STR_LIT:status>"], trun["<STR_LIT:status>"] != "<STR_LIT>")<EOL>return trun["<STR_LIT>"]["<STR_LIT>"] + trun["<STR_LIT>"]["<STR_LIT>"]<EOL>


ssh = cij.env_to_dict(PREFIX, REQUIRED)<EOL>if "<STR_LIT>" in ssh:<EOL><INDENT>ssh["<STR_LIT>"] = cij.util.expand_path(ssh["<STR_LIT>"])<EOL><DEDENT>if cij.ENV.get("<STR_LIT>") is None:<EOL><INDENT>cij.ENV["<STR_LIT>"] = "<STR_LIT>"<EOL>cij.warn("<STR_LIT>" % (<EOL>cij.ENV.get("<STR_LIT>")<EOL>))<EOL><DEDENT>if cij.ENV.get("<STR_LIT>") is None:<EOL><INDENT>cij.ENV["<STR_LIT>"] = "<STR_LIT:1>"<EOL>cij.warn("<STR_LIT>" % (<EOL>cij.ENV.get("<STR_LIT>")<EOL>))<EOL><DEDENT>return <NUM_LIT:0><EOL>

Verify SSH variables and construct exported variables


def command(cmd, shell=True, echo=True, suffix=None):

if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>prefix = []<EOL>if cij.ENV.get("<STR_LIT>") == "<STR_LIT:1>":<EOL><INDENT>prefix.append("<STR_LIT>")<EOL><DEDENT>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>prefix.append("<STR_LIT>")<EOL>prefix.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>prefix.append("<STR_LIT>")<EOL>args = []<EOL>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>args.append("<STR_LIT:@>".join([cij.ENV.get("<STR_LIT>"), cij.ENV.get("<STR_LIT>")]))<EOL>wrapped = prefix + args + ["<STR_LIT>" % "<STR_LIT:U+0020>".join(cmd)]<EOL>if suffix:<EOL><INDENT>wrapped += suffix<EOL><DEDENT>return cij.util.execute(wrapped, shell, echo)<EOL>

SSH: Run the given command over SSH as defined in environment


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>args = []<EOL>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>if folder:<EOL><INDENT>args.append("<STR_LIT>")<EOL><DEDENT>target = "<STR_LIT>" % ("<STR_LIT:@>".join([cij.ENV.get("<STR_LIT>"), cij.ENV.get("<STR_LIT>")]), dst)<EOL>wrapped = ["<STR_LIT>", "<STR_LIT:U+0020>".join(args), src, target]<EOL>return cij.util.execute(wrapped, shell=True, echo=True)<EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>args = []<EOL>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>if cij.ENV.get("<STR_LIT>"):<EOL><INDENT>args.append("<STR_LIT>")<EOL>args.append(cij.ENV.get("<STR_LIT>"))<EOL><DEDENT>if folder:<EOL><INDENT>args.append("<STR_LIT>")<EOL><DEDENT>target = "<STR_LIT>" % ("<STR_LIT:@>".join([cij.ENV.get("<STR_LIT>"), cij.ENV.get("<STR_LIT>")]), src)<EOL>wrapped = ["<STR_LIT>", "<STR_LIT:U+0020>".join(args), target, dst]<EOL>return cij.util.execute(wrapped, shell=True, echo=True)<EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>timeout_backup = cij.ENV.get("<STR_LIT>")<EOL>try:<EOL><INDENT>time_start = time.time()<EOL>cij.ENV["<STR_LIT>"] = "<STR_LIT:3>"<EOL>while True:<EOL><INDENT>time_current = time.time()<EOL>if (time_current - time_start) > timeout:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>status, _, _ = command(["<STR_LIT>"], shell=True, echo=False)<EOL>if not status:<EOL><INDENT>break<EOL><DEDENT><DEDENT>cij.info("<STR_LIT>" % (time_current - time_start))<EOL><DEDENT>finally:<EOL><INDENT>if timeout_backup is None:<EOL><INDENT>del cij.ENV["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>cij.ENV["<STR_LIT>"] = timeout_backup<EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


if env():<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>timeout_backup = cij.ENV.get("<STR_LIT>")<EOL>try:<EOL><INDENT>time_start = time.time()<EOL>status, last_uptime, _ = command(["<STR_LIT>"], shell=True, echo=False)<EOL>if status:<EOL><INDENT>return <NUM_LIT:1><EOL><DEDENT>cij.ENV["<STR_LIT>"] = "<STR_LIT:3>"<EOL>cij.info("<STR_LIT>" % cij.ENV.get("<STR_LIT>"))<EOL>command(["<STR_LIT>" % extra], shell=True, echo=False)<EOL>while True:<EOL><INDENT>time_current = time.time()<EOL>if (time_current - time_start) > timeout:<EOL><INDENT>cij.err("<STR_LIT>")<EOL>return <NUM_LIT:1><EOL><DEDENT>status, current_uptime, _ = command(["<STR_LIT>"], shell=True, echo=False)<EOL>if not status and current_uptime != last_uptime:<EOL><INDENT>break<EOL><DEDENT><DEDENT>cij.info("<STR_LIT>" % (time_current - time_start))<EOL><DEDENT>finally:<EOL><INDENT>if timeout_backup is None:<EOL><INDENT>del cij.ENV["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>cij.ENV["<STR_LIT>"] = timeout_backup<EOL><DEDENT><DEDENT>return <NUM_LIT:0><EOL>


cmd = ["<STR_LIT>"]<EOL>rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)<EOL>return cij.test.FAIL if rcode else cij.test.PASS<EOL>

@returns cij.test.PASS on success and cij.test.FAIL otherwise


tree = radix.Radix()<EOL>for item in l:<EOL><INDENT>try:<EOL><INDENT>tree.add(item)<EOL><DEDENT>except (ValueError) as err:<EOL><INDENT>raise Exception("<STR_LIT>".format(item))<EOL><DEDENT><DEDENT>return aggregate_tree(tree).prefixes()<EOL>

Aggregate a `list` of prefixes.

    Keyword arguments:
    l -- a python list of prefixes

    Example use:
    >>> aggregate(["10.0.0.0/8", "10.0.0.0/24"])
    ['10.0.0.0/8']


def _aggregate_phase1(tree):<EOL><INDENT>n_tree = radix.Radix()<EOL>for prefix in tree.prefixes():<EOL><INDENT>if tree.search_worst(prefix).prefix == prefix:<EOL><INDENT>n_tree.add(prefix)<EOL><DEDENT><DEDENT>return n_tree<EOL><DEDENT>def _aggregate_phase2(tree):<EOL><INDENT>n_tree = radix.Radix()<EOL>for rnode in tree:<EOL><INDENT>p = text(ip_network(text(rnode.prefix)).supernet())<EOL>r = tree.search_covered(p)<EOL>if len(r) == <NUM_LIT:2>:<EOL><INDENT>if r[<NUM_LIT:0>].prefixlen == r[<NUM_LIT:1>].prefixlen == rnode.prefixlen:<EOL><INDENT>n_tree.add(p)<EOL><DEDENT>else:<EOL><INDENT>n_tree.add(rnode.prefix)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>n_tree.add(rnode.prefix)<EOL><DEDENT><DEDENT>return n_tree<EOL><DEDENT>l_tree = _aggregate_phase1(l_tree)<EOL>if len(l_tree.prefixes()) == <NUM_LIT:1>:<EOL><INDENT>return l_tree<EOL><DEDENT>while True:<EOL><INDENT>r_tree = _aggregate_phase2(l_tree)<EOL>if l_tree.prefixes() == r_tree.prefixes():<EOL><INDENT>break<EOL><DEDENT>else:<EOL><INDENT>l_tree = r_tree<EOL>del r_tree<EOL><DEDENT><DEDENT>return l_tree<EOL>

Walk a py-radix tree and aggregate it.

    Arguments
    l_tree -- radix.Radix() object


field_path = field_name.split('<STR_LIT:.>')<EOL>if field_path[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>context_index = <NUM_LIT:0><EOL>field_path.pop(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>context_index = -<NUM_LIT:1><EOL>while field_path[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>context_index -= <NUM_LIT:1><EOL>field_path.pop(<NUM_LIT:0>)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>field_value = context_list[context_index]<EOL>while len(field_path):<EOL><INDENT>field = field_path.pop(<NUM_LIT:0>)<EOL>if isinstance(field_value, (list, tuple, ListModel)):<EOL><INDENT>if field.isdigit():<EOL><INDENT>field = int(field)<EOL><DEDENT>field_value = field_value[field]<EOL><DEDENT>elif isinstance(field_value, dict):<EOL><INDENT>try:<EOL><INDENT>field_value = field_value[field]<EOL><DEDENT>except KeyError:<EOL><INDENT>if field.isdigit():<EOL><INDENT>field = int(field)<EOL>field_value = field_value[field]<EOL><DEDENT>else:<EOL><INDENT>field_value = None<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>field_value = getattr(field_value, field)<EOL><DEDENT><DEDENT>return field_value<EOL><DEDENT>except (IndexError, AttributeError, KeyError, TypeError):<EOL><INDENT>return None<EOL><DEDENT>

Helper to get field value from string path.
String '<context>' is used to go up on context stack. It just
can be used at the beginning of path: <context>.<context>.field_name_1
On the other hand, '<root>' is used to start lookup from first item on context.


return value<EOL>

It must be override on descendant validators


def __init__(self, error_code_map=None, error_messages=None,<EOL>message_values=None, hidden=False, *args, **kwargs):

self.error_code_map = self.error_code_map.copy()<EOL>self.error_messages = self.error_messages.copy()<EOL>self.message_values = self.message_values.copy()<EOL>if error_code_map:<EOL><INDENT>self.error_code_map.update(error_code_map)<EOL><DEDENT>if error_messages:<EOL><INDENT>self.error_messages.update(error_messages)<EOL><DEDENT>if message_values:<EOL><INDENT>self.message_values.update(message_values)<EOL><DEDENT>self.messages = {}<EOL>self.hidden = hidden<EOL>

:param error_code_map: Map of orginial error codes to custom error codes
:rparam error_code_map: dict
:param error_messages: Map of error codes to error messages
:rparam error_messages: dict
:param message_values: Map of placeholders to values
:rparam error_messages: dict


def error(self, error_code, value, **kwargs):

code = self.error_code_map.get(error_code, error_code)<EOL>try:<EOL><INDENT>message = Template(self.error_messages[code])<EOL><DEDENT>except KeyError:<EOL><INDENT>message = Template(self.error_messages[error_code])<EOL><DEDENT>placeholders = {"<STR_LIT:value>": self.hidden_value if self.hidden else value}<EOL>placeholders.update(kwargs)<EOL>placeholders.update(self.message_values)<EOL>self.messages[code] = message.safe_substitute(placeholders)<EOL>

Helper to add error to messages field. It fills placeholder with extra call parameters
or values from message_value map.

:param error_code: Error code to use
:rparam error_code: str
:param value: Value checked
:param kwargs: Map of values to use in placeholders


def __init__(self, comp_value=None, *args, **kwargs):

super(EqualTo, self).__init__(*args, **kwargs)<EOL>self.comp_value = comp_value<EOL>self.message_values.update({'<STR_LIT>': self.comp_value})<EOL>

:param comp_value: Static value to use on check


super(NotEqualTo, self).__init__(*args, **kwargs)<EOL>self.comp_value = comp_value<EOL>self.message_values.update({'<STR_LIT>': self.comp_value})<EOL>


def __init__(self, token=None, case_sensitive=True, *args, **kwargs):

super(StringNotContaining, self).__init__(*args, **kwargs)<EOL>self.token = token<EOL>self.case_sensitive = case_sensitive<EOL>self.message_values.update({'<STR_LIT>': self.token})<EOL>

:param token: Static value to see check it is contained in the string
:param case_sensitive: Boolean to check the string matching case or not


@wraps(func)<EOL>def wrapped(request, *args, **kwargs):<EOL><INDENT>if '<STR_LIT:date>' in request.GET and request.GET['<STR_LIT:date>'] == '<STR_LIT>':<EOL><INDENT>raise Http404("<STR_LIT>")<EOL><DEDENT>elif '<STR_LIT:date>' not in request.GET:<EOL><INDENT>date = datetime.today()<EOL>return func(request, date)<EOL><DEDENT>else:<EOL><INDENT>date = tuple(int(intValue) for intValue in request.GET['<STR_LIT:date>'].split('<STR_LIT:->'))<EOL>if len(date) == <NUM_LIT:3>:<EOL><INDENT>date = datetime(*date)<EOL><DEDENT>elif len(date) == <NUM_LIT:2>:<EOL><INDENT>date = datetime(*date, day = <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>date = datetime(*date, month = <NUM_LIT:1>, day = <NUM_LIT:1>)<EOL><DEDENT>return func(request, date)<EOL><DEDENT><DEDENT>return wrapped<EOL>

An decorator checking whether date parameter is passing in or not. If not, default date value is all PTT data.
            Else, return PTT data with right date.
    Args:
            func: function you want to decorate.
            request: WSGI request parameter getten from django.

    Returns:
            date:
                    a datetime variable, you can only give year, year + month or year + month + day, three type.
                    The missing part would be assigned default value 1 (for month is Jan, for day is 1).


def _dec(function):<EOL><INDENT>@wraps(function)<EOL>def _wrap(request, *args, **kwargs):<EOL><INDENT>for i in strList:<EOL><INDENT>if i not in request.GET:<EOL><INDENT>raise Http404("<STR_LIT>")<EOL><DEDENT><DEDENT>return function(request, *args, **kwargs)<EOL><DEDENT>return _wrap<EOL><DEDENT>return _dec<EOL>

An decorator checking whether queryString key is valid or not
    Args:
            str: allowed queryString key

    Returns:
            if contains invalid queryString key, it will raise exception.


def _dec(function):<EOL><INDENT>@wraps(function)<EOL>def _wrap(classInstance, request, *args, **kwargs):<EOL><INDENT>for i in strList:<EOL><INDENT>if i not in request.GET:<EOL><INDENT>raise Http404("<STR_LIT>")<EOL><DEDENT><DEDENT>return function(classInstance, request, *args, **kwargs)<EOL><DEDENT>return _wrap<EOL><DEDENT>return _dec<EOL>


jsonText = view(request)<EOL>jsonText = json.loads(jsonText.content.decode('<STR_LIT:utf-8>'))<EOL>return jsonText<EOL>

Return json from querying Web Api

            Args:
                    view: django view function.
                    request: http request object got from django.

            Returns: json format dictionary


def _set_range(self, start, stop, value, value_len):

assert stop >= start and value_len >= <NUM_LIT:0><EOL>range_len = stop - start<EOL>if range_len < value_len:<EOL><INDENT>self._insert_zeros(stop, stop + value_len - range_len)<EOL>self._copy_to_range(start, value, value_len)<EOL><DEDENT>elif range_len > value_len:<EOL><INDENT>self._del_range(stop - (range_len - value_len), stop)<EOL>self._copy_to_range(start, value, value_len)<EOL><DEDENT>else:<EOL><INDENT>self._copy_to_range(start, value, value_len)<EOL><DEDENT>

Assumes that start and stop are already in 'buffer' coordinates. value is a byte iterable.
value_len is fractional.


fields = self._all_fields()<EOL>ctx = PackContext(self, fields)<EOL>for field in fields:<EOL><INDENT>if field.pack_if.deref(ctx):<EOL><INDENT>try:<EOL><INDENT>ctx.output_buffer.set(field.pack_ref.deref(ctx), field.pack_absolute_position_ref.deref(ctx))<EOL><DEDENT>except:<EOL><INDENT>raise chain_exceptions(InstructBufferError("<STR_LIT>", ctx, type(self),<EOL>field.attr_name()))<EOL><DEDENT><DEDENT><DEDENT>result = bytearray(ctx.output_buffer.get())<EOL>static_byte_size = type(self).byte_size<EOL>if static_byte_size:<EOL><INDENT>static_byte_size = int(math.ceil(static_byte_size))<EOL>assert len(result) <= static_byte_size,("<STR_LIT>" +<EOL>"<STR_LIT>").format(type(self), len(result),<EOL>static_byte_size)<EOL>if len(result) < static_byte_size:<EOL><INDENT>result += bytearray(static_byte_size - len(result))<EOL><DEDENT><DEDENT>return result<EOL>

Packs the object and returns a buffer representing the packed object.


fields = self._all_fields()<EOL>ctx = UnpackContext(self, fields, buffer)<EOL>for field in fields:<EOL><INDENT>try:<EOL><INDENT>if field.unpack_if.deref(ctx):<EOL><INDENT>for prev_field in field.unpack_after:<EOL><INDENT>prev_field.unpack_value_ref.deref(ctx)<EOL><DEDENT>field.unpack_value_ref.deref(ctx)<EOL><DEDENT>else:<EOL><INDENT>setattr(self, field.attr_name(), None)<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise chain_exceptions(InstructBufferError("<STR_LIT>", ctx, type(self), field.attr_name()))<EOL><DEDENT><DEDENT>return self.calc_byte_size(ctx)<EOL>

Unpacks the object's fields from buffer.


if ctx is None:<EOL><INDENT>ctx = PackContext(self, type(self).__fields__)<EOL><DEDENT>return TotalSizeReference().deref(ctx)<EOL>

Returns this instance's size. If the size has to be calculated it may require packing some of the fields.


self.start = start if start is not None else <NUM_LIT:0><EOL>self.stop = stop<EOL>assert self.start >= <NUM_LIT:0>, "<STR_LIT>".format(self.start)<EOL>if self.stop is not None:<EOL><INDENT>assert self.stop >= <NUM_LIT:0>, "<STR_LIT>".format(self.stop)<EOL>assert self.start <= self.stop, "<STR_LIT>".format(self.start, self.stop)<EOL><DEDENT>

:param start: range start (non-negative)
:type start: int or float
:param stop: range stop (can be None for open range). If not None, must be >= start.
:type stop: int, float or None


return self.stop is None<EOL>

:returns: True if the range is open, i.e. stop is None
:rtype: bool


assert new_stop >= self.start, "<STR_LIT>".format(self.start, new_stop)<EOL>return SequentialRange(self.start, self.stop if self.stop is not None else new_stop)<EOL>

:param new_stop: new stop value to use if existing stop is None. Must be >= start.
:type new_stop: int or float
:returns: new closed range, using the current stop if self is a closed range or new_stop if self is open.
:rtype: SequentialRange


if self.is_open():<EOL><INDENT>return None<EOL><DEDENT>return self.stop - self.start<EOL>

:returns: length of range if a closed range or None if open range.
:rtype: int or float or None


return self.stop<EOL>


return slice(self.start, self.stop, <NUM_LIT:1>)<EOL>

:returns: a slice object from start to stop
:rtype: slice


a, b = (self, other) if self.start <= other.start else (other, self)<EOL>a_imag_stop = a.to_closed(sys.maxsize).stop<EOL>return a_imag_stop > b.start<EOL>

:returns: True if there's an overlap (non-empty intersection) between two ranges
:rtype: bool


return self.start <= point and (self.stop is None or self.stop > point)<EOL>

:param point: point to check if contained in range
:type point: int or float
:returns: True if the range contains `point`
:rtype: bool


sum = <NUM_LIT:0><EOL>for r in self:<EOL><INDENT>if r.is_open():<EOL><INDENT>return None<EOL><DEDENT>sum += r.byte_length()<EOL><DEDENT>return sum<EOL>

:returns: sum of lengthes of all ranges or None if one of the ranges is open
:rtype: int, float or None


sorted_list = sorted(self)<EOL>for i in range(<NUM_LIT:0>, len(sorted_list) - <NUM_LIT:1>):<EOL><INDENT>if sorted_list[i].overlaps(sorted_list[i + <NUM_LIT:1>]):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

:returns: True if one or more range in the list overlaps with another
:rtype: bool


m = <NUM_LIT:0><EOL>for r in self:<EOL><INDENT>if r.is_open():<EOL><INDENT>return None<EOL><DEDENT>m = max(m, r.stop)<EOL><DEDENT>return m<EOL>

:returns: maximum stop in list or None if there's at least one open range
:type: int, float or None


return SequentialRangeList(sorted(self))<EOL>

:returns: a new sorted SequentialRangeList
:rtype: SequentialRangeList


remaining_bytes = bytes<EOL>for r in self:<EOL><INDENT>if r.is_open() or r.byte_length() >= remaining_bytes:<EOL><INDENT>return r.start + remaining_bytes<EOL><DEDENT>else:<EOL><INDENT>remaining_bytes -= r.byte_length()<EOL><DEDENT><DEDENT>assert False, "<STR_LIT>".format(bytes, self)<EOL>

Maps `bytes` length to a sequence's offset. For example, if we do byte_offset(5) and our list of sequences is
[(0, 2), (10, 11), (40, 45)] then the returned value will be 42.
Note that `bytes` must be <= byte_length().
:returns: actual offset in one of the sequences in the range for request byte length.
:rtype: int or float


if self in ctx.call_nodes:<EOL><INDENT>raise CyclicReferenceError(ctx, self)<EOL><DEDENT>if self in ctx.cached_results:<EOL><INDENT>return ctx.cached_results[self]<EOL><DEDENT>try:<EOL><INDENT>ctx.call_nodes.add(self)<EOL>ctx.call_stack.append(self)<EOL>result = self.evaluate(ctx)<EOL>ctx.cached_results[self] = result<EOL>return result<EOL><DEDENT>except:<EOL><INDENT>if ctx.exception_call_stack is None:<EOL><INDENT>ctx.exception_call_stack = list(ctx.call_stack)<EOL><DEDENT>raise<EOL><DEDENT>finally:<EOL><INDENT>ctx.call_stack.pop()<EOL>ctx.call_nodes.remove(self)<EOL><DEDENT>

Returns the value this reference is pointing to. This method uses 'ctx' to resolve the reference and return
the value this reference references.
If the call was already made, it returns a cached result.
It also makes sure there's no cyclic reference, and if so raises CyclicReferenceError.


def __new__(cls, min_val_or_tuple=<NUM_LIT:0>, max_val=sys.maxsize):

if isinstance(min_val_or_tuple, (list, tuple, MinMax)):<EOL><INDENT>assert len(min_val_or_tuple) == <NUM_LIT:2><EOL>min_val, max_val = min_val_or_tuple<EOL><DEDENT>else:<EOL><INDENT>min_val = min_val_or_tuple<EOL><DEDENT>assert min_val <= max_val<EOL>return super(MinMax, cls).__new__(cls, max(<NUM_LIT:0>, min_val), min(max_val, sys.maxsize))<EOL>

Initialize a new MinMax instance. This initializer works in four different modes:
 * Copy constructor. If you pass it a MinMax object it'll copy the min/max values.
 * Construct from tuple/list. Construct the min/max by accessing [0] and [1].
 * Construct from a single argument. User provides the min value, max is set to sys.maxsize.
 * Construct from two arguments. User provides the min and max values.


def create_from_string(self, string, context=EMPTY_CONTEXT, *args, **kwargs):

if not PY2 and not isinstance(string, bytes):<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>io = StringIO(string)<EOL>instance = self.create_from_stream(io, context, *args, **kwargs)<EOL>io.close()<EOL>return instance<EOL>

Deserializes a new instance from a string.
This is a convenience method that creates a StringIO object and calls create_instance_from_stream().


try:<EOL><INDENT>obj_repr = repr(obj)<EOL><DEDENT>except:<EOL><INDENT>obj_repr = "<STR_LIT>".format(type(obj), id(obj))<EOL><DEDENT>return obj_repr<EOL>

Returns a repr of an object and falls back to a minimal representation of type and ID if the call to repr raised
    an error.

    :param obj: object to safe repr
    :returns: repr string or '(type<id> repr error)' string
    :rtype: str


def keep_kwargs_partial(func, *args, **keywords):

def newfunc(*fargs, **fkeywords):<EOL><INDENT>newkeywords = fkeywords.copy()<EOL>newkeywords.update(keywords)<EOL>return func(*(args + fargs), **newkeywords)<EOL><DEDENT>newfunc.func = func<EOL>newfunc.args = args<EOL>newfunc.keywords = keywords<EOL>return newfunc<EOL>

Like functools.partial but instead of using the new kwargs, keeps the old ones.


def ConstField(name, value, marshal=None):

if marshal is None:<EOL><INDENT>marshal = value<EOL><DEDENT>if isinstance(marshal, Struct):<EOL><INDENT>marshal = type(marshal)<EOL><DEDENT>elif not isinstance(marshal, Marshal):<EOL><INDENT>raise InstructError("<STR_LIT>" %<EOL>(name, value))<EOL><DEDENT>return OrigConstField(name, marshal, value)<EOL>

This macro can be used in several methods:

>>> ConstField("foo", 5, UBInt8)

This created a constant field called ``foo`` with a value of 5 and is serialized/deserialized using UBInt8.

>>> ConstField("foo", MyStruct(my_field=1, my_other_field=2))

This time ``foo`` is set with the ``MyStruct`` instance passed here. Notice that we don't need to pass an I/O
argument because the value is an I/O instance by itself.

:param name: name of the field
:param value: the value to use as a constant
:param marshal: a marshal instance to serialize/deserialize this field (optional if ``value`` is a marshal)
:rtype: Field


def generate_image(source, outname, settings, options=None):

logger = logging.getLogger(__name__)<EOL>if settings['<STR_LIT>'] or source.endswith('<STR_LIT>'):<EOL><INDENT>utils.copy(source, outname, symlink=settings['<STR_LIT>'])<EOL>return<EOL><DEDENT>img = _read_image(source)<EOL>original_format = img.format<EOL>if settings['<STR_LIT>'] and settings['<STR_LIT>']:<EOL><INDENT>logger.warning("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if settings['<STR_LIT>'] and _has_exif_tags(img):<EOL><INDENT>if options is not None:<EOL><INDENT>options = deepcopy(options)<EOL><DEDENT>else:<EOL><INDENT>options = {}<EOL><DEDENT>options['<STR_LIT>'] = img.info['<STR_LIT>']<EOL><DEDENT>if settings['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>img = Transpose().process(img)<EOL><DEDENT>except (IOError, IndexError):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if settings['<STR_LIT>']:<EOL><INDENT>try:<EOL><INDENT>logger.debug('<STR_LIT>', settings['<STR_LIT>'])<EOL>processor_cls = getattr(pilkit.processors,<EOL>settings['<STR_LIT>'])<EOL><DEDENT>except AttributeError:<EOL><INDENT>logger.error('<STR_LIT>', settings['<STR_LIT>'])<EOL>sys.exit()<EOL><DEDENT>width, height = settings['<STR_LIT>']<EOL>if img.size[<NUM_LIT:0>] < img.size[<NUM_LIT:1>]:<EOL><INDENT>height, width = width, height<EOL><DEDENT>processor = processor_cls(width, height, upscale=False)<EOL>img = processor.process(img)<EOL><DEDENT>for receiver in signals.img_resized.receivers_for(img):<EOL><INDENT>img = receiver(img, settings=settings)<EOL><DEDENT>outformat = img.format or original_format or '<STR_LIT>'<EOL>logger.debug('<STR_LIT>', outname, outformat)<EOL>save_image(img, outname, outformat, options=options, autoconvert=True)<EOL>

Image processor, rotate and resize the image.

    :param source: path to an image
    :param outname: output filename
    :param settings: settings dict
    :param options: dict with PIL options (quality, optimize, progressive)


def generate_thumbnail(source, outname, box, fit=True, options=None,<EOL>thumb_fit_centering=(<NUM_LIT:0.5>, <NUM_LIT:0.5>)):

logger = logging.getLogger(__name__)<EOL>img = _read_image(source)<EOL>original_format = img.format<EOL>if fit:<EOL><INDENT>img = ImageOps.fit(img, box, PILImage.ANTIALIAS,<EOL>centering=thumb_fit_centering)<EOL><DEDENT>else:<EOL><INDENT>img.thumbnail(box, PILImage.ANTIALIAS)<EOL><DEDENT>outformat = img.format or original_format or '<STR_LIT>'<EOL>logger.debug('<STR_LIT>', outname, outformat)<EOL>save_image(img, outname, outformat, options=options, autoconvert=True)<EOL>


def process_image(filepath, outpath, settings):

logger = logging.getLogger(__name__)<EOL>logger.info('<STR_LIT>', filepath)<EOL>filename = os.path.split(filepath)[<NUM_LIT:1>]<EOL>outname = os.path.join(outpath, filename)<EOL>ext = os.path.splitext(filename)[<NUM_LIT:1>]<EOL>if ext in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>options = settings['<STR_LIT>']<EOL><DEDENT>elif ext == '<STR_LIT>':<EOL><INDENT>options = {'<STR_LIT>': True}<EOL><DEDENT>else:<EOL><INDENT>options = {}<EOL><DEDENT>try:<EOL><INDENT>generate_image(filepath, outname, settings, options=options)<EOL>if settings['<STR_LIT>']:<EOL><INDENT>thumb_name = os.path.join(outpath, get_thumb(settings, filename))<EOL>generate_thumbnail(<EOL>outname, thumb_name, settings['<STR_LIT>'],<EOL>fit=settings['<STR_LIT>'], options=options,<EOL>thumb_fit_centering=settings["<STR_LIT>"])<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.info('<STR_LIT>', e)<EOL>if logger.getEffectiveLevel() == logging.DEBUG:<EOL><INDENT>raise<EOL><DEDENT>else:<EOL><INDENT>return Status.FAILURE<EOL><DEDENT><DEDENT>return Status.SUCCESS<EOL>

Process one image: resize, create thumbnail.


try:<EOL><INDENT>im = _read_image(file_path)<EOL><DEDENT>except (IOError, IndexError, TypeError, AttributeError) as e:<EOL><INDENT>logger = logging.getLogger(__name__)<EOL>logger.error("<STR_LIT>", file_path, e)<EOL><DEDENT>else:<EOL><INDENT>width, height = im.size<EOL>return {<EOL>'<STR_LIT:width>': width,<EOL>'<STR_LIT>': height<EOL>}<EOL><DEDENT>


logger = logging.getLogger(__name__)<EOL>img = _read_image(filename)<EOL>try:<EOL><INDENT>exif = img._getexif() or {}<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>return None<EOL><DEDENT>data = {TAGS.get(tag, tag): value for tag, value in exif.items()}<EOL>if '<STR_LIT>' in data:<EOL><INDENT>try:<EOL><INDENT>data['<STR_LIT>'] = {GPSTAGS.get(tag, tag): value<EOL>for tag, value in data['<STR_LIT>'].items()}<EOL><DEDENT>except AttributeError:<EOL><INDENT>logger = logging.getLogger(__name__)<EOL>logger.info('<STR_LIT>')<EOL>del data['<STR_LIT>']<EOL><DEDENT><DEDENT>return data<EOL>


logger = logging.getLogger(__name__)<EOL>iptc_data = {}<EOL>raw_iptc = {}<EOL>try:<EOL><INDENT>img = _read_image(filename)<EOL>raw_iptc = IptcImagePlugin.getiptcinfo(img)<EOL><DEDENT>except SyntaxError:<EOL><INDENT>logger.info('<STR_LIT>', filename)<EOL><DEDENT>if raw_iptc and (<NUM_LIT:2>, <NUM_LIT:5>) in raw_iptc:<EOL><INDENT>iptc_data["<STR_LIT:title>"] = raw_iptc[(<NUM_LIT:2>, <NUM_LIT:5>)].decode('<STR_LIT:utf-8>', errors='<STR_LIT:replace>')<EOL><DEDENT>if raw_iptc and (<NUM_LIT:2>, <NUM_LIT>) in raw_iptc:<EOL><INDENT>iptc_data["<STR_LIT:description>"] = raw_iptc[(<NUM_LIT:2>, <NUM_LIT>)].decode('<STR_LIT:utf-8>',<EOL>errors='<STR_LIT:replace>')<EOL><DEDENT>if raw_iptc and (<NUM_LIT:2>, <NUM_LIT>) in raw_iptc:<EOL><INDENT>iptc_data["<STR_LIT>"] = raw_iptc[(<NUM_LIT:2>, <NUM_LIT>)].decode('<STR_LIT:utf-8>',<EOL>errors='<STR_LIT:replace>')<EOL><DEDENT>return iptc_data<EOL>


d = float(v[<NUM_LIT:0>][<NUM_LIT:0>]) / float(v[<NUM_LIT:0>][<NUM_LIT:1>])<EOL>m = float(v[<NUM_LIT:1>][<NUM_LIT:0>]) / float(v[<NUM_LIT:1>][<NUM_LIT:1>])<EOL>s = float(v[<NUM_LIT:2>][<NUM_LIT:0>]) / float(v[<NUM_LIT:2>][<NUM_LIT:1>])<EOL>return d + (m / <NUM_LIT>) + (s / <NUM_LIT>)<EOL>

Convert degree/minute/second to decimal degrees.


logger = logging.getLogger(__name__)<EOL>simple = {}<EOL>for tag in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if tag in data:<EOL><INDENT>if isinstance(data[tag], tuple):<EOL><INDENT>simple[tag] = data[tag][<NUM_LIT:0>].strip()<EOL><DEDENT>else:<EOL><INDENT>simple[tag] = data[tag].strip()<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>fnumber = data['<STR_LIT>']<EOL>try:<EOL><INDENT>simple['<STR_LIT>'] = float(fnumber[<NUM_LIT:0>]) / fnumber[<NUM_LIT:1>]<EOL><DEDENT>except Exception:<EOL><INDENT>logger.debug('<STR_LIT>', fnumber, exc_info=True)<EOL><DEDENT><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>focal = data['<STR_LIT>']<EOL>try:<EOL><INDENT>simple['<STR_LIT>'] = round(float(focal[<NUM_LIT:0>]) / focal[<NUM_LIT:1>])<EOL><DEDENT>except Exception:<EOL><INDENT>logger.debug('<STR_LIT>', focal,<EOL>exc_info=True)<EOL><DEDENT><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>exptime = data['<STR_LIT>']<EOL>if isinstance(exptime, tuple):<EOL><INDENT>try:<EOL><INDENT>simple['<STR_LIT>'] = str(fractions.Fraction(exptime[<NUM_LIT:0>],<EOL>exptime[<NUM_LIT:1>]))<EOL><DEDENT>except ZeroDivisionError:<EOL><INDENT>logger.info('<STR_LIT>', exptime)<EOL><DEDENT><DEDENT>elif isinstance(exptime, int):<EOL><INDENT>simple['<STR_LIT>'] = str(exptime)<EOL><DEDENT>else:<EOL><INDENT>logger.info('<STR_LIT>', exptime)<EOL><DEDENT><DEDENT>if data.get('<STR_LIT>'):<EOL><INDENT>simple['<STR_LIT>'] = data['<STR_LIT>']<EOL><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>date = data['<STR_LIT>'].rsplit('<STR_LIT:\x00>')[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>simple['<STR_LIT>'] = datetime.strptime(date, '<STR_LIT>')<EOL>simple['<STR_LIT>'] = simple['<STR_LIT>'].strftime(datetime_format)<EOL><DEDENT>except (ValueError, TypeError) as e:<EOL><INDENT>logger.info('<STR_LIT>', e)<EOL><DEDENT><DEDENT>if '<STR_LIT>' in data:<EOL><INDENT>info = data['<STR_LIT>']<EOL>lat_info = info.get('<STR_LIT>')<EOL>lon_info = info.get('<STR_LIT>')<EOL>lat_ref_info = info.get('<STR_LIT>')<EOL>lon_ref_info = info.get('<STR_LIT>')<EOL>if lat_info and lon_info and lat_ref_info and lon_ref_info:<EOL><INDENT>try:<EOL><INDENT>lat = dms_to_degrees(lat_info)<EOL>lon = dms_to_degrees(lon_info)<EOL><DEDENT>except (ZeroDivisionError, ValueError, TypeError):<EOL><INDENT>logger.info('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>simple['<STR_LIT>'] = {<EOL>'<STR_LIT>': - lat if lat_ref_info != '<STR_LIT:N>' else lat,<EOL>'<STR_LIT>': - lon if lon_ref_info != '<STR_LIT:E>' else lon,<EOL>}<EOL><DEDENT><DEDENT><DEDENT>return simple<EOL>

Make a simplified version with common tags from raw EXIF data.


logger = logging.getLogger(name)<EOL>logger.setLevel(level)<EOL>try:<EOL><INDENT>if os.isatty(sys.stdout.fileno()) andnot sys.platform.startswith('<STR_LIT>'):<EOL><INDENT>formatter = ColoredFormatter()<EOL><DEDENT>elif level == logging.DEBUG:<EOL><INDENT>formatter = Formatter('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>formatter = Formatter('<STR_LIT>')<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>formatter = Formatter('<STR_LIT>')<EOL><DEDENT>handler = logging.StreamHandler()<EOL>handler.setFormatter(formatter)<EOL>logger.addHandler(handler)<EOL>

Logging config

    Set the level and create a more detailed formatter for debug mode.


if not hasattr(album.gallery, "<STR_LIT>"):<EOL><INDENT>_restore_cache(album.gallery)<EOL><DEDENT>cache = album.gallery.exifCache<EOL>for media in album.medias:<EOL><INDENT>if media.type == "<STR_LIT:image>":<EOL><INDENT>key = os.path.join(media.path, media.filename)<EOL>if key in cache:<EOL><INDENT>media.exif = cache[key]<EOL><DEDENT><DEDENT><DEDENT>

Loads the exif data of all images in an album from cache


cachePath = os.path.join(gallery.settings["<STR_LIT>"], "<STR_LIT>")<EOL>try:<EOL><INDENT>if os.path.exists(cachePath):<EOL><INDENT>with open(cachePath, "<STR_LIT:rb>") as cacheFile:<EOL><INDENT>gallery.exifCache = pickle.load(cacheFile)<EOL>logger.debug("<STR_LIT>", len(gallery.exifCache))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>gallery.exifCache = {}<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.warn("<STR_LIT>", e)<EOL>gallery.exifCache = {}<EOL><DEDENT>

Restores the exif data cache from the cache file


if hasattr(gallery, "<STR_LIT>"):<EOL><INDENT>cache = gallery.exifCache<EOL><DEDENT>else:<EOL><INDENT>cache = gallery.exifCache = {}<EOL><DEDENT>for album in gallery.albums.values():<EOL><INDENT>for image in album.images:<EOL><INDENT>cache[os.path.join(image.path, image.filename)] = image.exif<EOL><DEDENT><DEDENT>cachePath = os.path.join(gallery.settings["<STR_LIT>"], "<STR_LIT>")<EOL>if len(cache) == <NUM_LIT:0>:<EOL><INDENT>if os.path.exists(cachePath):<EOL><INDENT>os.remove(cachePath)<EOL><DEDENT>return<EOL><DEDENT>try:<EOL><INDENT>with open(cachePath, "<STR_LIT:wb>") as cacheFile:<EOL><INDENT>pickle.dump(cache, cacheFile)<EOL>logger.debug("<STR_LIT>", len(gallery.exifCache))<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>logger.warn("<STR_LIT>", e)<EOL>os.remove(cachePath)<EOL><DEDENT>

Stores the exif data of all images in the gallery


nomediapath = os.path.join(album.src_path, "<STR_LIT>")<EOL>if os.path.isfile(nomediapath):<EOL><INDENT>if os.path.getsize(nomediapath) == <NUM_LIT:0>:<EOL><INDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT>", album.name)<EOL>_remove_albums_with_subdirs(album.gallery.albums, [album.path])<EOL>try:<EOL><INDENT>os.rmdir(album.dst_path)<EOL><DEDENT>except OSError as e:<EOL><INDENT>pass<EOL><DEDENT>album.subdirs = []<EOL>album.medias = []<EOL><DEDENT>else:<EOL><INDENT>with open(nomediapath, "<STR_LIT:r>") as nomediaFile:<EOL><INDENT>logger.info("<STR_LIT>"<EOL>"<STR_LIT>", album.name)<EOL>ignored = nomediaFile.read().split("<STR_LIT:\n>")<EOL>album.medias = [media for media in album.medias<EOL>if media.src_filename not in ignored]<EOL>album.subdirs = [dirname for dirname in album.subdirs<EOL>if dirname not in ignored]<EOL>_remove_albums_with_subdirs(album.gallery.albums,<EOL>ignored, album.path + os.path.sep)<EOL><DEDENT><DEDENT><DEDENT>

Removes all filtered Media and subdirs from an Album


def do_compress(self, filename, compressed_filename):

raise NotImplementedError<EOL>

Perform actual compression.
This should be implemented by subclasses.


compressed_filename = self.get_compressed_filename(filename)<EOL>if not compressed_filename:<EOL><INDENT>return<EOL><DEDENT>self.do_compress(filename, compressed_filename)<EOL>