jiahuimbzuai/CoQuIR huggingface dataset:
// gen_signed_cert [python]
def gen_signed_cert(domain, ca_crt, ca_key, ca_pass, certs_folder):
    """
    This function takes a domain name as a parameter and then creates a certificate and key with the
    domain name(replacing dots by underscores), finally signing the certificate using specified CA and
    returns the path of key and cert files. If you are yet to generate a CA then check the top comments
    """
    key_path = os.path.join(certs_folder, re.sub('[^-0-9a-zA-Z_]', '_', domain) + ".key")
    cert_path = os.path.join(certs_folder, re.sub('[^-0-9a-zA-Z_]', '_', domain) + ".crt")

    # The first conditions checks if file exists, and does nothing if true
    # If file doenst exist lock is obtained for writing (Other processes in race must wait)
    # After obtaining lock another check to handle race conditions gracefully
    if os.path.exists(key_path) and os.path.exists(cert_path):
        pass
    else:
        with FileLock(cert_path, timeout=2):
            # Check happens if the certificate and key pair already exists for a domain
            if os.path.exists(key_path) and os.path.exists(cert_path):
                pass
            else:
                # Serial Generation - Serial number must be unique for each certificate,
                # so serial is generated based on domain name
                md5_hash = hashlib.md5()
                md5_hash.update(domain)
                serial = int(md5_hash.hexdigest(), 36)

                # The CA stuff is loaded from the same folder as this script
                ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, open(ca_crt).read())
                # The last parameter is the password for your CA key file
                ca_key = crypto.load_privatekey(crypto.FILETYPE_PEM, open(ca_key).read(), ca_pass)

                key = crypto.PKey()
                key.generate_key(crypto.TYPE_RSA, 2048)

                cert = crypto.X509()
                cert.get_subject().C = "IN"
                cert.get_subject().ST = "AP"
                cert.get_subject().L = "127.0.0.1"
                cert.get_subject().O = "OWTF"
                cert.get_subject().OU = "Inbound-Proxy"
                cert.get_subject().CN = domain
                cert.gmtime_adj_notBefore(0)
                cert.gmtime_adj_notAfter(365 * 24 * 60 * 60)
                cert.set_serial_number(serial)
                cert.set_issuer(ca_cert.get_subject())
                cert.set_pubkey(key)
                cert.sign(ca_key, "sha1")

                # The key and cert files are dumped and their paths are returned
                domain_key = open(key_path, "w")
                domain_key.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, key))

                domain_cert = open(cert_path, "w")
                domain_cert.write(crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
    return key_path, cert_path
// register_user [python]
def register_user(request):
    settings = request.registry.settings
    if not is_registration_enabled(settings):
        raise exc.exception_response(503)
    handle_history(request)
    _ = request.translate
    config = Config(load(get_path_to_form_config('auth.xml', 'ringo')))
    form_config = config.get_form('register_user')
    form = Form(form_config, csrf_token=request.session.get_csrf_token())
    # Do extra validation which is not handled by formbar.
    # Is the login unique?
    validator = Validator('login',
                          'There is already a user with this name',
                          is_login_unique)
    form.add_validator(validator)
    if request.POST:
        if form.validate(request.params.mixed()):
            # 1. Create user. Do not activate him. Default role is user.
            ufac = User.get_item_factory()
            # TODO: Check why we not use the get_item_factory_method
            # here. Do we use plain factories because the need full
            # controll of depended relations? (ti) <2014-04-08 17:07> 
            pfac = BaseFactory(Profile)
            gfac = BaseFactory(Usergroup)
            user = ufac.create(None)
            # Set login from formdata
            user.login = form.data['login']
            # Encrypt password and save
            user.password = encrypt_password(form.data['pass'])
            # Deactivate the user. To activate the user needs to confirm
            # with the activation link
            user.activated = False
            atoken = str(uuid.uuid4())
            user.activation_token = atoken
            # Set profile data
            user.profile[0].email = form.data['email']
            # Set user group
            group = gfac.load(USER_GROUP_ID)
            user.groups.append(group)
            # Set default user group.
            user.gid = group.id
            DBSession.add(user)

            # 3. Send confirmation email. The user will be activated
            #    after the user clicks on the confirmation link
            mailer = Mailer(request)
            recipient = user.profile[0].email
            subject = _('Confirm user registration for %s' % get_app_name())
            values = {'url': request.route_url('confirm_user', token=atoken),
                      'app_name': get_app_name(),
                      'email': settings['mail.default_sender'],
                      '_': _}
            mail = Mail([recipient], subject, template="register_user", values=values)
            mailer.send(mail)

            target_url = request.route_path('login')
            headers = forget(request)
            msg = _("User has been created and a confirmation mail was sent"
                    " to the users email adress. Please check your email :)")
            request.session.flash(msg, 'success')
            return HTTPFound(location=target_url, headers=headers)
    return {'form': form.render()}
// register_user [python]
def register_user(request):
    settings = request.registry.settings
    if not is_registration_enabled(settings):
        raise exc.exception_response(503)
    handle_history(request)
    _ = request.translate
    config = Config(load(get_path_to_form_config('auth.xml', 'ringo')))
    form_config = config.get_form('register_user')
    form = Form(form_config, csrf_token=request.session.get_csrf_token())
    # Do extra validation which is not handled by formbar.
    # Is the login unique?
    validator = Validator('login',
                          'There is already a user with this name',
                          is_login_unique)
    form.add_validator(validator)
    if request.POST:
        if form.validate(request.params.mixed()):
            # 1. Create user. Do not activate him. Default role is user.
            ufac = User.get_item_factory()
            # TODO: Check why we not use the get_item_factory_method
            # here. Do we use plain factories because the need full
            # controll of depended relations? (ti) <2014-04-08 17:07> 
            pfac = BaseFactory(Profile)
            gfac = BaseFactory(Usergroup)
            user = ufac.create(None)
            # Set login from formdata
            user.login = form.data['login']
            # Encrypt password and save
            pw = hashlib.md5()
            pw.update(form.data['pass'])
            user.password = pw.hexdigest()
            # Deactivate the user. To activate the user needs to confirm
            # with the activation link
            user.activated = False
            atoken = str(uuid.uuid4())
            user.activation_token = atoken
            # Set profile data
            user.profile[0].email = form.data['email']
            # Set user group
            group = gfac.load(USER_GROUP_ID)
            user.groups.append(group)
            # Set default user group.
            user.gid = group.id
            DBSession.add(user)

            # 3. Send confirmation email. The user will be activated
            #    after the user clicks on the confirmation link
            mailer = Mailer(request)
            recipient = user.profile[0].email
            subject = _('Confirm user registration for %s' % get_app_name())
            values = {'url': request.route_url('confirm_user', token=atoken),
                      'app_name': get_app_name(),
                      'email': settings['mail.default_sender'],
                      '_': _}
            mail = Mail([recipient], subject, template="register_user", values=values)
            mailer.send(mail)

            target_url = request.route_path('login')
            headers = forget(request)
            msg = _("User has been created and a confirmation mail was sent"
                    " to the users email adress. Please check your email :)")
            request.session.flash(msg, 'success')
            return HTTPFound(location=target_url, headers=headers)
    return {'form': form.render()}
// read_mdisk [c]
static int read_mdisk(metric_disk *mdisk)
{
   mdisk_header md_header;
   uint32_t busy;
   uint32_t sig;
   int fd;
   char *path;

   DIR* dir;
   struct dirent* entry;

   dir = opendir(SYS_BLOCK);
   if (dir == NULL)
      goto error;

   while((entry = readdir(dir))) {
retry:
#ifndef DEBUG_FROM_DOM0
      if (strcmp(entry->d_name, ".") == 0 ||
            strcmp(entry->d_name, "..") == 0)
         continue;

      if (asprintf(&path, "/dev/%s", entry->d_name) < 0)
          goto error;
#else
      path = strdup("/dev/shm/vhostmd0");
#endif
      /* Open with O_DIRECT to avoid kernel keeping old copies around
       * in the cache.
       */
      fd = open (path, O_RDONLY|O_DIRECT);
      if (fd == -1) {
         free (path);
         continue;
      }
      if (odirect_read (fd, &md_header, 0, sizeof md_header) == -1) {
         free (path);
	 close (fd);
         continue;
      }

      if ((sig = ntohl(md_header.sig)) == MDISK_SIGNATURE) {
         busy = ntohl(md_header.busy);
         if (busy) {
	     close(fd);
             free(path);
             sleep(1);
             goto retry;
         }
         mdisk->sum = ntohl(md_header.sum);
         mdisk->length = ntohl(md_header.length);
         mdisk->buffer = malloc(mdisk->length);
         mdisk->disk_name = strdup(path);
	 /* XXX check return value */
         odirect_read (fd, mdisk->buffer, sizeof md_header, mdisk->length);
	 free(path);

         /* Verify data still valid */
         if (odirect_read (fd, &md_header, 0, sizeof md_header) == -1) {
	     mdisk_content_free();
             close (fd);
             sleep(1);
             goto retry;
         }
         busy = ntohl(md_header.busy);
         if (busy || mdisk->sum != ntohl(md_header.sum)) {
             mdisk_content_free();
             close (fd);
             sleep(1);
             goto retry;
         }
         close (fd);
	 break;
      }
      close (fd);
   }

   if (mdisk->buffer == NULL)
      goto error;

   /* Set up a parser context */
   mdisk->pctxt = xmlNewParserCtxt();
   if (!mdisk->pctxt || !mdisk->pctxt->sax) {
      goto error;
   }

   mdisk->doc = xmlCtxtReadMemory(mdisk->pctxt, mdisk->buffer, 
                                  mdisk->length, "mdisk.xml", NULL, 
                                  XML_PARSE_NONET | XML_PARSE_NOWARNING);
   if (!mdisk->doc) {
      libmsg("%s(): libxml failed to parse mdisk.xml buffer\n", __func__);
      goto error;
   }

   closedir(dir);

   return 0;
error:
   if (dir)
       closedir(dir);

   libmsg("%s(): Unable to read metrics disk\n", __func__);

   return -1;
}
// read_mdisk [c]
static int read_mdisk(metric_disk *mdisk)
{
   mdisk_header md_header;
   uint32_t busy;
   uint32_t sig;
   int fd;
   char *path;

   DIR* dir;
   struct dirent* entry;

   dir = opendir(SYS_BLOCK);
   if (dir == NULL)
      goto error;

   while((entry = readdir(dir))) {
retry:
#ifndef DEBUG_FROM_DOM0
      if (strcmp(entry->d_name, ".") == 0 ||
            strcmp(entry->d_name, "..") == 0)
         continue;

      if (asprintf(&path, "/dev/%s", entry->d_name) < 0)
          goto error;
#else
      path = strdup("/dev/shm/vhostmd0");
#endif
      /* Open with O_DIRECT to avoid kernel keeping old copies around
       * in the cache.
       */
      fd = open (path, O_RDONLY|O_DIRECT);
      if (fd == -1) {
         free (path);
         continue;
      }
      if (odirect_read (fd, &md_header, 0, sizeof md_header) == -1) {
         free (path);
	 close (fd);
         continue;
      }

      if ((sig = ntohl(md_header.sig)) == MDISK_SIGNATURE) {
         busy = ntohl(md_header.busy);
         if (busy) {
	     close(fd);
             free(path);
             sleep(1);
             goto retry;
         }
         mdisk->sum = ntohl(md_header.sum);
         mdisk->length = ntohl(md_header.length);
         mdisk->buffer = malloc(mdisk->length);
         mdisk->disk_name = strdup(path);
	 /* XXX check return value */
         odirect_read (fd, mdisk->buffer, sizeof md_header, mdisk->length);
	 free(path);

         /* Verify data still valid */
         if (odirect_read (fd, &md_header, 0, sizeof md_header) == -1) {
	     mdisk_content_free();
             close (fd);
             sleep(1);
             goto retry;
         }
         busy = ntohl(md_header.busy);
         if (busy || mdisk->sum != ntohl(md_header.sum)) {
             mdisk_content_free();
             close (fd);
             sleep(1);
             goto retry;
         }
         close (fd);
	 break;
      }
      close (fd);
   }

   if (mdisk->buffer == NULL)
      goto error;

   /* Set up a parser context */
   mdisk->pctxt = xmlNewParserCtxt();
   if (!mdisk->pctxt || !mdisk->pctxt->sax) {
      goto error;
   }

   mdisk->doc = xmlCtxtReadMemory(mdisk->pctxt, mdisk->buffer, 
           mdisk->length, "mdisk.xml", NULL, 
           XML_PARSE_NOENT | XML_PARSE_NONET |
           XML_PARSE_NOWARNING);
   if (!mdisk->doc) {
      libmsg("%s(): libxml failed to parse mdisk.xml buffer\n", __func__);
      goto error;
   }

   closedir(dir);

   return 0;
error:
   if (dir)
       closedir(dir);

   libmsg("%s(): Unable to read metrics disk\n", __func__);

   return -1;
}
// ResponseParser::parse [java]
	public Object parse(SerializerHandler serializerHandler, InputStream response, boolean debugMode) throws XMLRPCException {

		try {
			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();

			// Ensure the xml parser won't allow exploitation of the vuln CWE-611
			// (described on https://cwe.mitre.org/data/definitions/611.html )
			factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
			factory.setExpandEntityReferences(false);
			factory.setNamespaceAware(true);
			factory.setXIncludeAware(false);
			factory.setExpandEntityReferences(false);
			// End of the configuration of the parser for CWE-611

			DocumentBuilder builder = factory.newDocumentBuilder();
			Document dom = builder.parse(response);
			if (debugMode ){
				printDocument(dom, System.out);
			}
			Element e = dom.getDocumentElement();


			// Check for root tag
			if(!e.getNodeName().equals(XMLRPCClient.METHOD_RESPONSE)) {
				throw new XMLRPCException("MethodResponse root tag is missing.");
			}

			e = XMLUtil.getOnlyChildElement(e.getChildNodes());

			if(e.getNodeName().equals(XMLRPCClient.PARAMS)) {

				e = XMLUtil.getOnlyChildElement(e.getChildNodes());

				if(!e.getNodeName().equals(XMLRPCClient.PARAM)) {
					throw new XMLRPCException("The params tag must contain a param tag.");
				}

				return getReturnValueFromElement(serializerHandler, e);

			} else if(e.getNodeName().equals(XMLRPCClient.FAULT)) {

				@SuppressWarnings("unchecked")
				Map<String,Object> o = (Map<String,Object>)getReturnValueFromElement(serializerHandler, e);

				throw new XMLRPCServerException((String)o.get(FAULT_STRING), (Integer)o.get(FAULT_CODE));

			}

			throw new XMLRPCException("The methodResponse tag must contain a fault or params tag.");

		} catch(XMLRPCServerException e) {
			throw e;
		} catch (Exception ex) {
			throw new XMLRPCException("Error getting result from server.", ex);
		}

	}
// ResponseParser::parse [java]
	public Object parse(SerializerHandler serializerHandler, InputStream response, boolean debugMode) throws XMLRPCException {

		try {

			DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
			factory.setNamespaceAware(true);
			DocumentBuilder builder = factory.newDocumentBuilder();
			Document dom = builder.parse(response);
			if (debugMode ){
				printDocument(dom, System.out);
			}
			Element e = dom.getDocumentElement();


			// Check for root tag
			if(!e.getNodeName().equals(XMLRPCClient.METHOD_RESPONSE)) {
				throw new XMLRPCException("MethodResponse root tag is missing.");
			}

			e = XMLUtil.getOnlyChildElement(e.getChildNodes());

			if(e.getNodeName().equals(XMLRPCClient.PARAMS)) {

				e = XMLUtil.getOnlyChildElement(e.getChildNodes());

				if(!e.getNodeName().equals(XMLRPCClient.PARAM)) {
					throw new XMLRPCException("The params tag must contain a param tag.");
				}

				return getReturnValueFromElement(serializerHandler, e);

			} else if(e.getNodeName().equals(XMLRPCClient.FAULT)) {

				@SuppressWarnings("unchecked")
				Map<String,Object> o = (Map<String,Object>)getReturnValueFromElement(serializerHandler, e);

				throw new XMLRPCServerException((String)o.get(FAULT_STRING), (Integer)o.get(FAULT_CODE));

			}

			throw new XMLRPCException("The methodResponse tag must contain a fault or params tag.");

		} catch(XMLRPCServerException e) {
			throw e;
		} catch (Exception ex) {
			throw new XMLRPCException("Error getting result from server.", ex);
		}

	}
// TestPlanAnalyzer::getNodeListWithClassNames [java]
    private NodeList getNodeListWithClassNames(String path) {
        try {
            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
            factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
            factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
            factory.setXIncludeAware(false);
            factory.setExpandEntityReferences(false);

            DocumentBuilder builder = factory.newDocumentBuilder();
            byte[] bytes = overrideXmlVersion(readBytesFromFile(path));
            Document doc = (bytes == null) ? builder.parse(path) : builder.parse(new ByteArrayInputStream(bytes));
            XPathFactory xPathfactory = XPathFactory.newInstance();
            XPath xpath = xPathfactory.newXPath();
            XPathExpression expr = xpath.compile("//*[@guiclass|@testclass]");
            return (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
        } catch (ParserConfigurationException pex) {
            log.warn("Cannot set the required parser config", pex);
            return null;
        } catch (Exception ex) {
            log.warn("Cannot parse file: " + path, ex);
            return null;
        }
    }
// TestPlanAnalyzer::getNodeListWithClassNames [java]
    private NodeList getNodeListWithClassNames(String path) {
        try {
            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
            DocumentBuilder builder = factory.newDocumentBuilder();
            byte[] bytes = overrideXmlVersion(readBytesFromFile(path));
            Document doc = (bytes == null) ? builder.parse(path) : builder.parse(new ByteArrayInputStream(bytes));
            XPathFactory xPathfactory = XPathFactory.newInstance();
            XPath xpath = xPathfactory.newXPath();
            XPathExpression expr = xpath.compile("//*[@guiclass|@testclass]");
            return (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
        } catch (Exception ex) {
            log.warn("Cannot parse file: " + path, ex);
            return null;
        }
    }
// do_file_clean [c]
static void do_file_clean(int flags, char *name)
{
	char *page;
	char fn[30];
	snprintf(fn, 30, TMPDIR "~test%d", tmpcount++);
	int fd = open(fn, O_RDWR|O_TRUNC|O_CREAT, 0664);
	if (fd < 0)
		err("open temp file");
	write(fd, fn, 4);
	page = checked_mmap(NULL, PS, PROT_READ|PROT_WRITE, MAP_SHARED|flags,
		fd, 0);
	fsync(fd);
	close(fd);
	testmem(name, page, MREAD_OK);
	 /* reread page from disk */
	printf("\t reading %x\n", *(unsigned char *)page);
	testmem(name, page, MWRITE_OK);
}
// do_file_clean [c]
static void do_file_clean(int flags, char *name)
{
	char *page;
	char fn[30];
	snprintf(fn, 30, TMPDIR "~test%d", tmpcount++);
	int fd = open(fn, O_RDWR|O_TRUNC|O_CREAT);
	if (fd < 0)
		err("open temp file");
	write(fd, fn, 4);
	page = checked_mmap(NULL, PS, PROT_READ|PROT_WRITE, MAP_SHARED|flags,
		fd, 0);
	fsync(fd);
	close(fd);
	testmem(name, page, MREAD_OK);
	 /* reread page from disk */
	printf("\t reading %x\n", *(unsigned char *)page);
	testmem(name, page, MWRITE_OK);
}
// make_tarfile [python]
def make_tarfile(
    output_filename: str,
    source_dir: str,
    archive_name: str,
    custom_filter: Optional[Callable] = None,
) -> None:
    # Helper for filtering out modification timestamps
    def _filter_timestamps(tar_info: "tarfile.TarInfo") -> Optional["tarfile.TarInfo"]:
        tar_info.mtime = 0
        return tar_info if custom_filter is None else custom_filter(tar_info)

    descriptor, unzipped_filename = tempfile.mkstemp()
    try:
        with tarfile.open(unzipped_filename, "w") as tar:
            tar.add(source_dir, arcname=archive_name, filter=_filter_timestamps)
        # When gzipping the tar, don't include the tar's filename or modification time in the
        # zipped archive (see https://docs.python.org/3/library/gzip.html#gzip.GzipFile)
        with gzip.GzipFile(
            filename="", fileobj=open(output_filename, "wb"), mode="wb", mtime=0
        ) as gzipped_tar, open(unzipped_filename, "rb") as tar_file:
            gzipped_tar.write(tar_file.read())
    finally:
        os.close(descriptor)
        os.remove(unzipped_filename)
// make_tarfile [python]
def make_tarfile(
    output_filename: str,
    source_dir: str,
    archive_name: str,
    custom_filter: Optional[Callable] = None,
) -> None:
    # Helper for filtering out modification timestamps
    def _filter_timestamps(tar_info: "tarfile.TarInfo") -> Optional["tarfile.TarInfo"]:
        tar_info.mtime = 0
        return tar_info if custom_filter is None else custom_filter(tar_info)

    unzipped_filename = tempfile.mktemp()
    try:
        with tarfile.open(unzipped_filename, "w") as tar:
            tar.add(source_dir, arcname=archive_name, filter=_filter_timestamps)
        # When gzipping the tar, don't include the tar's filename or modification time in the
        # zipped archive (see https://docs.python.org/3/library/gzip.html#gzip.GzipFile)
        with gzip.GzipFile(
            filename="", fileobj=open(output_filename, "wb"), mode="wb", mtime=0
        ) as gzipped_tar, open(unzipped_filename, "rb") as tar_file:
            gzipped_tar.write(tar_file.read())
    finally:
        os.remove(unzipped_filename)
// main [python]
def main(_):
  # Generate some fake Iris data.
  # It is okay for this example because this example is about how to use the
  # debugger, not how to use machine learning to solve the Iris classification
  # problem.
  def training_input_fn():
    return ({
        "features": tf.random_normal([128, 4])
    }, tf.random_uniform([128], minval=0, maxval=3, dtype=tf.int32))

  def test_input_fn():
    return ({
        "features": tf.random_normal([32, 4])
    }, tf.random_uniform([32], minval=0, maxval=3, dtype=tf.int32))

  feature_columns = [tf.feature_column.numeric_column("features", shape=(4,))]

  # Build 3 layer DNN with 10, 20, 10 units respectively.
  model_dir = FLAGS.model_dir or tempfile.mkdtemp(prefix="debug_tflearn_iris_")

  classifier = tf.estimator.DNNClassifier(
      feature_columns=feature_columns,
      hidden_units=[10, 20, 10],
      n_classes=3,
      model_dir=model_dir)

  if FLAGS.debug and FLAGS.tensorboard_debug_address:
    raise ValueError(
        "The --debug and --tensorboard_debug_address flags are mutually "
        "exclusive.")
  hooks = []
  if FLAGS.debug:
    if FLAGS.use_random_config_path:
      # TODO(mihaimaruseac): Safe to ignore fd here?
      _, config_file_path = tempfile.mkstemp(".tfdbg_config")
    else:
      config_file_path = None
    hooks.append(
        tf_debug.LocalCLIDebugHook(
            ui_type=FLAGS.ui_type,
            dump_root=FLAGS.dump_root,
            config_file_path=config_file_path))
  elif FLAGS.tensorboard_debug_address:
    hooks.append(tf_debug.TensorBoardDebugHook(FLAGS.tensorboard_debug_address))

  # Train model, using tfdbg hook.
  classifier.train(training_input_fn, steps=FLAGS.train_steps, hooks=hooks)

  # Evaluate accuracy, using tfdbg hook.
  accuracy_score = classifier.evaluate(
      test_input_fn, steps=FLAGS.eval_steps, hooks=hooks)["accuracy"]

  print("After training %d steps, Accuracy = %f" %
        (FLAGS.train_steps, accuracy_score))

  # Make predictions, using tfdbg hook.
  predict_results = classifier.predict(test_input_fn, hooks=hooks)
  print("A prediction result: %s" % next(predict_results))
// main [python]
def main(_):
  # Generate some fake Iris data.
  # It is okay for this example because this example is about how to use the
  # debugger, not how to use machine learning to solve the Iris classification
  # problem.
  def training_input_fn():
    return ({
        "features": tf.random_normal([128, 4])
    }, tf.random_uniform([128], minval=0, maxval=3, dtype=tf.int32))

  def test_input_fn():
    return ({
        "features": tf.random_normal([32, 4])
    }, tf.random_uniform([32], minval=0, maxval=3, dtype=tf.int32))

  feature_columns = [tf.feature_column.numeric_column("features", shape=(4,))]

  # Build 3 layer DNN with 10, 20, 10 units respectively.
  model_dir = FLAGS.model_dir or tempfile.mkdtemp(prefix="debug_tflearn_iris_")

  classifier = tf.estimator.DNNClassifier(
      feature_columns=feature_columns,
      hidden_units=[10, 20, 10],
      n_classes=3,
      model_dir=model_dir)

  if FLAGS.debug and FLAGS.tensorboard_debug_address:
    raise ValueError(
        "The --debug and --tensorboard_debug_address flags are mutually "
        "exclusive.")
  hooks = []
  if FLAGS.debug:
    config_file_path = (
        tempfile.mktemp(".tfdbg_config")
        if FLAGS.use_random_config_path else None)
    hooks.append(
        tf_debug.LocalCLIDebugHook(
            ui_type=FLAGS.ui_type,
            dump_root=FLAGS.dump_root,
            config_file_path=config_file_path))
  elif FLAGS.tensorboard_debug_address:
    hooks.append(tf_debug.TensorBoardDebugHook(FLAGS.tensorboard_debug_address))

  # Train model, using tfdbg hook.
  classifier.train(training_input_fn, steps=FLAGS.train_steps, hooks=hooks)

  # Evaluate accuracy, using tfdbg hook.
  accuracy_score = classifier.evaluate(
      test_input_fn, steps=FLAGS.eval_steps, hooks=hooks)["accuracy"]

  print("After training %d steps, Accuracy = %f" %
        (FLAGS.train_steps, accuracy_score))

  # Make predictions, using tfdbg hook.
  predict_results = classifier.predict(test_input_fn, hooks=hooks)
  print("A prediction result: %s" % next(predict_results))
// testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError [python]
  def testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError(self):
    # Create an unrelated source file.
    fd, unrelated_source_path = tempfile.mkstemp()
    with open(fd, "wt") as source_file:
      source_file.write("print('hello, world')\n")

    self.assertEqual({},
                     source_utils.annotate_source(self.dump,
                                                  unrelated_source_path))

    # Clean up unrelated source file.
    os.remove(unrelated_source_path)
// testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError [python]
  def testCallingAnnotateSourceOnUnrelatedSourceFileDoesNotError(self):
    # Create an unrelated source file.
    unrelated_source_path = tempfile.mktemp()
    with open(unrelated_source_path, "wt") as source_file:
      source_file.write("print('hello, world')\n")

    self.assertEqual({},
                     source_utils.annotate_source(self.dump,
                                                  unrelated_source_path))

    # Clean up unrelated source file.
    os.remove(unrelated_source_path)
// testDebugDumpDir_nonexistentDumpRoot [python]
  def testDebugDumpDir_nonexistentDumpRoot(self):
    with self.assertRaisesRegex(IOError, "does not exist"):
      debug_data.DebugDumpDir(tempfile.mkdtemp() + "_foo")
// testDebugDumpDir_nonexistentDumpRoot [python]
  def testDebugDumpDir_nonexistentDumpRoot(self):
    with self.assertRaisesRegex(IOError, "does not exist"):
      debug_data.DebugDumpDir(tempfile.mktemp() + "_foo")
// testNonExistentFile [python]
  def testNonExistentFile(self):
    converter = upgrade_schema_lib.Converter()
    _, non_existent = tempfile.mkstemp(suffix=".json")  # safe to ignore fd
    with self.assertRaisesRegex(IOError, "No such file or directory"):
      converter.Convert(non_existent, non_existent)
// testNonExistentFile [python]
  def testNonExistentFile(self):
    converter = upgrade_schema_lib.Converter()
    non_existent = tempfile.mktemp(suffix=".json")
    with self.assertRaisesRegex(IOError, "No such file or directory"):
      converter.Convert(non_existent, non_existent)
// testWriteToFileSucceeds [python]
  def testWriteToFileSucceeds(self):
    screen_output = debugger_cli_common.RichTextLines(
        ["Roses are red", "Violets are blue"],
        font_attr_segs={0: [(0, 5, "red")],
                        1: [(0, 7, "blue")]})

    _, file_path = tempfile.mkstemp()  # safe to ignore fd here
    screen_output.write_to_file(file_path)

    with gfile.Open(file_path, "r") as f:
      self.assertEqual("Roses are red\nViolets are blue\n", f.read())

    # Clean up.
    gfile.Remove(file_path)
// testWriteToFileSucceeds [python]
  def testWriteToFileSucceeds(self):
    screen_output = debugger_cli_common.RichTextLines(
        ["Roses are red", "Violets are blue"],
        font_attr_segs={0: [(0, 5, "red")],
                        1: [(0, 7, "blue")]})

    file_path = tempfile.mktemp()
    screen_output.write_to_file(file_path)

    with gfile.Open(file_path, "r") as f:
      self.assertEqual("Roses are red\nViolets are blue\n", f.read())

    # Clean up.
    gfile.Remove(file_path)
// setUp [python]
  def setUp(self):
    _, self._history_file_path = tempfile.mkstemp()  # safe to ignore fd here
    self._cmd_hist = debugger_cli_common.CommandHistory(
        limit=3, history_file_path=self._history_file_path)
// setUp [python]
  def setUp(self):
    self._history_file_path = tempfile.mktemp()
    self._cmd_hist = debugger_cli_common.CommandHistory(
        limit=3, history_file_path=self._history_file_path)
// __init__ [python]
  def __init__(self,
               height,
               width,
               command_sequence=None):
    self._height = height
    self._width = width

    self._command_sequence = command_sequence
    self._command_counter = 0

    # The mock class has no actual textbox. So use this variable to keep
    # track of what's entered in the textbox on creation.
    self._curr_existing_command = ""

    # Observers for test.
    # Observers of screen output.
    self.unwrapped_outputs = []
    self.wrapped_outputs = []
    self.scroll_messages = []
    self.output_array_pointer_indices = []

    self.output_pad_rows = []

    # Observers of command textbox.
    self.existing_commands = []

    # Observer for tab-completion candidates.
    self.candidates_lists = []

    # Observer for the main menu.
    self.main_menu_list = []

    # Observer for toast messages.
    self.toasts = []

    curses_ui.CursesUI.__init__(
        self,
        config=cli_config.CLIConfig(
            config_file_path=os.path.join(tempfile.mkdtemp(), ".tfdbg_config")))

    # Override the default path to the command history file to avoid test
    # concurrency issues.
    _, history_file_path = tempfile.mkstemp()  # safe to ignore fd
    self._command_history_store = debugger_cli_common.CommandHistory(
        history_file_path=history_file_path)
// __init__ [python]
  def __init__(self,
               height,
               width,
               command_sequence=None):
    self._height = height
    self._width = width

    self._command_sequence = command_sequence
    self._command_counter = 0

    # The mock class has no actual textbox. So use this variable to keep
    # track of what's entered in the textbox on creation.
    self._curr_existing_command = ""

    # Observers for test.
    # Observers of screen output.
    self.unwrapped_outputs = []
    self.wrapped_outputs = []
    self.scroll_messages = []
    self.output_array_pointer_indices = []

    self.output_pad_rows = []

    # Observers of command textbox.
    self.existing_commands = []

    # Observer for tab-completion candidates.
    self.candidates_lists = []

    # Observer for the main menu.
    self.main_menu_list = []

    # Observer for toast messages.
    self.toasts = []

    curses_ui.CursesUI.__init__(
        self,
        config=cli_config.CLIConfig(
            config_file_path=os.path.join(tempfile.mkdtemp(), ".tfdbg_config")))

    # Override the default path to the command history file to avoid test
    # concurrency issues.
    self._command_history_store = debugger_cli_common.CommandHistory(
        history_file_path=tempfile.mktemp())
// is_case_sensitive [python]
    def is_case_sensitive(self):
        if self._case_sensitive is None:
            lock = self.unlock_path(self.base_folder, unlock_parent=False)
            path = tempfile.mkdtemp(prefix='.caseTest_', dir=self.base_folder)
            if os.path.exists(path.upper()):
                self._case_sensitive = False
            else:
                self._case_sensitive = True
            os.rmdir(path)
            self.lock_path(self.base_folder, lock)
        return self._case_sensitive
// is_case_sensitive [python]
    def is_case_sensitive(self):
        if self._case_sensitive is None:
            lock = self.unlock_path(self.base_folder, unlock_parent=False)
            path = os.tempnam(self.base_folder, '.caseTest_')
            os.mkdir(path)
            if os.path.exists(path.upper()):
                self._case_sensitive = False
            else:
                self._case_sensitive = True
            os.rmdir(path)
            self.lock_path(self.base_folder, lock)
        return self._case_sensitive
// rename [python]
    def rename(self, ref, to_name):
        """Rename a local file or folder

        Return the actualized info object.

        """
        new_name = safe_filename(to_name)
        source_os_path = self._abspath(ref)
        parent = ref.rsplit(u'/', 1)[0]
        old_name = ref.rsplit(u'/', 1)[1]
        parent = u'/' if parent == '' else parent
        locker = self.unlock_ref(ref)
        try:
            # Check if only case renaming
            if (old_name != new_name and old_name.lower() == new_name.lower()
                and not self.is_case_sensitive()):
                # Must use a temp rename as FS is not case sensitive
                prefix = LocalClient.CASE_RENAME_PREFIX + old_name + '_'
                _, temp_path = tempfile.mkstemp(dir=self._abspath(parent),
                                                prefix=prefix)
                if AbstractOSIntegration.is_windows():
                    import ctypes
                    ctypes.windll.kernel32.SetFileAttributesW(
                                                unicode(temp_path), 2)
                os.rename(source_os_path, temp_path)
                source_os_path = temp_path
                # Try the os rename part
                target_os_path = self._abspath(os.path.join(parent, new_name))
            else:
                target_os_path, new_name = self._abspath_deduped(parent,
                                                                new_name, old_name)
            if old_name != new_name:
                os.rename(source_os_path, target_os_path)
            if AbstractOSIntegration.is_windows():
                import ctypes
                # See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
                ctypes.windll.kernel32.SetFileAttributesW(
                                            unicode(target_os_path), 128)
            new_ref = self.get_children_ref(parent, new_name)
            return self.get_info(new_ref)
        finally:
            self.lock_ref(ref, locker & 2)
// rename [python]
    def rename(self, ref, to_name):
        """Rename a local file or folder

        Return the actualized info object.

        """
        new_name = safe_filename(to_name)
        source_os_path = self._abspath(ref)
        parent = ref.rsplit(u'/', 1)[0]
        old_name = ref.rsplit(u'/', 1)[1]
        parent = u'/' if parent == '' else parent
        locker = self.unlock_ref(ref)
        try:
            # Check if only case renaming
            if (old_name != new_name and old_name.lower() == new_name.lower()
                and not self.is_case_sensitive()):
                # Must use a temp rename as FS is not case sensitive
                temp_path = os.tempnam(self._abspath(parent),
                                       LocalClient.CASE_RENAME_PREFIX + old_name + '_')
                if AbstractOSIntegration.is_windows():
                    import ctypes
                    ctypes.windll.kernel32.SetFileAttributesW(
                                                unicode(temp_path), 2)
                os.rename(source_os_path, temp_path)
                source_os_path = temp_path
                # Try the os rename part
                target_os_path = self._abspath(os.path.join(parent, new_name))
            else:
                target_os_path, new_name = self._abspath_deduped(parent,
                                                                new_name, old_name)
            if old_name != new_name:
                os.rename(source_os_path, target_os_path)
            if AbstractOSIntegration.is_windows():
                import ctypes
                # See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
                ctypes.windll.kernel32.SetFileAttributesW(
                                            unicode(target_os_path), 128)
            new_ref = self.get_children_ref(parent, new_name)
            return self.get_info(new_ref)
        finally:
            self.lock_ref(ref, locker & 2)
// __init__ [python]
  def __init__(self, on_ui_exit=None, command_sequence=None):
    _, config_file_path = tempfile.mkstemp()  # safe to ignore fd
    readline_ui.ReadlineUI.__init__(
        self,
        on_ui_exit=on_ui_exit,
        config=cli_config.CLIConfig(config_file_path=config_file_path))

    self._command_sequence = command_sequence
    self._command_counter = 0

    self.observers = {"screen_outputs": []}
// __init__ [python]
  def __init__(self, on_ui_exit=None, command_sequence=None):
    readline_ui.ReadlineUI.__init__(
        self, on_ui_exit=on_ui_exit,
        config=cli_config.CLIConfig(config_file_path=tempfile.mktemp()))

    self._command_sequence = command_sequence
    self._command_counter = 0

    self.observers = {"screen_outputs": []}
// test_dynamic_dependencies [python]
    def test_dynamic_dependencies(self):

        class DynamicRequires(Task):
            p = luigi.Parameter()

            def output(self):
                return luigi.LocalTarget(os.path.join(self.p, 'parent'))

            def run(self):
                dummy_targets = yield [DynamicDummyTask(os.path.join(self.p, str(i)))
                                     for i in range(5)]
                dummy_targets += yield [DynamicDummyTask(os.path.join(self.p, str(i)))
                                       for i in range(5, 7)]
                with self.output().open('w') as f:
                    for i, d in enumerate(dummy_targets):
                        for line in d.open('r'):
                            print >>f, '%d: %s' % (i, line.strip())

        p = tempfile.mkdtemp()
        try:
            t = DynamicRequires(p=p)
            luigi.build([t], local_scheduler=True)
            self.assertTrue(t.complete())

            # loop through output and verify
            f = t.output().open('r')
            for i in xrange(7):
                self.assertEqual(f.readline().strip(), '%d: Done!' % i)
        finally:
            shutil.rmtree(p)
// test_dynamic_dependencies [python]
    def test_dynamic_dependencies(self):

        class DynamicRequires(Task):
            p = luigi.Parameter()

            def output(self):
                return luigi.LocalTarget(os.path.join(self.p, 'parent'))

            def run(self):
                dummy_targets = yield [DynamicDummyTask(os.path.join(self.p, str(i)))
                                     for i in range(5)]
                dummy_targets += yield [DynamicDummyTask(os.path.join(self.p, str(i)))
                                       for i in range(5, 7)]
                with self.output().open('w') as f:
                    for i, d in enumerate(dummy_targets):
                        for line in d.open('r'):
                            print >>f, '%d: %s' % (i, line.strip())

        t = DynamicRequires(p=tempfile.mktemp())
        luigi.build([t], local_scheduler=True)
        self.assertTrue(t.complete())

        # loop through output and verify
        f = t.output().open('r')
        for i in xrange(7):
            self.assertEqual(f.readline().strip(), '%d: Done!' % i)
// plot [python]
    def plot(self, log_files, sort='time', limit=10, nfl_filter='',
             metric_selected='cc', plot_type='bar'):
        if not PLOTLIB_INSTALLED:
            raise PLOTLIBNotInstalled(_('python-matplotlib not installed.'))
        if len(log_files) == 0:
            raise NotFoundException(_('no log file found'))
        try:
            stats = Stats2(*log_files)
            stats.sort_stats(sort)
            stats_dict = stats.stats
            __, func_list = stats.get_print_list([nfl_filter, limit])
            nfls = []
            performance = []
            names = {'nc': 'Total Call Count', 'cc': 'Primitive Call Count',
                     'tt': 'Total Time', 'ct': 'Cumulative Time'}
            for func in func_list:
                cc, nc, tt, ct, __ = stats_dict[func]
                metric = {'cc': cc, 'nc': nc, 'tt': tt, 'ct': ct}
                nfls.append(func[2])
                performance.append(metric[metric_selected])
            y_pos = range(len(nfls))
            error = [random.random() for __ in y_pos]
            plt.clf()
            if plot_type == 'pie':
                plt.pie(x=performance, explode=None, labels=nfls,
                        autopct='%1.1f%%')
            else:
                plt.barh(y_pos, performance, xerr=error, align='center',
                         alpha=0.4)
                plt.yticks(y_pos, nfls)
                plt.xlabel(names[metric_selected])
            plt.title('Profile Statistics (by %s)' % names[metric_selected])
            #plt.gcf().tight_layout(pad=1.2)
            profile_img = tempfile.TemporaryFile()
            plt.savefig(profile_img, format='png', dpi=300)
            profile_img.seek(0)
            data = profile_img.read()
            os.close(profile_img)
            return data, [('content-type', 'image/jpg')]
        except Exception as ex:
            raise ProfileException(_('plotting results failed due to %s') % ex)
// plot [python]
    def plot(self, log_files, sort='time', limit=10, nfl_filter='',
             metric_selected='cc', plot_type='bar'):
        if not PLOTLIB_INSTALLED:
            raise PLOTLIBNotInstalled(_('python-matplotlib not installed.'))
        if len(log_files) == 0:
            raise NotFoundException(_('no log file found'))
        try:
            stats = Stats2(*log_files)
            stats.sort_stats(sort)
            stats_dict = stats.stats
            __, func_list = stats.get_print_list([nfl_filter, limit])
            nfls = []
            performance = []
            names = {'nc': 'Total Call Count', 'cc': 'Primitive Call Count',
                     'tt': 'Total Time', 'ct': 'Cumulative Time'}
            for func in func_list:
                cc, nc, tt, ct, __ = stats_dict[func]
                metric = {'cc': cc, 'nc': nc, 'tt': tt, 'ct': ct}
                nfls.append(func[2])
                performance.append(metric[metric_selected])
            y_pos = range(len(nfls))
            error = [random.random() for __ in y_pos]
            plt.clf()
            if plot_type == 'pie':
                plt.pie(x=performance, explode=None, labels=nfls,
                        autopct='%1.1f%%')
            else:
                plt.barh(y_pos, performance, xerr=error, align='center',
                         alpha=0.4)
                plt.yticks(y_pos, nfls)
                plt.xlabel(names[metric_selected])
            plt.title('Profile Statistics (by %s)' % names[metric_selected])
            #plt.gcf().tight_layout(pad=1.2)
            profile_img = tempfile.mktemp('.png', 'plot')
            plt.savefig(profile_img, dpi=300)
            data = open(profile_img).read()
            os.remove(profile_img)
            return data, [('content-type', 'image/jpg')]
        except Exception as ex:
            raise ProfileException(_('plotting results failed due to %s') % ex)
// main [python]
def main(_):
  # Create a dummy dataset.
  num_examples = 8
  steps_per_epoch = 2
  input_dims = 3
  output_dims = 1
  xs = np.zeros([num_examples, input_dims])
  ys = np.zeros([num_examples, output_dims])
  dataset = tf.data.Dataset.from_tensor_slices(
      (xs, ys)).repeat(num_examples).batch(int(num_examples / steps_per_epoch))

  sess = tf.Session()
  if FLAGS.debug:
    # Use the command-line interface (CLI) of tfdbg.
    if FLAGS.use_random_config_path:
      # TODO(mihaimaruseac): Safe to ignore fd here?
      _, config_file_path = tempfile.mkstemp(".tfdbg_config")
    else:
      config_file_path = None
    sess = tf_debug.LocalCLIDebugWrapperSession(
        sess, ui_type=FLAGS.ui_type, config_file_path=config_file_path)
  elif FLAGS.tensorboard_debug_address:
    # Use the TensorBoard Debugger Plugin (GUI of tfdbg).
    sess = tf_debug.TensorBoardDebugWrapperSession(
        sess, FLAGS.tensorboard_debug_address)
  tf.keras.backend.set_session(sess)

  # Create a dummy model.
  model = tf.keras.Sequential(
      [tf.keras.layers.Dense(1, input_shape=[input_dims])])
  model.compile(loss="mse", optimizer="sgd")

  # Train the model using the dummy dataset created above.
  model.fit(dataset, epochs=FLAGS.epochs, steps_per_epoch=steps_per_epoch)
// main [python]
def main(_):
  # Create a dummy dataset.
  num_examples = 8
  steps_per_epoch = 2
  input_dims = 3
  output_dims = 1
  xs = np.zeros([num_examples, input_dims])
  ys = np.zeros([num_examples, output_dims])
  dataset = tf.data.Dataset.from_tensor_slices(
      (xs, ys)).repeat(num_examples).batch(int(num_examples / steps_per_epoch))

  sess = tf.Session()
  if FLAGS.debug:
    # Use the command-line interface (CLI) of tfdbg.
    config_file_path = (
        tempfile.mktemp(".tfdbg_config")
        if FLAGS.use_random_config_path else None)
    sess = tf_debug.LocalCLIDebugWrapperSession(
        sess, ui_type=FLAGS.ui_type, config_file_path=config_file_path)
  elif FLAGS.tensorboard_debug_address:
    # Use the TensorBoard Debugger Plugin (GUI of tfdbg).
    sess = tf_debug.TensorBoardDebugWrapperSession(
        sess, FLAGS.tensorboard_debug_address)
  tf.keras.backend.set_session(sess)

  # Create a dummy model.
  model = tf.keras.Sequential(
      [tf.keras.layers.Dense(1, input_shape=[input_dims])])
  model.compile(loss="mse", optimizer="sgd")

  # Train the model using the dummy dataset created above.
  model.fit(dataset, epochs=FLAGS.epochs, steps_per_epoch=steps_per_epoch)
// setUp [python]
  def setUp(self):
    self._dump_root = tempfile.mkdtemp()
// setUp [python]
  def setUp(self):
    self._dump_root = tempfile.mktemp()
    os.mkdir(self._dump_root)