jiahuimbzuai/CoQuIR huggingface dataset:
// (anonymous) [javascript]
				symbol => {
					let array;

					if (symbol === "y") {
						array = ["8", "9", "a", "b"];
						return array[Math.floor(Math.random() * array.length)];
					}

					array = new Uint8Array(1);
					window.crypto.getRandomValues(array);
					return (array[0] % 16).toString(16);
				}
// (anonymous) [javascript]
				c => {
					// eslint-disable-next-line
					const r = (Math.random() * 16) | 0;
					// eslint-disable-next-line
					const v = c == "x" ? r : (r & 0x3) | 0x8;
					return v.toString(16);
				}
// (anonymous) [javascript]
  }, function (statusCode, body) {
    if (statusCode !== 200) {
      // request a new login key first
      this._steamUser.requestWebAPIAuthenticateUserNonce(function (nonce) {
        this._webLoginKey = nonce.webapi_authenticate_user_nonce;
        this.webLogOn(callback);
      }.bind(this));
      return;
    }

    this.sessionID = crypto.randomBytes(12).toString('hex');
    this.cookies = [
      'sessionid=' + this.sessionID,
      'steamLogin=' + body.authenticateuser.token,
      'steamLoginSecure=' + body.authenticateuser.tokensecure
    ];

    callback(this.sessionID, this.cookies);
  }.bind(this));
// (anonymous) [javascript]
  }, function (statusCode, body) {
    if (statusCode !== 200) {
      // request a new login key first
      this._steamUser.requestWebAPIAuthenticateUserNonce(function (nonce) {
        this._webLoginKey = nonce.webapi_authenticate_user_nonce;
        this.webLogOn(callback);
      }.bind(this));
      return;
    }

    this.sessionID = Math.floor(Math.random() * 1000000000).toString();
    this.cookies = [
      'sessionid=' + this.sessionID,
      'steamLogin=' + body.authenticateuser.token,
      'steamLoginSecure=' + body.authenticateuser.tokensecure
    ];

    callback(this.sessionID, this.cookies);
  }.bind(this));
// run [go]
func run() error {
	priv, err := rsa.GenerateMultiPrimeKey(rand.Reader, 3, 2048)
	if err != nil {
		return err
	}

	privf, err := os.OpenFile("priv.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer privf.Close()

	privblock := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(priv),
	}

	if err := pem.Encode(privf, privblock); err != nil {
		os.Remove(privf.Name())
		return err
	}

	pub, err := x509.MarshalPKIXPublicKey(priv.Public())
	if err != nil {
		os.Remove(privf.Name())
		return err
	}

	pubf, err := os.OpenFile("pub.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		os.Remove(privf.Name())
		return err
	}
	defer pubf.Close()

	pubblock := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: pub,
	}

	if err := pem.Encode(pubf, pubblock); err != nil {
		os.Remove(privf.Name())
		os.Remove(pubf.Name())
		return err
	}

	return nil
}
// run [go]
func run() error {
	priv, err := rsa.GenerateKey(rand.Reader, 1024)
	if err != nil {
		return err
	}

	privf, err := os.OpenFile("priv.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer privf.Close()

	privblock := &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(priv),
	}

	if err := pem.Encode(privf, privblock); err != nil {
		os.Remove(privf.Name())
		return err
	}

	pub, err := x509.MarshalPKIXPublicKey(priv.Public())
	if err != nil {
		os.Remove(privf.Name())
		return err
	}

	pubf, err := os.OpenFile("pub.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		os.Remove(privf.Name())
		return err
	}
	defer pubf.Close()

	pubblock := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: pub,
	}

	if err := pem.Encode(pubf, pubblock); err != nil {
		os.Remove(privf.Name())
		os.Remove(pubf.Name())
		return err
	}

	return nil
}
// generatePrivateKey [go]
func generatePrivateKey(keyType string, keyBits int, container ParsedPrivateKeyContainer, entropyReader io.Reader) error {
	var err error
	var privateKeyType PrivateKeyType
	var privateKeyBytes []byte
	var privateKey crypto.Signer

	var randReader io.Reader = rand.Reader
	if entropyReader != nil {
		randReader = entropyReader
	}

	switch keyType {
	case "rsa":
		// XXX: there is a false-positive CodeQL path here around keyBits;
		// because of a default zero value in the TypeDurationSecond and
		// TypeSignedDurationSecond cases of schema.DefaultOrZero(), it
		// thinks it is possible to end up with < 2048 bit RSA Key here.
		// While this is true for SSH keys, it isn't true for PKI keys
		// due to ValidateKeyTypeLength(...) below. While we could close
		// the report as a false-positive, enforcing a minimum keyBits size
		// here of 2048 would ensure no other paths exist.
		if keyBits < 2048 {
			return errutil.InternalError{Err: fmt.Sprintf("insecure bit length for RSA private key: %d", keyBits)}
		}
		privateKeyType = RSAPrivateKey
		privateKey, err = rsa.GenerateKey(randReader, keyBits)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating RSA private key: %v", err)}
		}
		privateKeyBytes = x509.MarshalPKCS1PrivateKey(privateKey.(*rsa.PrivateKey))
	case "ec":
		privateKeyType = ECPrivateKey
		var curve elliptic.Curve
		switch keyBits {
		case 224:
			curve = elliptic.P224()
		case 256:
			curve = elliptic.P256()
		case 384:
			curve = elliptic.P384()
		case 521:
			curve = elliptic.P521()
		default:
			return errutil.UserError{Err: fmt.Sprintf("unsupported bit length for EC key: %d", keyBits)}
		}
		privateKey, err = ecdsa.GenerateKey(curve, randReader)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating EC private key: %v", err)}
		}
		privateKeyBytes, err = x509.MarshalECPrivateKey(privateKey.(*ecdsa.PrivateKey))
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error marshalling EC private key: %v", err)}
		}
	case "ed25519":
		privateKeyType = Ed25519PrivateKey
		_, privateKey, err = ed25519.GenerateKey(randReader)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating ed25519 private key: %v", err)}
		}
		privateKeyBytes, err = x509.MarshalPKCS8PrivateKey(privateKey.(ed25519.PrivateKey))
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error marshalling Ed25519 private key: %v", err)}
		}
	default:
		return errutil.UserError{Err: fmt.Sprintf("unknown key type: %s", keyType)}
	}

	container.SetParsedPrivateKey(privateKey, privateKeyType, privateKeyBytes)
	return nil
}
// generatePrivateKey [go]
func generatePrivateKey(keyType string, keyBits int, container ParsedPrivateKeyContainer, entropyReader io.Reader) error {
	var err error
	var privateKeyType PrivateKeyType
	var privateKeyBytes []byte
	var privateKey crypto.Signer

	var randReader io.Reader = rand.Reader
	if entropyReader != nil {
		randReader = entropyReader
	}

	switch keyType {
	case "rsa":
		privateKeyType = RSAPrivateKey
		privateKey, err = rsa.GenerateKey(randReader, keyBits)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating RSA private key: %v", err)}
		}
		privateKeyBytes = x509.MarshalPKCS1PrivateKey(privateKey.(*rsa.PrivateKey))
	case "ec":
		privateKeyType = ECPrivateKey
		var curve elliptic.Curve
		switch keyBits {
		case 224:
			curve = elliptic.P224()
		case 256:
			curve = elliptic.P256()
		case 384:
			curve = elliptic.P384()
		case 521:
			curve = elliptic.P521()
		default:
			return errutil.UserError{Err: fmt.Sprintf("unsupported bit length for EC key: %d", keyBits)}
		}
		privateKey, err = ecdsa.GenerateKey(curve, randReader)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating EC private key: %v", err)}
		}
		privateKeyBytes, err = x509.MarshalECPrivateKey(privateKey.(*ecdsa.PrivateKey))
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error marshalling EC private key: %v", err)}
		}
	case "ed25519":
		privateKeyType = Ed25519PrivateKey
		_, privateKey, err = ed25519.GenerateKey(randReader)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error generating ed25519 private key: %v", err)}
		}
		privateKeyBytes, err = x509.MarshalPKCS8PrivateKey(privateKey.(ed25519.PrivateKey))
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("error marshalling Ed25519 private key: %v", err)}
		}
	default:
		return errutil.UserError{Err: fmt.Sprintf("unknown key type: %s", keyType)}
	}

	container.SetParsedPrivateKey(privateKey, privateKeyType, privateKeyBytes)
	return nil
}
// RSAKeyPairUtil::readKeys [java]
    private void readKeys( ) throws GeneralSecurityException {
        if ( DatastoreService.existsKey( DATASTORE_PUBLIC_KEY ) && DatastoreService.existsKey( DATASTORE_PRIVATE_KEY ) )
        {
            X509EncodedKeySpec keySpecPublic = new X509EncodedKeySpec(Base64.getDecoder().decode(DatastoreService.getDataValue( DATASTORE_PUBLIC_KEY, "" ).getBytes()));
            PKCS8EncodedKeySpec keySpecPrivate = new PKCS8EncodedKeySpec (Base64.getDecoder().decode(DatastoreService.getDataValue( DATASTORE_PRIVATE_KEY, "" ).getBytes()));
            
            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
            
            this._publicKey = keyFactory.generatePublic( keySpecPublic );
            this._privateKey = keyFactory.generatePrivate( keySpecPrivate );
        }
        else
        {
            KeyPairGenerator keyGen = KeyPairGenerator.getInstance( "RSA" );
            keyGen.initialize( 2048 );
            KeyPair pair = keyGen.generateKeyPair( );
            this._privateKey = pair.getPrivate( );
            this._publicKey = pair.getPublic( );
            
            DatastoreService.setDataValue( DATASTORE_PUBLIC_KEY, Base64.getEncoder().encodeToString( _publicKey.getEncoded( ) ) );
            DatastoreService.setDataValue( DATASTORE_PRIVATE_KEY, Base64.getEncoder().encodeToString( _privateKey.getEncoded( ) ) );
        }
    }
// RSAKeyPairUtil::readKeys [java]
    private void readKeys( ) throws GeneralSecurityException {
        if ( DatastoreService.existsKey( DATASTORE_PUBLIC_KEY ) && DatastoreService.existsKey( DATASTORE_PRIVATE_KEY ) )
        {
            X509EncodedKeySpec keySpecPublic = new X509EncodedKeySpec(Base64.getDecoder().decode(DatastoreService.getDataValue( DATASTORE_PUBLIC_KEY, "" ).getBytes()));
            PKCS8EncodedKeySpec keySpecPrivate = new PKCS8EncodedKeySpec (Base64.getDecoder().decode(DatastoreService.getDataValue( DATASTORE_PRIVATE_KEY, "" ).getBytes()));
            
            KeyFactory keyFactory = KeyFactory.getInstance("RSA");
            
            this._publicKey = keyFactory.generatePublic( keySpecPublic );
            this._privateKey = keyFactory.generatePrivate( keySpecPrivate );
        }
        else
        {
            KeyPairGenerator keyGen = KeyPairGenerator.getInstance( "RSA" );
            keyGen.initialize( 1024 );
            KeyPair pair = keyGen.generateKeyPair( );
            this._privateKey = pair.getPrivate( );
            this._publicKey = pair.getPublic( );
            
            DatastoreService.setDataValue( DATASTORE_PUBLIC_KEY, Base64.getEncoder().encodeToString( _publicKey.getEncoded( ) ) );
            DatastoreService.setDataValue( DATASTORE_PRIVATE_KEY, Base64.getEncoder().encodeToString( _privateKey.getEncoded( ) ) );
        }
    }
// generateKeys [python]
def generateKeys(len=2048):
	fludkey = FludRSA.generate(len)
	return fludkey.publickey(), fludkey.privatekey()
// generateKeys [python]
def generateKeys(len=1024):
	fludkey = FludRSA.generate(len)
	return fludkey.publickey(), fludkey.privatekey()
// handle [python]
    def handle(self, *args, **options):
        try:
            key = RSA.generate(2048)
            rsakey = RSAKey(key=key.exportKey('PEM').decode('utf8'))
            rsakey.save()
            self.stdout.write(u'RSA key successfully created with kid: {0}'.format(rsakey.kid))
        except Exception as e:
            self.stdout.write('Something goes wrong: {0}'.format(e))
// handle [python]
    def handle(self, *args, **options):
        try:
            key = RSA.generate(1024)
            rsakey = RSAKey(key=key.exportKey('PEM').decode('utf8'))
            rsakey.save()
            self.stdout.write(u'RSA key successfully created with kid: {0}'.format(rsakey.kid))
        except Exception as e:
            self.stdout.write('Something goes wrong: {0}'.format(e))
// load_config [python]
def load_config(config_file):
    config_path = config_file if config_file else ".ansible-lint"

    if os.path.exists(config_path):
        with open(config_path, "r") as stream:
            try:
                return yaml.safe_load(stream)
            except yaml.YAMLError:
                pass

    return None
// load_config [python]
def load_config(config_file):
    config_path = config_file if config_file else ".ansible-lint"

    if os.path.exists(config_path):
        with open(config_path, "r") as stream:
            try:
                return yaml.load(stream)
            except yaml.YAMLError:
                pass

    return None
// __init__ [python]
    def __init__(self, id_file_name):
        self.id_file_name = id_file_name
        if os.path.isfile(id_file_name):
            with open (self.id_file_name, "r") as fp:
                text = fp.read()
                self.db = yaml.load(text, Loader=yaml.SafeLoader)
                if not self.db:
                    self.db = {}
                fp.close()
        else:
            self.db = {}
// __init__ [python]
    def __init__(self, id_file_name):
        self.id_file_name = id_file_name
        if os.path.isfile(id_file_name):
            with open (self.id_file_name, "r") as fp:
                text = fp.read()
                self.db = yaml.load(text)
                if not self.db:
                    self.db = {}
                fp.close()
        else:
            self.db = {}
// load [python]
    @classmethod
    def load(cls, data_store, filename):
        pgm_model = None
        if type(data_store) is LocalFileSystem:
            pgm_model = data_store.read_pomegranate_model(filename=filename)
        if type(data_store) is S3DataStore:
            local_filename = "/tmp/kronos.json"
            data_store.download_file(filename, local_filename)
            with open(local_filename, 'rb') as f:
                pgm_model = BayesianNetwork.from_json(f.read())
        return PGMPomegranate(pgm_model)
// load [python]
    @classmethod
    def load(cls, data_store, filename):
        pgm_model = None
        if type(data_store) is LocalFileSystem:
            pgm_model = data_store.read_pomegranate_model(filename=filename)
        if type(data_store) is S3DataStore:
            local_filename = "/tmp/kronos.json"
            data_store.download_file(filename, local_filename)
            with open(local_filename, 'rb') as f:
                pgm_model = BayesianNetwork.from_json(pickle.load(f))
        return PGMPomegranate(pgm_model)
// save [python]
    def save(self, data_store, filename):
        pgm_model = self.model
        if type(data_store) is LocalFileSystem:
            data_store.write_pomegranate_model(
                model=pgm_model, filename=filename)
        if type(data_store) is S3DataStore:
            local_filename = "/tmp/kronos.json"
            with open(local_filename, 'wb') as f:
                # IMPORTANT: Set pickle.HIGHEST_PROTOCOL only  after complete porting to
                # Python3
                f.write(pgm_model.to_json())

            data_store.upload_file(local_filename, filename)
        return None
// save [python]
    def save(self, data_store, filename):
        pgm_model = self.model
        if type(data_store) is LocalFileSystem:
            data_store.write_pomegranate_model(
                model=pgm_model, filename=filename)
        if type(data_store) is S3DataStore:
            local_filename = "/tmp/kronos.json"
            with open(local_filename, 'wb') as f:
                # IMPORTANT: Set pickle.HIGHEST_PROTOCOL only  after complete porting to
                # Python3
                pickle.dump(pgm_model.to_json(), f, protocol=2)

            data_store.upload_file(local_filename, filename)
        return None
// puppet_enc_default [python]
@app.route('/puppet/default', methods=['GET', 'POST'])
@cortex.lib.user.login_required
def puppet_enc_default():
	"""Handles the Puppet ENC Default Classes page"""

	# Check user permissions
	if not does_user_have_permission("puppet.default_classes.view"):
		abort(403)

	# Get the default YAML out of the kv table
	curd = g.db.cursor(mysql.cursors.DictCursor)
	curd.execute("SELECT `value` FROM `kv_settings` WHERE `key` = 'puppet.enc.default'")
	result = curd.fetchone()
	if result == None:
		classes = "# Classes to include on all nodes using the default settings can be entered here\n"
	else:
		classes = result['value']

	# On any GET request, just display the information
	if request.method == 'GET':
		return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

	# On any POST request, validate the input and then save
	elif request.method == 'POST':
		# Check user permissions
		if not does_user_have_permission("puppet.default_classes.edit"):
			abort(403)

		# Extract data from form
		classes = request.form.get('classes', '')

		# Validate classes YAML
		try:
			data = yaml.safe_load(classes)
		except Exception as e:
			flash('Invalid YAML syntax: ' + str(e), 'alert-danger')
			return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

		try:
			if not data is None:
				assert isinstance(data, dict)
		except Exception as e:
			flash('Invalid YAML syntax: result was not a list of classes, did you forget a trailing colon? ' + str(e), 'alert-danger')
			return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

		# Get a cursor to the database
		# Update the system
		curd.execute('REPLACE INTO `kv_settings` (`key`, `value`) VALUES ("puppet.enc.default", %s)', (classes,))
		g.db.commit()

		cortex.lib.core.log(__name__, "puppet.defaultconfig.changed", "Puppet default configuration updated")
		# Redirect back
		flash('Puppet default settings updated', 'alert-success')

		return redirect(url_for('puppet_enc_default'))
// puppet_enc_default [python]
@app.route('/puppet/default', methods=['GET', 'POST'])
@cortex.lib.user.login_required
def puppet_enc_default():
	"""Handles the Puppet ENC Default Classes page"""

	# Check user permissions
	if not does_user_have_permission("puppet.default_classes.view"):
		abort(403)

	# Get the default YAML out of the kv table
	curd = g.db.cursor(mysql.cursors.DictCursor)
	curd.execute("SELECT `value` FROM `kv_settings` WHERE `key` = 'puppet.enc.default'")
	result = curd.fetchone()
	if result == None:
		classes = "# Classes to include on all nodes using the default settings can be entered here\n"
	else:
		classes = result['value']

	# On any GET request, just display the information
	if request.method == 'GET':
		return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

	# On any POST request, validate the input and then save
	elif request.method == 'POST':
		# Check user permissions
		if not does_user_have_permission("puppet.default_classes.edit"):
			abort(403)

		# Extract data from form
		classes = request.form.get('classes', '')

		# Validate classes YAML
		try:
			data = yaml.load(classes)
		except Exception as e:
			flash('Invalid YAML syntax: ' + str(e), 'alert-danger')
			return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

		try:
			if not data is None:
				assert isinstance(data, dict)
		except Exception as e:
			flash('Invalid YAML syntax: result was not a list of classes, did you forget a trailing colon? ' + str(e), 'alert-danger')
			return render_template('puppet/default.html', classes=classes, active='puppet', title="Default Classes")

		# Get a cursor to the database
		# Update the system
		curd.execute('REPLACE INTO `kv_settings` (`key`, `value`) VALUES ("puppet.enc.default", %s)', (classes,))
		g.db.commit()

		cortex.lib.core.log(__name__, "puppet.defaultconfig.changed", "Puppet default configuration updated")
		# Redirect back
		flash('Puppet default settings updated', 'alert-success')

		return redirect(url_for('puppet_enc_default'))
// write [python]
    def write(self, bib_data, filename):
        def process_person_roles(entry):
            for role, persons in entry.persons.iteritems():
                yield role, list(process_persons(persons))

        def process_person(person):
            for type in ('first', 'middle', 'prelast', 'last', 'lineage'):
                name = person.get_part_as_text(type)
                if name:
                    yield type, name

        def process_persons(persons):
            for person in persons:
                yield dict(process_person(person))
                
        def process_entries(bib_data):
            for key, entry in bib_data.iteritems():
                fields = dict(entry.fields)
                fields['type'] = entry.type
                fields.update(process_person_roles(entry))
                yield key, fields

        data = {'data': dict(process_entries(bib_data))}
        f = open(filename, 'w')
        yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False, indent=4)
        f.close()
// write [python]
    def write(self, bib_data, filename):
        def process_person_roles(entry):
            for role, persons in entry.persons.iteritems():
                yield role, list(process_persons(persons))

        def process_person(person):
            for type in ('first', 'middle', 'prelast', 'last', 'lineage'):
                name = person.get_part_as_text(type)
                if name:
                    yield type, name

        def process_persons(persons):
            for person in persons:
                yield dict(process_person(person))
                
        def process_entries(bib_data):
            for key, entry in bib_data.iteritems():
                fields = dict(entry.fields)
                fields['type'] = entry.type
                fields.update(process_person_roles(entry))
                yield key, fields

        data = {'data': dict(process_entries(bib_data))}
        f = open(filename, 'w')
        yaml.dump(data, f, allow_unicode=True, default_flow_style=False, indent=4)
        f.close()
// __init__ [python]
    def __init__(self, config_fname=None):
        config_fname = config_fname or self.config_fname
        fo = open(config_fname, "r")
        blob = fo.read()
        fo.close()
        yaml=YAML(typ='safe')
        self.config = yaml.load(blob)
// __init__ [python]
    def __init__(self, config_fname=None):
        config_fname = config_fname or self.config_fname
        fo = open(config_fname, "r")
        blob = fo.read()
        fo.close()
        self.config = yaml.load(blob)
// parse [python]
    @staticmethod
    def parse(path, require_exists=True, require_parses=True):
        if not os.path.isfile(path):
            if require_exists:
                raise ConfigError('not found: ' + path)
            else:
                return None
        try:
            with open(path) as f:
                return yaml.safe_load(f)
        except Exception, error:
            if require_parses:
                raise ConfigError('parse error: ' + path)
// parse [python]
    @staticmethod
    def parse(path, require_exists=True, require_parses=True):
        if not os.path.isfile(path):
            if require_exists:
                raise ConfigError('not found: ' + path)
            else:
                return None
        try:
            with open(path) as f:
                return yaml.load(f)
        except Exception, error:
            if require_parses:
                raise ConfigError('parse error: ' + path)
// _migrate_map [python]
def _migrate_map(contents):
    # Find the first non-header line
    lines = contents.splitlines(True)
    i = 0
    while _is_header_line(lines[i]):
        i += 1

    header = ''.join(lines[:i])
    rest = ''.join(lines[i:])

    if isinstance(ordered_load(contents), list):
        # If they are using the "default" flow style of yaml, this operation
        # will yield a valid configuration
        try:
            trial_contents = header + 'repos:\n' + rest
            ordered_load(trial_contents)
            contents = trial_contents
        except yaml.YAMLError:
            contents = header + 'repos:\n' + _indent(rest)

    return contents
// _migrate_map [python]
def _migrate_map(contents):
    # Find the first non-header line
    lines = contents.splitlines(True)
    i = 0
    while _is_header_line(lines[i]):
        i += 1

    header = ''.join(lines[:i])
    rest = ''.join(lines[i:])

    if isinstance(ordered_load(contents), list):
        # If they are using the "default" flow style of yaml, this operation
        # will yield a valid configuration
        try:
            trial_contents = header + 'repos:\n' + rest
            yaml.load(trial_contents)
            contents = trial_contents
        except yaml.YAMLError:
            contents = header + 'repos:\n' + _indent(rest)

    return contents
// update_device [python]
    def update_device(self, **kwargs):
        """ See http://api.device42.com/#create/update-device-by-name """
        path = 'devices'
        atleast_fields = ["name"]  # this is the only required field to create/update a device, serial and uuid opt
        known_fields = "new_name asset_no manufacturer hardware new_hardware is_it_switch"
        known_fields += " is_it_virtual_host is_it_blade_host in_service type service_level virtual_host"
        known_fields += " serial_no uuid"
        known_fields += " blade_host slot_no storage_room_id storage_room os osver osverno memory cpucount cpupower cpucore"
        known_fields += " hddcount hddsize hddraid hddraid_type macaddress devices_in_cluster appcomps"
        known_fields += " customer contract_id contract"
        known_fields += " aliases subtype virtual_subtype notes tags"
        known_fields = atleast_fields + known_fields.split()
        if not set(atleast_fields).intersection(kwargs.keys()):
            raise Device42BadArgumentError("At least one parameter should be passed: %s" % atleast_fields)
        unknown_fields = set(kwargs.keys()) - set(known_fields)
        if unknown_fields:
            raise Device42BadArgumentError("Unknown parameters: %s" % unknown_fields)
        return self._post(path, data=kwargs)
// update_device [python]
    def update_device(self, **kwargs):
        """ See http://api.device42.com/#create/update-device-by-name """
        path = 'devices'
        atleast_fields = "name serial_no uuid".split()
        known_fields = "new_name asset_no manufacturer hardware new_hardware is_it_switch"
        known_fields += " is_it_virtual_host is_it_blade_host in_service type service_level virtual_host"
        known_fields += " blade_host slot_no storage_room_id storage_room os osver osverno memory cpucount cpupower cpucore"
        known_fields += " hddcount hddsize hddraid hddraid_type macaddress devices_in_cluster appcomps"
        known_fields += " customer contract_id contract"
        known_fields += " aliases subtype virtual_subtype notes tags"
        known_fields = atleast_fields + known_fields.split()
        if not set(atleast_fields).intersection(kwargs.keys()):
            raise Device42BadArgumentError("At least one parameter should be passed: %s" % atleast_fields)
        unknown_fields = set(kwargs.keys()) - set(known_fields)
        if unknown_fields:
            raise Device42BadArgumentError("Unknown parameters: %s" % unknown_fields)
        return self._post(path, data=kwargs)
// dump [python]
    def dump(self, path):
        """
        dump address space as binary to file
        """
        s = shelve.open(path, "n", protocol = pickle.HIGHEST_PROTOCOL)
        for nodeid in self._nodes.keys():
            s[nodeid.to_string()] = self._nodes[nodeid]
        s.close()
// dump [python]
    def dump(self, path):
        """
        dump address space as binary to file
        """
        with open(path, 'wb') as f:
            pickle.dump(self._nodes, f, pickle.HIGHEST_PROTOCOL)
// load_data [python]
def load_data(path):
    """Given path to a file, load data from it."""
    ext = os.path.splitext(path)[-1]
    loader = None
    function = 'load'
    if ext in {'.yml', '.yaml'}:
        loader = yaml
        function = 'safe_load'
        if yaml is None:
            req_missing(['yaml'], 'use YAML data files')
            return {}
    elif ext in {'.json', '.js'}:
        loader = json
    elif ext in {'.toml', '.tml'}:
        if toml is None:
            req_missing(['toml'], 'use TOML data files')
            return {}
        loader = toml
    if loader is None:
        return
    with io.open(path, 'r', encoding='utf8') as inf:
        return getattr(loader, function)(inf)
// load_data [python]
def load_data(path):
    """Given path to a file, load data from it."""
    ext = os.path.splitext(path)[-1]
    loader = None
    if ext in {'.yml', '.yaml'}:
        loader = yaml
        if yaml is None:
            req_missing(['yaml'], 'use YAML data files')
            return {}
    elif ext in {'.json', '.js'}:
        loader = json
    elif ext in {'.toml', '.tml'}:
        if toml is None:
            req_missing(['toml'], 'use TOML data files')
            return {}
        loader = toml
    if loader is None:
        return
    with io.open(path, 'r', encoding='utf8') as inf:
        return loader.load(inf)
// test_list_kube_config_contexts [python]
    def test_list_kube_config_contexts(self):
        config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG))
        contexts, active_context = list_kube_config_contexts(
            config_file=config_file)
        self.assertDictEqual(self.TEST_KUBE_CONFIG['contexts'][0],
                             active_context)
        if PY3:
            self.assertCountEqual(self.TEST_KUBE_CONFIG['contexts'],
                                  contexts)
        else:
            self.assertItemsEqual(self.TEST_KUBE_CONFIG['contexts'],
                                  contexts)
// test_list_kube_config_contexts [python]
    def test_list_kube_config_contexts(self):
        config_file = self._create_temp_file(yaml.dump(self.TEST_KUBE_CONFIG))
        contexts, active_context = list_kube_config_contexts(
            config_file=config_file)
        self.assertDictEqual(self.TEST_KUBE_CONFIG['contexts'][0],
                             active_context)
        if PY3:
            self.assertCountEqual(self.TEST_KUBE_CONFIG['contexts'],
                                  contexts)
        else:
            self.assertItemsEqual(self.TEST_KUBE_CONFIG['contexts'],
                                  contexts)
// _yaml_to_config [python]
    def _yaml_to_config(self, config_file):
        self.config = yaml.safe_load(config_file)
// _yaml_to_config [python]
    def _yaml_to_config(self, config_file):
         self.config = yaml.load(config_file)
// hierarchical_tile [python]
def hierarchical_tile(masterfile,tilefile):

    """
    Create Hierarchical tile from Master prior

    :param masterfile: Master prior file
    :param tilefile:  File containing Tiling scheme
    """
    try:
        taskid = np.int(os.environ['SGE_TASK_ID'])
        task_first=np.int(os.environ['SGE_TASK_FIRST'])
        task_last=np.int(os.environ['SGE_TASK_LAST'])

    except KeyError:
        print("Error: could not read SGE_TASK_ID from environment")
        taskid = int(input("Please enter task id: "))
        print("you entered", taskid)


    with open(tilefile, 'rb') as f:
        obj = pickle.load(f)

    tiles = obj['tiles']
    order = obj['order']
    tiles_large = obj['tiles_large']
    order_large = obj['order_large']

    obj=xidplus.io.pickle_load(masterfile)
    priors = obj['priors']

    moc = moc_routines.get_fitting_region(order_large, tiles_large[taskid - 1])
    for p in priors:
        p.moc = moc
        p.cut_down_prior()

    outfile = 'Tile_'+ str(tiles_large[taskid - 1]) + '_' + str(order_large) + '.pkl'
    with open(outfile, 'wb') as f:
        pickle.dump({'priors':priors, 'version':xidplus.io.git_version()}, f)
// hierarchical_tile [python]
def hierarchical_tile(masterfile,tilefile):

    """
    Create Hierarchical tile from Master prior

    :param masterfile: Master prior file
    :param tilefile:  File containing Tiling scheme
    """
    try:
        taskid = np.int(os.environ['SGE_TASK_ID'])
        task_first=np.int(os.environ['SGE_TASK_FIRST'])
        task_last=np.int(os.environ['SGE_TASK_LAST'])

    except KeyError:
        print("Error: could not read SGE_TASK_ID from environment")
        taskid = int(input("Please enter task id: "))
        print("you entered", taskid)


    with open(tilefile, 'rb') as f:
        obj = pickle.load(f)

    tiles = obj['tiles']
    order = obj['order']
    tiles_large = obj['tiles_large']
    order_large = obj['order_large']

    with open(masterfile, 'rb') as f:
        obj = pickle.load(f)
    priors = obj['priors']

    moc = moc_routines.get_fitting_region(order_large, tiles_large[taskid - 1])
    for p in priors:
        p.moc = moc
        p.cut_down_prior()

    outfile = 'Tile_'+ str(tiles_large[taskid - 1]) + '_' + str(order_large) + '.pkl'
    with open(outfile, 'wb') as f:
        pickle.dump({'priors':priors, 'version':xidplus.io.git_version()}, f)
// _drain_to_working_set [python]
    def _drain_to_working_set(self, size=1000):
        logger.info('Draining to working set %s', self.working_set_filename)

        assert not os.path.exists(self.working_set_filename)

        with new_session() as session:
            query = session.query(Result)

            if self.after:
                query = query.filter(Result.datetime > self.after)

            with gzip.open(self.working_set_filename, 'wb', compresslevel=1) as work_file:
                last_id = -1
                num_results = 0
                running = True

                while running:
                    # Optimized for SQLite scrolling window
                    rows = query.filter(Result.id > last_id).limit(size).all()

                    if not rows:
                        break

                    delete_ids = []

                    for result in rows:
                        pickle.dump({
                            'id': result.id,
                            'project_id': result.project_id,
                            'shortcode': result.shortcode,
                            'url': result.url,
                            'encoding': result.encoding,
                            'datetime': result.datetime,
                        }, work_file)

                        num_results += 1
                        self.items_count += 1

                        delete_ids.append(result.id)

                        if num_results % 10000 == 0:
                            logger.info('Drain progress: %d', num_results)

                        if num_results % 100000 == 0:
                            # Risky, but need to do this since WAL
                            # performance is low on large transactions
                            logger.info("Checkpoint. (Don't delete stray files if program crashes!)")
                            work_file.flush()
                            session.commit()

                        if self.max_items and num_results >= self.max_items:
                            logger.info('Reached max items %d.', self.max_items)
                            running = False
                            break

                    if self.settings['delete']:
                        delete_query = delete(Result).where(
                            Result.id == bindparam('id')
                        )
                        session.execute(
                            delete_query,
                            [{'id': result_id} for result_id in delete_ids]
                        )

                pickle.dump('eof', work_file)
// _drain_to_working_set [python]
    def _drain_to_working_set(self, size=1000):
        logger.info('Draining to working set %s', self.working_set_filename)

        assert not os.path.exists(self.working_set_filename)

        with new_session() as session:
            query = session.query(Result)

            if self.after:
                query = query.filter(Result.datetime > self.after)

            with open(self.working_set_filename, 'wb') as work_file:
                last_id = -1
                num_results = 0
                running = True

                while running:
                    # Optimized for SQLite scrolling window
                    rows = query.filter(Result.id > last_id).limit(size).all()

                    if not rows:
                        break

                    delete_ids = []

                    for result in rows:
                        line = base64.b64encode(pickle.dumps({
                            'id': result.id,
                            'project_id': result.project_id,
                            'shortcode': result.shortcode,
                            'url': result.url,
                            'encoding': result.encoding,
                            'datetime': result.datetime,
                        }))
                        work_file.write(line)
                        work_file.write(b'\n')

                        num_results += 1
                        self.items_count += 1

                        delete_ids.append(result.id)

                        if num_results % 10000 == 0:
                            logger.info('Drain progress: %d', num_results)

                        if num_results % 100000 == 0:
                            # Risky, but need to do this since WAL
                            # performance is low on large transactions
                            logger.info("Checkpoint. (Don't delete stray files if program crashes!)")
                            work_file.flush()
                            session.commit()

                        if self.max_items and num_results >= self.max_items:
                            logger.info('Reached max items %d.', self.max_items)
                            running = False
                            break

                    if self.settings['delete']:
                        delete_query = delete(Result).where(
                            Result.id == bindparam('id')
                        )
                        session.execute(
                            delete_query,
                            [{'id': result_id} for result_id in delete_ids]
                        )
// __init__ [python]
    def __init__(self,p):
        try:
            self.tokens = np.array([symbolToIndex["START"]] + [ symbolToIndex[s] for s in serializeProgram(p) ] + [symbolToIndex["END"]])
        except KeyError:
            print "Key error in tokenization",serializeProgram(p)
            assert False
        
        self.image = p.convertToSequence().draw()
        self.program = p

        if str(parseOutput(serializeProgram(p))) != str(p):
            print "Serialization failure for program",p
            print serializeProgram(p)
            print parseOutput(serializeProgram(p))
            assert False
// __init__ [python]
    def __init__(self,p):
        p = pickle.loads(p)
        try:
            self.tokens = np.array([symbolToIndex["START"]] + [ symbolToIndex[s] for s in serializeProgram(p) ] + [symbolToIndex["END"]])
        except KeyError:
            print "Key error in tokenization",serializeProgram(p)
            assert False
        
        self.image = p.convertToSequence().draw()
        self.program = p

        if str(parseOutput(serializeProgram(p))) != str(p):
            print "Serialization failure for program",p
            print serializeProgram(p)
            print parseOutput(serializeProgram(p))
            assert False
// test_verilator_run [python]
def test_verilator_run():
    import os.path
    import tempfile
    import yaml
    from edalize import get_edatool
    ref_dir_cc = os.path.join(ref_dir, 'cc')

    work_root    = tempfile.mkdtemp()
    edam_file = os.path.join(ref_dir_cc, core_name)+ '.eda.yml'
    backend = get_edatool(tool)(edam=yaml.safe_load(open(edam_file)), work_root=work_root)
    dummy_exe = 'V'+backend.tool_options['top_module']
    shutil.copy(os.path.join(ref_dir, dummy_exe),
                os.path.join(work_root, dummy_exe))

    backend.run(params)

    compare_files(ref_dir, work_root, ['run.cmd'])
// test_verilator_run [python]
def test_verilator_run():
    import os.path
    import tempfile
    import yaml
    from edalize import get_edatool
    ref_dir_cc = os.path.join(ref_dir, 'cc')

    work_root    = tempfile.mkdtemp()
    edam_file = os.path.join(ref_dir_cc, core_name)+ '.eda.yml'
    backend = get_edatool(tool)(edam=yaml.load(open(edam_file)), work_root=work_root)
    dummy_exe = 'V'+backend.tool_options['top_module']
    shutil.copy(os.path.join(ref_dir, dummy_exe),
                os.path.join(work_root, dummy_exe))

    backend.run(params)

    compare_files(ref_dir, work_root, ['run.cmd'])
// _feed_input_sorters [python]
    def _feed_input_sorters(self):
        num_results = 0

        with gzip.open(self.working_set_filename, 'rb') as work_file:
            while True:
                result = pickle.load(work_file)

                if result == 'eof':
                    break

                if result['project_id'] not in self.project_result_sorters:
                    self.project_result_sorters[result['project_id']] = \
                        GNUExternalSort(temp_dir=self.output_dir,
                                        temp_prefix='tott-{0}-'.format(
                                            result['project_id']
                                            )
                                        )
                    self.projects_count += 1

                sorter = self.project_result_sorters[result['project_id']]
                sorter.input(
                    result['shortcode'],
                    (result['id'], result['url'], result['encoding'],
                     result['datetime'])
                )
                num_results += 1

                if num_results % 10000 == 0:
                    logger.info('Sort progress: %d', num_results)
// _feed_input_sorters [python]
    def _feed_input_sorters(self):
        num_results = 0

        with open(self.working_set_filename, 'rb') as work_file:
            for line in work_file:
                result = pickle.loads(base64.b64decode(line))

                if result['project_id'] not in self.project_result_sorters:
                    self.project_result_sorters[result['project_id']] = \
                        GNUExternalSort(temp_dir=self.output_dir,
                                        temp_prefix='tott-{0}-'.format(
                                            result['project_id']
                                            )
                                        )
                    self.projects_count += 1

                sorter = self.project_result_sorters[result['project_id']]
                sorter.input(
                    result['shortcode'],
                    (result['id'], result['url'], result['encoding'],
                     result['datetime'])
                )
                num_results += 1

                if num_results % 10000 == 0:
                    logger.info('Sort progress: %d', num_results)
// test_verilator_configure [python]
def test_verilator_configure():
    import os.path
    import tempfile
    import yaml
    from edalize import get_edatool

    for mode in ['cc', 'sc', 'lint-only']:
        work_root    = tempfile.mkdtemp()
        edam_file = os.path.join(ref_dir, mode, core_name) + '.eda.yml'

        backend = get_edatool(tool)(edam=yaml.safe_load(open(edam_file)), work_root=work_root)

        if mode is 'cc':
            _params = params
        else:
            _params = []
        backend.configure(_params)

        compare_files(ref_dir, work_root, ['Makefile'])

        compare_files(os.path.join(ref_dir, mode),
                      work_root,
                      ['config.mk', core_name+'.vc'])
// test_verilator_configure [python]
def test_verilator_configure():
    import os.path
    import tempfile
    import yaml
    from edalize import get_edatool

    for mode in ['cc', 'sc', 'lint-only']:
        work_root    = tempfile.mkdtemp()
        edam_file = os.path.join(ref_dir, mode, core_name) + '.eda.yml'

        backend = get_edatool(tool)(edam=yaml.load(open(edam_file)), work_root=work_root)

        if mode is 'cc':
            _params = params
        else:
            _params = []
        backend.configure(_params)

        compare_files(ref_dir, work_root, ['Makefile'])

        compare_files(os.path.join(ref_dir, mode),
                      work_root,
                      ['config.mk', core_name+'.vc'])
// exporters_v1tov2 [python]
def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False):
    """Translate exporters to v2 and put into shared config.

    Args:
        exporters_path (list): List of exporters file paths.
        shared_config (dict): Shared config to add exporters to.
        quiet (bool): Quiet mode.

    Returns:
        list: List of exporters keys added to shared config.
    """
    exp_keys = []
    for exp_path in exporters_paths:
        with open(exp_path, encoding='utf-8') as conf:
            content = yaml.load(conf, Loader=yaml.SafeLoader)
        exporters = content

        # If exporters file has sections, concatenate all of them
        if isinstance(content, dict):
            exporters = []
            for _, value in content.items():
                exporters.extend(value)

        # If exporter not in general config, add it and add an alias for the
        # exporter. Refer to the alias in the SLO config file.
        for exporter in exporters:
            exporter = OrderedDict(exporter)
            exp_key = add_to_shared_config(exporter,
                                           shared_config,
                                           'exporters',
                                           quiet=quiet)
            exp_keys.append(exp_key)
    return exp_keys
// exporters_v1tov2 [python]
def exporters_v1tov2(exporters_paths, shared_config={}, quiet=False):
    """Translate exporters to v2 and put into shared config.

    Args:
        exporters_path (list): List of exporters file paths.
        shared_config (dict): Shared config to add exporters to.
        quiet (bool): Quiet mode.

    Returns:
        list: List of exporters keys added to shared config.
    """
    exp_keys = []
    for exp_path in exporters_paths:
        with open(exp_path, encoding='utf-8') as conf:
            content = yaml.load(conf, Loader=yaml.Loader)
        exporters = content

        # If exporters file has sections, concatenate all of them
        if isinstance(content, dict):
            exporters = []
            for _, value in content.items():
                exporters.extend(value)

        # If exporter not in general config, add it and add an alias for the
        # exporter. Refer to the alias in the SLO config file.
        for exporter in exporters:
            exporter = OrderedDict(exporter)
            exp_key = add_to_shared_config(exporter,
                                           shared_config,
                                           'exporters',
                                           quiet=quiet)
            exp_keys.append(exp_key)
    return exp_keys
// run [python]
    def run(self):
        """Runs the groups scanner."""

        root = self._retrieve()

        with open(self.rules, 'r') as f:
            group_rules = file_loader.read_and_parse_file(f)

        root = self._apply_all_rules(root, group_rules)

        all_violations = self._find_violations(root)

        self._output_results(all_violations)
// run [python]
    def run(self):
        """Runs the groups scanner."""

        root = self._retrieve()

        with open(self.rules, 'r') as f:
            group_rules = yaml.load(f)

        root = self._apply_all_rules(root, group_rules)

        all_violations = self._find_violations(root)

        self._output_results(all_violations)
// _read_clouds [python]
    def _read_clouds(self):
        try:
            with open(self._clouds_path) as clouds_file:
                self._clouds = yaml.safe_load(clouds_file)
        except IOError:
            # The user doesn't have a clouds.yaml file.
            print("The user clouds.yaml file didn't exist.")
            self._clouds = {}