Commit message: crypto: prefix module autoloading with "crypto-"

This prefixes all crypto module loading with "crypto-" so we never run
the risk of exposing module auto-loading to userspace via a crypto API,
as demonstrated by Mathias Krause:

https://lkml.org/lkml/2013/3/4/70

Signed-off-by: Kees Cook <keescook@chromium.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
CVE: CVE-2013-7421
CWE: CWE-264
Vulnerability description: The Crypto API in the Linux kernel before 3.18.5 allows local users to load arbitrary kernel modules via a bind system call for an AF_ALG socket with a module name in the salg_name field, a different vulnerability than CVE-2014-9644.

Project: linux

static int cbc_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;
	int ts_state;

	padlock_reset_key(&ctx->cword.encrypt);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	ts_state = irq_ts_save();
	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->D, walk.iv, &ctx->cword.decrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}

	irq_ts_restore(ts_state);

	padlock_store_cword(&ctx->cword.encrypt);

	return err;
}

Project: linux

static int cbc_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;
	int ts_state;

	padlock_reset_key(&ctx->cword.encrypt);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	ts_state = irq_ts_save();
	while ((nbytes = walk.nbytes)) {
		u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
					    walk.dst.virt.addr, ctx->E,
					    walk.iv, &ctx->cword.encrypt,
					    nbytes / AES_BLOCK_SIZE);
		memcpy(walk.iv, iv, AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	irq_ts_restore(ts_state);

	padlock_store_cword(&ctx->cword.decrypt);

	return err;
}

Project: linux

static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
			    u8 *iv, struct cword *cword, int count)
{
	/* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
	if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE))
		return cbc_crypt_copy(in, out, key, iv, cword, count);

	return rep_xcrypt_cbc(in, out, key, iv, cword, count);
}

Project: linux

static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
			   u8 *iv, struct cword *cword, int count)
{
	/*
	 * Padlock prefetches extra data so we must provide mapped input buffers.
	 * Assume there are at least 16 bytes of stack already in use.
	 */
	u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
	u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);

	memcpy(tmp, in, count * AES_BLOCK_SIZE);
	return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
}

Project: linux

static int ecb_aes_decrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;
	int ts_state;

	padlock_reset_key(&ctx->cword.decrypt);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	ts_state = irq_ts_save();
	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->D, &ctx->cword.decrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	irq_ts_restore(ts_state);

	padlock_store_cword(&ctx->cword.encrypt);

	return err;
}

Project: linux

static int ecb_aes_encrypt(struct blkcipher_desc *desc,
			   struct scatterlist *dst, struct scatterlist *src,
			   unsigned int nbytes)
{
	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;
	int ts_state;

	padlock_reset_key(&ctx->cword.encrypt);

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);

	ts_state = irq_ts_save();
	while ((nbytes = walk.nbytes)) {
		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
				   ctx->E, &ctx->cword.encrypt,
				   nbytes / AES_BLOCK_SIZE);
		nbytes &= AES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	irq_ts_restore(ts_state);

	padlock_store_cword(&ctx->cword.encrypt);

	return err;
}

Project: linux

static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
			     struct cword *cword, int count)
{
	/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
	 * We could avoid some copying here but it's probably not worth it.
	 */
	if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) {
		ecb_crypt_copy(in, out, key, cword, count);
		return;
	}

	rep_xcrypt_ecb(in, out, key, cword, count);
}

Project: linux

static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
			   struct cword *cword, int count)
{
	/*
	 * Padlock prefetches extra data so we must provide mapped input buffers.
	 * Assume there are at least 16 bytes of stack already in use.
	 */
	u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
	u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);

	memcpy(tmp, in, count * AES_BLOCK_SIZE);
	rep_xcrypt_ecb(tmp, out, key, cword, count);
}

Project: linux

static void __exit padlock_fini(void)
{
	crypto_unregister_alg(&cbc_aes_alg);
	crypto_unregister_alg(&ecb_aes_alg);
	crypto_unregister_alg(&aes_alg);
}

Project: linux

static inline void padlock_reset_key(struct cword *cword)
{
	int cpu = raw_smp_processor_id();

	if (cword != per_cpu(paes_last_cword, cpu))
#ifndef CONFIG_X86_64
		asm volatile ("pushfl; popfl");
#else
		asm volatile ("pushfq; popfq");
#endif
}

Project: linux

static inline void padlock_store_cword(struct cword *cword)
{
	per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
}

Project: linux

static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
				      void *control_word, u32 count)
{
	u32 initial = count & (ecb_fetch_blocks - 1);

	if (count < ecb_fetch_blocks) {
		ecb_crypt(input, output, key, control_word, count);
		return;
	}

	if (initial)
		asm volatile (".byte 0xf3,0x0f,0xa7,0xc8"	/* rep xcryptecb */
			      : "+S"(input), "+D"(output)
			      : "d"(control_word), "b"(key), "c"(initial));

	asm volatile (".byte 0xf3,0x0f,0xa7,0xc8"	/* rep xcryptecb */
		      : "+S"(input), "+D"(output)
		      : "d"(control_word), "b"(key), "c"(count - initial));
}

Project: linux

static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
				  struct cword *control_word, int count)
{
	asm volatile (".byte 0xf3,0x0f,0xa7,0xc8"	/* rep xcryptecb */
		      : "+S"(input), "+D"(output)
		      : "d"(control_word), "b"(key), "c"(count));
}

Project: linux

static void padlock_cra_exit(struct crypto_tfm *tfm)
{
	struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);

	crypto_free_shash(ctx->fallback);
}

Project: linux

static int padlock_cra_init(struct crypto_tfm *tfm)
{
	struct crypto_shash *hash = __crypto_shash_cast(tfm);
	const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
	struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_shash *fallback_tfm;
	int err = -ENOMEM;

	/* Allocate a fallback and abort if it failed. */
	fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
					  CRYPTO_ALG_NEED_FALLBACK);
	if (IS_ERR(fallback_tfm)) {
		printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
		       fallback_driver_name);
		err = PTR_ERR(fallback_tfm);
		goto out;
	}

	ctx->fallback = fallback_tfm;
	hash->descsize += crypto_shash_descsize(fallback_tfm);
	return 0;

out:
	return err;
}

Project: linux

static void __exit padlock_fini(void)
{
	struct cpuinfo_x86 *c = &cpu_data(0);

	if (c->x86_model >= 0x0f) {
		crypto_unregister_shash(&sha1_alg_nano);
		crypto_unregister_shash(&sha256_alg_nano);
	} else {
		crypto_unregister_shash(&sha1_alg);
		crypto_unregister_shash(&sha256_alg);
	}
}

Project: linux

static int __init padlock_init(void)
{
	int rc = -ENODEV;
	struct cpuinfo_x86 *c = &cpu_data(0);
	struct shash_alg *sha1;
	struct shash_alg *sha256;

	if (!x86_match_cpu(padlock_sha_ids) || !cpu_has_phe_enabled)
		return -ENODEV;

	/* Register the newly added algorithm module if on *
	* VIA Nano processor, or else just do as before */
	if (c->x86_model < 0x0f) {
		sha1 = &sha1_alg;
		sha256 = &sha256_alg;
	} else {
		sha1 = &sha1_alg_nano;
		sha256 = &sha256_alg_nano;
	}

	rc = crypto_register_shash(sha1);
	if (rc)
		goto out;

	rc = crypto_register_shash(sha256);
	if (rc)
		goto out_unreg1;

	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");

	return 0;

out_unreg1:
	crypto_unregister_shash(sha1);

out:
	printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
	return rc;
}

Project: linux

static inline void padlock_output_block(uint32_t *src,
		 	uint32_t *dst, size_t count)
{
	while (count--)
		*dst++ = swab32(*src++);
}

Project: linux

static int padlock_sha1_final(struct shash_desc *desc, u8 *out)
{
	u8 buf[4];

	return padlock_sha1_finup(desc, buf, 0, out);
}

Project: linux

static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out)
{
	struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc);
	unsigned int partial, padlen;
	__be64 bits;
	static const u8 padding[64] = { 0x80, };

	bits = cpu_to_be64(state->count << 3);

	/* Pad out to 56 mod 64 */
	partial = state->count & 0x3f;
	padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
	padlock_sha1_update_nano(desc, padding, padlen);

	/* Append length field bytes */
	padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits));

	/* Swap to output */
	padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5);

	return 0;
}

Project: linux

static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in,
			      unsigned int count, u8 *out)
{
	/* We can't store directly to *out as it may be unaligned. */
	/* BTW Don't reduce the buffer size below 128 Bytes!
	 *     PadLock microcode needs it that big. */
	char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
		((aligned(STACK_ALIGN)));
	char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
	struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
	struct sha1_state state;
	unsigned int space;
	unsigned int leftover;
	int ts_state;
	int err;

	dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
	err = crypto_shash_export(&dctx->fallback, &state);
	if (err)
		goto out;

	if (state.count + count > ULONG_MAX)
		return crypto_shash_finup(&dctx->fallback, in, count, out);

	leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1;
	space =  SHA1_BLOCK_SIZE - leftover;
	if (space) {
		if (count > space) {
			err = crypto_shash_update(&dctx->fallback, in, space) ?:
			      crypto_shash_export(&dctx->fallback, &state);
			if (err)
				goto out;
			count -= space;
			in += space;
		} else {
			memcpy(state.buffer + leftover, in, count);
			in = state.buffer;
			count += leftover;
			state.count &= ~(SHA1_BLOCK_SIZE - 1);
		}
	}

	memcpy(result, &state.state, SHA1_DIGEST_SIZE);

	/* prevent taking the spurious DNA fault with padlock. */
	ts_state = irq_ts_save();
	asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
		      : \
		      : "c"((unsigned long)state.count + count), \
			"a"((unsigned long)state.count), \
			"S"(in), "D"(result));
	irq_ts_restore(ts_state);

	padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);

out:
	return err;
}

Project: linux

static int padlock_sha1_init_nano(struct shash_desc *desc)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);

	*sctx = (struct sha1_state){
		.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
	};

	return 0;
}

Project: linux

static int padlock_sha1_update_nano(struct shash_desc *desc,
			const u8 *data,	unsigned int len)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);
	unsigned int partial, done;
	const u8 *src;
	/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
	u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
		((aligned(STACK_ALIGN)));
	u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
	int ts_state;

	partial = sctx->count & 0x3f;
	sctx->count += len;
	done = 0;
	src = data;
	memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE);

	if ((partial + len) >= SHA1_BLOCK_SIZE) {

		/* Append the bytes in state's buffer to a block to handle */
		if (partial) {
			done = -partial;
			memcpy(sctx->buffer + partial, data,
				done + SHA1_BLOCK_SIZE);
			src = sctx->buffer;
			ts_state = irq_ts_save();
			asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
			: "+S"(src), "+D"(dst) \
			: "a"((long)-1), "c"((unsigned long)1));
			irq_ts_restore(ts_state);
			done += SHA1_BLOCK_SIZE;
			src = data + done;
		}

		/* Process the left bytes from the input data */
		if (len - done >= SHA1_BLOCK_SIZE) {
			ts_state = irq_ts_save();
			asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
			: "+S"(src), "+D"(dst)
			: "a"((long)-1),
			"c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE)));
			irq_ts_restore(ts_state);
			done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE);
			src = data + done;
		}
		partial = 0;
	}
	memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE);
	memcpy(sctx->buffer + partial, src, len - done);

	return 0;
}

Project: linux

static int padlock_sha256_final(struct shash_desc *desc, u8 *out)
{
	u8 buf[4];

	return padlock_sha256_finup(desc, buf, 0, out);
}

Project: linux

static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out)
{
	struct sha256_state *state =
		(struct sha256_state *)shash_desc_ctx(desc);
	unsigned int partial, padlen;
	__be64 bits;
	static const u8 padding[64] = { 0x80, };

	bits = cpu_to_be64(state->count << 3);

	/* Pad out to 56 mod 64 */
	partial = state->count & 0x3f;
	padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
	padlock_sha256_update_nano(desc, padding, padlen);

	/* Append length field bytes */
	padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits));

	/* Swap to output */
	padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8);

	return 0;
}

Project: linux

static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in,
				unsigned int count, u8 *out)
{
	/* We can't store directly to *out as it may be unaligned. */
	/* BTW Don't reduce the buffer size below 128 Bytes!
	 *     PadLock microcode needs it that big. */
	char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
		((aligned(STACK_ALIGN)));
	char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
	struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
	struct sha256_state state;
	unsigned int space;
	unsigned int leftover;
	int ts_state;
	int err;

	dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
	err = crypto_shash_export(&dctx->fallback, &state);
	if (err)
		goto out;

	if (state.count + count > ULONG_MAX)
		return crypto_shash_finup(&dctx->fallback, in, count, out);

	leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1;
	space =  SHA256_BLOCK_SIZE - leftover;
	if (space) {
		if (count > space) {
			err = crypto_shash_update(&dctx->fallback, in, space) ?:
			      crypto_shash_export(&dctx->fallback, &state);
			if (err)
				goto out;
			count -= space;
			in += space;
		} else {
			memcpy(state.buf + leftover, in, count);
			in = state.buf;
			count += leftover;
			state.count &= ~(SHA1_BLOCK_SIZE - 1);
		}
	}

	memcpy(result, &state.state, SHA256_DIGEST_SIZE);

	/* prevent taking the spurious DNA fault with padlock. */
	ts_state = irq_ts_save();
	asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
		      : \
		      : "c"((unsigned long)state.count + count), \
			"a"((unsigned long)state.count), \
			"S"(in), "D"(result));
	irq_ts_restore(ts_state);

	padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);

out:
	return err;
}

Project: linux

static int padlock_sha256_init_nano(struct shash_desc *desc)
{
	struct sha256_state *sctx = shash_desc_ctx(desc);

	*sctx = (struct sha256_state){
		.state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \
				SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7},
	};

	return 0;
}

Project: linux

static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data,
			  unsigned int len)
{
	struct sha256_state *sctx = shash_desc_ctx(desc);
	unsigned int partial, done;
	const u8 *src;
	/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
	u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
		((aligned(STACK_ALIGN)));
	u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
	int ts_state;

	partial = sctx->count & 0x3f;
	sctx->count += len;
	done = 0;
	src = data;
	memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE);

	if ((partial + len) >= SHA256_BLOCK_SIZE) {

		/* Append the bytes in state's buffer to a block to handle */
		if (partial) {
			done = -partial;
			memcpy(sctx->buf + partial, data,
				done + SHA256_BLOCK_SIZE);
			src = sctx->buf;
			ts_state = irq_ts_save();
			asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
			: "+S"(src), "+D"(dst)
			: "a"((long)-1), "c"((unsigned long)1));
			irq_ts_restore(ts_state);
			done += SHA256_BLOCK_SIZE;
			src = data + done;
		}

		/* Process the left bytes from input data*/
		if (len - done >= SHA256_BLOCK_SIZE) {
			ts_state = irq_ts_save();
			asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
			: "+S"(src), "+D"(dst)
			: "a"((long)-1),
			"c"((unsigned long)((len - done) / 64)));
			irq_ts_restore(ts_state);
			done += ((len - done) - (len - done) % 64);
			src = data + done;
		}
		partial = 0;
	}
	memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE);
	memcpy(sctx->buf + partial, src, len - done);

	return 0;
}

Project: linux

static int padlock_sha_export(struct shash_desc *desc, void *out)
{
	struct padlock_sha_desc *dctx = shash_desc_ctx(desc);

	return crypto_shash_export(&dctx->fallback, out);
}

Project: linux

static int padlock_sha_export_nano(struct shash_desc *desc,
				void *out)
{
	int statesize = crypto_shash_statesize(desc->tfm);
	void *sctx = shash_desc_ctx(desc);

	memcpy(out, sctx, statesize);
	return 0;
}

Project: linux

static int padlock_sha_import(struct shash_desc *desc, const void *in)
{
	struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
	struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);

	dctx->fallback.tfm = ctx->fallback;
	dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
	return crypto_shash_import(&dctx->fallback, in);
}

Project: linux

static int padlock_sha_update(struct shash_desc *desc,
			      const u8 *data, unsigned int length)
{
	struct padlock_sha_desc *dctx = shash_desc_ctx(desc);

	dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
	return crypto_shash_update(&dctx->fallback, data, length);
}

Project: linux

static int adf_add_key_value_data(struct adf_accel_dev *accel_dev,
				  const char *section,
				  const struct adf_user_cfg_key_val *key_val)
{
	if (key_val->type == ADF_HEX) {
		long *ptr = (long *)key_val->val;
		long val = *ptr;

		if (adf_cfg_add_key_value_param(accel_dev, section,
						key_val->key, (void *)val,
						key_val->type)) {
			pr_err("QAT: failed to add keyvalue.\n");
			return -EFAULT;
		}
	} else {
		if (adf_cfg_add_key_value_param(accel_dev, section,
						key_val->key, key_val->val,
						key_val->type)) {
			pr_err("QAT: failed to add keyvalue.\n");
			return -EFAULT;
		}
	}
	return 0;
}

Project: linux

static int adf_chr_drv_create(void)
{
	dev_t dev_id;
	struct device *drv_device;

	if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) {
		pr_err("QAT: unable to allocate chrdev region\n");
		return -EFAULT;
	}

	adt_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME);
	if (IS_ERR(adt_ctl_drv.drv_class)) {
		pr_err("QAT: class_create failed for adf_ctl\n");
		goto err_chrdev_unreg;
	}
	adt_ctl_drv.major = MAJOR(dev_id);
	cdev_init(&adt_ctl_drv.drv_cdev, &adf_ctl_ops);
	if (cdev_add(&adt_ctl_drv.drv_cdev, dev_id, 1)) {
		pr_err("QAT: cdev add failed\n");
		goto err_class_destr;
	}

	drv_device = device_create(adt_ctl_drv.drv_class, NULL,
				   MKDEV(adt_ctl_drv.major, 0),
				   NULL, DEVICE_NAME);
	if (IS_ERR(drv_device)) {
		pr_err("QAT: failed to create device\n");
		goto err_cdev_del;
	}
	return 0;
err_cdev_del:
	cdev_del(&adt_ctl_drv.drv_cdev);
err_class_destr:
	class_destroy(adt_ctl_drv.drv_class);
err_chrdev_unreg:
	unregister_chrdev_region(dev_id, 1);
	return -EFAULT;
}

Project: linux

static void adf_chr_drv_destroy(void)
{
	device_destroy(adt_ctl_drv.drv_class, MKDEV(adt_ctl_drv.major, 0));
	cdev_del(&adt_ctl_drv.drv_cdev);
	class_destroy(adt_ctl_drv.drv_class);
	unregister_chrdev_region(MKDEV(adt_ctl_drv.major, 0), 1);
}

Project: linux

static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev,
				   struct adf_user_cfg_ctl_data *ctl_data)
{
	struct adf_user_cfg_key_val key_val;
	struct adf_user_cfg_key_val *params_head;
	struct adf_user_cfg_section section, *section_head;

	section_head = ctl_data->config_section;

	while (section_head) {
		if (copy_from_user(&section, (void __user *)section_head,
				   sizeof(*section_head))) {
			pr_err("QAT: failed to copy section info\n");
			goto out_err;
		}

		if (adf_cfg_section_add(accel_dev, section.name)) {
			pr_err("QAT: failed to add section.\n");
			goto out_err;
		}

		params_head = section_head->params;

		while (params_head) {
			if (copy_from_user(&key_val, (void __user *)params_head,
					   sizeof(key_val))) {
				pr_err("QAT: Failed to copy keyvalue.\n");
				goto out_err;
			}
			if (adf_add_key_value_data(accel_dev, section.name,
						   &key_val)) {
				goto out_err;
			}
			params_head = key_val.next;
		}
		section_head = section.next;
	}
	return 0;
out_err:
	adf_cfg_del_all(accel_dev);
	return -EFAULT;
}

Project: linux

static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data,
				   unsigned long arg)
{
	struct adf_user_cfg_ctl_data *cfg_data;

	cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL);
	if (!cfg_data)
		return -ENOMEM;

	/* Initialize device id to NO DEVICE as 0 is a valid device id */
	cfg_data->device_id = ADF_CFG_NO_DEVICE;

	if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) {
		pr_err("QAT: failed to copy from user cfg_data.\n");
		kfree(cfg_data);
		return -EIO;
	}

	*ctl_data = cfg_data;
	return 0;
}

Project: linux

static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
{
	int ret;

	if (mutex_lock_interruptible(&adf_ctl_lock))
		return -EFAULT;

	switch (cmd) {
	case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS:
		ret = adf_ctl_ioctl_dev_config(fp, cmd, arg);
		break;

	case IOCTL_STOP_ACCEL_DEV:
		ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg);
		break;

	case IOCTL_START_ACCEL_DEV:
		ret = adf_ctl_ioctl_dev_start(fp, cmd, arg);
		break;

	case IOCTL_GET_NUM_DEVICES:
		ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg);
		break;

	case IOCTL_STATUS_ACCEL_DEV:
		ret = adf_ctl_ioctl_get_status(fp, cmd, arg);
		break;
	default:
		pr_err("QAT: Invalid ioctl\n");
		ret = -EFAULT;
		break;
	}
	mutex_unlock(&adf_ctl_lock);
	return ret;
}

Project: linux

static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd,
				    unsigned long arg)
{
	int ret;
	struct adf_user_cfg_ctl_data *ctl_data;
	struct adf_accel_dev *accel_dev;

	ret = adf_ctl_alloc_resources(&ctl_data, arg);
	if (ret)
		return ret;

	accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id);
	if (!accel_dev) {
		ret = -EFAULT;
		goto out;
	}

	if (adf_dev_started(accel_dev)) {
		ret = -EFAULT;
		goto out;
	}

	if (adf_copy_key_value_data(accel_dev, ctl_data)) {
		ret = -EFAULT;
		goto out;
	}
	set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
out:
	kfree(ctl_data);
	return ret;
}

Project: linux

static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd,
				  unsigned long arg)
{
	int ret;
	struct adf_user_cfg_ctl_data *ctl_data;

	ret = adf_ctl_alloc_resources(&ctl_data, arg);
	if (ret)
		return ret;

	if (adf_devmgr_verify_id(ctl_data->device_id)) {
		pr_err("QAT: Device %d not found\n", ctl_data->device_id);
		ret = -ENODEV;
		goto out;
	}

	ret = adf_ctl_is_device_in_use(ctl_data->device_id);
	if (ret)
		goto out;

	if (ctl_data->device_id == ADF_CFG_ALL_DEVICES)
		pr_info("QAT: Stopping all acceleration devices.\n");
	else
		pr_info("QAT: Stopping acceleration device qat_dev%d.\n",
			ctl_data->device_id);

	ret = adf_ctl_stop_devices(ctl_data->device_id);
	if (ret)
		pr_err("QAT: failed to stop device.\n");
out:
	kfree(ctl_data);
	return ret;
}

Project: linux

static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd,
					 unsigned long arg)
{
	uint32_t num_devices = 0;

	adf_devmgr_get_num_dev(&num_devices);
	if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices)))
		return -EFAULT;

	return 0;
}

Project: linux

static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd,
				    unsigned long arg)
{
	struct adf_hw_device_data *hw_data;
	struct adf_dev_status_info dev_info;
	struct adf_accel_dev *accel_dev;

	if (copy_from_user(&dev_info, (void __user *)arg,
			   sizeof(struct adf_dev_status_info))) {
		pr_err("QAT: failed to copy from user.\n");
		return -EFAULT;
	}

	accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id);
	if (!accel_dev) {
		pr_err("QAT: Device %d not found\n", dev_info.accel_id);
		return -ENODEV;
	}
	hw_data = accel_dev->hw_device;
	dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN;
	dev_info.num_ae = hw_data->get_num_aes(hw_data);
	dev_info.num_accel = hw_data->get_num_accels(hw_data);
	dev_info.num_logical_accel = hw_data->num_logical_accel;
	dev_info.banks_per_accel = hw_data->num_banks
					/ hw_data->num_logical_accel;
	strlcpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name));
	dev_info.instance_id = hw_data->instance_id;
	dev_info.type = hw_data->dev_class->type;
	dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number;
	dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn);
	dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn);

	if (copy_to_user((void __user *)arg, &dev_info,
			 sizeof(struct adf_dev_status_info))) {
		pr_err("QAT: failed to copy status.\n");
		return -EFAULT;
	}
	return 0;
}

Project: linux

static int adf_ctl_is_device_in_use(int id)
{
	struct list_head *itr, *head = adf_devmgr_get_head();

	list_for_each(itr, head) {
		struct adf_accel_dev *dev =
				list_entry(itr, struct adf_accel_dev, list);

		if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
			if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) {
				pr_info("QAT: device qat_dev%d is busy\n",
					dev->accel_id);
				return -EBUSY;
			}
		}
	}
	return 0;
}

Project: linux

static int adf_ctl_stop_devices(uint32_t id)
{
	struct list_head *itr, *head = adf_devmgr_get_head();
	int ret = 0;

	list_for_each(itr, head) {
		struct adf_accel_dev *accel_dev =
				list_entry(itr, struct adf_accel_dev, list);
		if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) {
			if (!adf_dev_started(accel_dev))
				continue;

			if (adf_dev_stop(accel_dev)) {
				pr_err("QAT: Failed to stop qat_dev%d\n", id);
				ret = -EFAULT;
			}
		}
	}
	return ret;
}

Project: linux

static int __init adf_register_ctl_device_driver(void)
{
	mutex_init(&adf_ctl_lock);

	if (qat_algs_init())
		goto err_algs_init;

	if (adf_chr_drv_create())
		goto err_chr_dev;

	if (adf_init_aer())
		goto err_aer;

	if (qat_crypto_register())
		goto err_crypto_register;

	return 0;

err_crypto_register:
	adf_exit_aer();
err_aer:
	adf_chr_drv_destroy();
err_chr_dev:
	qat_algs_exit();
err_algs_init:
	mutex_destroy(&adf_ctl_lock);
	return -EFAULT;
}

Project: linux

static void __exit adf_unregister_ctl_device_driver(void)
{
	adf_chr_drv_destroy();
	adf_exit_aer();
	qat_crypto_unregister();
	qat_algs_exit();
	mutex_destroy(&adf_ctl_lock);
}

Project: linux

static int ablk_crypt(struct ablkcipher_request *areq)
{
	struct ablkcipher_walk walk;
	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
	struct cryp_device_data *device_data;
	unsigned long src_paddr;
	unsigned long dst_paddr;
	int ret;
	int nbytes;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	ret = cryp_get_device_data(ctx, &device_data);
	if (ret)
		goto out;

	ablkcipher_walk_init(&walk, areq->dst, areq->src, areq->nbytes);
	ret = ablkcipher_walk_phys(areq, &walk);

	if (ret) {
		pr_err(DEV_DBG_NAME "[%s]: ablkcipher_walk_phys() failed!",
			__func__);
		goto out;
	}

	while ((nbytes = walk.nbytes) > 0) {
		ctx->iv = walk.iv;
		src_paddr = (page_to_phys(walk.src.page) + walk.src.offset);
		ctx->indata = phys_to_virt(src_paddr);

		dst_paddr = (page_to_phys(walk.dst.page) + walk.dst.offset);
		ctx->outdata = phys_to_virt(dst_paddr);

		ctx->datalen = nbytes - (nbytes % ctx->blocksize);

		ret = hw_crypt_noxts(ctx, device_data);
		if (ret)
			goto out;

		nbytes -= ctx->datalen;
		ret = ablkcipher_walk_done(areq, &walk, nbytes);
		if (ret)
			goto out;
	}
	ablkcipher_walk_complete(&walk);

out:
	/* Release the device */
	spin_lock(&device_data->ctx_lock);
	device_data->current_ctx = NULL;
	ctx->device = NULL;
	spin_unlock(&device_data->ctx_lock);

	/*
	 * The down_interruptible part for this semaphore is called in
	 * cryp_get_device_data.
	 */
	up(&driver_data.device_allocation);

	return ret;
}

Project: linux

static int ablk_dma_crypt(struct ablkcipher_request *areq)
{
	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
	struct cryp_device_data *device_data;

	int bytes_written = 0;
	int bytes_read = 0;
	int ret;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	ctx->datalen = areq->nbytes;
	ctx->outlen = areq->nbytes;

	ret = cryp_get_device_data(ctx, &device_data);
	if (ret)
		return ret;

	ret = cryp_setup_context(ctx, device_data);
	if (ret)
		goto out;

	/* We have the device now, so store the nents in the dma struct. */
	ctx->device->dma.nents_src = get_nents(areq->src, ctx->datalen);
	ctx->device->dma.nents_dst = get_nents(areq->dst, ctx->outlen);

	/* Enable DMA in- and output. */
	cryp_configure_for_dma(device_data, CRYP_DMA_ENABLE_BOTH_DIRECTIONS);

	bytes_written = cryp_dma_write(ctx, areq->src, ctx->datalen);
	bytes_read = cryp_dma_read(ctx, areq->dst, bytes_written);

	wait_for_completion(&ctx->device->dma.cryp_dma_complete);
	cryp_dma_done(ctx);

	cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
	ctx->updated = 1;

out:
	spin_lock(&device_data->ctx_lock);
	device_data->current_ctx = NULL;
	ctx->device = NULL;
	spin_unlock(&device_data->ctx_lock);

	/*
	 * The down_interruptible part for this semaphore is called in
	 * cryp_get_device_data.
	 */
	up(&driver_data.device_allocation);

	if (unlikely(bytes_written != bytes_read))
		return -EPERM;

	return 0;
}

Project: linux

static void add_session_id(struct cryp_ctx *ctx)
{
	/*
	 * We never want 0 to be a valid value, since this is the default value
	 * for the software context.
	 */
	if (unlikely(atomic_inc_and_test(&session_id)))
		atomic_inc(&session_id);

	ctx->session_id = atomic_read(&session_id);
}

Project: linux

static int aes_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
				 const u8 *key, unsigned int keylen)
{
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
	u32 *flags = &cipher->base.crt_flags;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	switch (keylen) {
	case AES_KEYSIZE_128:
		ctx->config.keysize = CRYP_KEY_SIZE_128;
		break;

	case AES_KEYSIZE_192:
		ctx->config.keysize = CRYP_KEY_SIZE_192;
		break;

	case AES_KEYSIZE_256:
		ctx->config.keysize = CRYP_KEY_SIZE_256;
		break;

	default:
		pr_err(DEV_DBG_NAME "[%s]: Unknown keylen!", __func__);
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		return -EINVAL;
	}

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;

	ctx->updated = 0;

	return 0;
}

Project: linux

static int cfg_ivs(struct cryp_device_data *device_data, struct cryp_ctx *ctx)
{
	int i;
	int status = 0;
	int num_of_regs = ctx->blocksize / 8;
	u32 iv[AES_BLOCK_SIZE / 4];

	dev_dbg(device_data->dev, "[%s]", __func__);

	/*
	 * Since we loop on num_of_regs we need to have a check in case
	 * someone provides an incorrect blocksize which would force calling
	 * cfg_iv with i greater than 2 which is an error.
	 */
	if (num_of_regs > 2) {
		dev_err(device_data->dev, "[%s] Incorrect blocksize %d",
			__func__, ctx->blocksize);
		return -EINVAL;
	}

	for (i = 0; i < ctx->blocksize / 4; i++)
		iv[i] = uint8p_to_uint32_be(ctx->iv + i*4);

	for (i = 0; i < num_of_regs; i++) {
		status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
				(enum cryp_init_vector_index) i);
		if (status != 0)
			return status;
	}
	return status;
}

Project: linux

static int cfg_keys(struct cryp_ctx *ctx)
{
	int i;
	int num_of_regs = ctx->keylen / 8;
	u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
	int cryp_error = 0;

	dev_dbg(ctx->device->dev, "[%s]", __func__);

	if (mode_is_aes(ctx->config.algomode)) {
		swap_words_in_key_and_bits_in_byte((u8 *)ctx->key,
						   (u8 *)swapped_key,
						   ctx->keylen);
	} else {
		for (i = 0; i < ctx->keylen / 4; i++)
			swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4);
	}

	for (i = 0; i < num_of_regs; i++) {
		cryp_error = set_key(ctx->device,
				     *(((u32 *)swapped_key)+i*2),
				     *(((u32 *)swapped_key)+i*2+1),
				     (enum cryp_key_reg_index) i);

		if (cryp_error != 0) {
			dev_err(ctx->device->dev, "[%s]: set_key() failed!",
					__func__);
			return cryp_error;
		}
	}
	return cryp_error;
}

Project: linux

static int cryp_algs_register_all(void)
{
	int ret;
	int i;
	int count;

	pr_debug("[%s]", __func__);

	for (i = 0; i < ARRAY_SIZE(cryp_algs); i++) {
		ret = crypto_register_alg(&cryp_algs[i].crypto);
		if (ret) {
			count = i;
			pr_err("[%s] alg registration failed",
					cryp_algs[i].crypto.cra_driver_name);
			goto unreg;
		}
	}
	return 0;
unreg:
	for (i = 0; i < count; i++)
		crypto_unregister_alg(&cryp_algs[i].crypto);
	return ret;
}

Project: linux

static void cryp_algs_unregister_all(void)
{
	int i;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	for (i = 0; i < ARRAY_SIZE(cryp_algs); i++)
		crypto_unregister_alg(&cryp_algs[i].crypto);
}

Project: linux

static int cryp_blk_decrypt(struct ablkcipher_request *areq)
{
	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	ctx->config.algodir = CRYP_ALGORITHM_DECRYPT;

	/* DMA does not work for DES due to a hw bug */
	if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
		return ablk_dma_crypt(areq);

	/* For everything except DMA, we run the non DMA version. */
	return ablk_crypt(areq);
}

Project: linux

static int cryp_blk_encrypt(struct ablkcipher_request *areq)
{
	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	ctx->config.algodir = CRYP_ALGORITHM_ENCRYPT;

	/*
	 * DMA does not work for DES due to a hw bug */
	if (cryp_mode == CRYP_MODE_DMA && mode_is_aes(ctx->config.algomode))
		return ablk_dma_crypt(areq);

	/* For everything except DMA, we run the non DMA version. */
	return ablk_crypt(areq);
}

Project: linux

static int cryp_cra_init(struct crypto_tfm *tfm)
{
	struct cryp_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_alg *alg = tfm->__crt_alg;
	struct cryp_algo_template *cryp_alg = container_of(alg,
			struct cryp_algo_template,
			crypto);

	ctx->config.algomode = cryp_alg->algomode;
	ctx->blocksize = crypto_tfm_alg_blocksize(tfm);

	return 0;
}

Project: linux

static int cryp_disable_power(struct device *dev,
			      struct cryp_device_data *device_data,
			      bool save_device_context)
{
	int ret = 0;

	dev_dbg(dev, "[%s]", __func__);

	spin_lock(&device_data->power_state_spinlock);
	if (!device_data->power_state)
		goto out;

	spin_lock(&device_data->ctx_lock);
	if (save_device_context && device_data->current_ctx) {
		cryp_save_device_context(device_data,
				&device_data->current_ctx->dev_ctx,
				cryp_mode);
		device_data->restore_dev_ctx = true;
	}
	spin_unlock(&device_data->ctx_lock);

	clk_disable(device_data->clk);
	ret = regulator_disable(device_data->pwr_regulator);
	if (ret)
		dev_err(dev, "[%s]: "
				"regulator_disable() failed!",
				__func__);

	device_data->power_state = false;

out:
	spin_unlock(&device_data->power_state_spinlock);

	return ret;
}

Project: linux

static void cryp_dma_out_callback(void *data)
{
	struct cryp_ctx *ctx = (struct cryp_ctx *) data;
	dev_dbg(ctx->device->dev, "[%s]: ", __func__);

	complete(&ctx->device->dma.cryp_dma_complete);
}

Project: linux

static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len)
{
	int error = cryp_set_dma_transfer(ctx, sg, len, DMA_FROM_DEVICE);
	if (error) {
		dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
			"failed", __func__);
		return error;
	}

	return len;
}

Project: linux

static void cryp_dma_setup_channel(struct cryp_device_data *device_data,
				   struct device *dev)
{
	struct dma_slave_config mem2cryp = {
		.direction = DMA_MEM_TO_DEV,
		.dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
		.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
		.dst_maxburst = 4,
        };
	struct dma_slave_config cryp2mem = {
		.direction = DMA_DEV_TO_MEM,
		.src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
		.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
		.src_maxburst = 4,
        };

	dma_cap_zero(device_data->dma.mask);
	dma_cap_set(DMA_SLAVE, device_data->dma.mask);

	device_data->dma.cfg_mem2cryp = mem_to_engine;
	device_data->dma.chan_mem2cryp =
		dma_request_channel(device_data->dma.mask,
				    stedma40_filter,
				    device_data->dma.cfg_mem2cryp);

	device_data->dma.cfg_cryp2mem = engine_to_mem;
	device_data->dma.chan_cryp2mem =
		dma_request_channel(device_data->dma.mask,
				    stedma40_filter,
				    device_data->dma.cfg_cryp2mem);

	dmaengine_slave_config(device_data->dma.chan_mem2cryp, &mem2cryp);
	dmaengine_slave_config(device_data->dma.chan_cryp2mem, &cryp2mem);

	init_completion(&device_data->dma.cryp_dma_complete);
}

Project: linux

static int cryp_dma_write(struct cryp_ctx *ctx, struct scatterlist *sg,
			  int len)
{
	int error = cryp_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
	dev_dbg(ctx->device->dev, "[%s]: ", __func__);

	if (error) {
		dev_dbg(ctx->device->dev, "[%s]: cryp_set_dma_transfer() "
			"failed", __func__);
		return error;
	}

	return len;
}

Project: linux

static int cryp_enable_power(
		struct device *dev,
		struct cryp_device_data *device_data,
		bool restore_device_context)
{
	int ret = 0;

	dev_dbg(dev, "[%s]", __func__);

	spin_lock(&device_data->power_state_spinlock);
	if (!device_data->power_state) {
		ret = regulator_enable(device_data->pwr_regulator);
		if (ret) {
			dev_err(dev, "[%s]: regulator_enable() failed!",
					__func__);
			goto out;
		}

		ret = clk_enable(device_data->clk);
		if (ret) {
			dev_err(dev, "[%s]: clk_enable() failed!",
					__func__);
			regulator_disable(device_data->pwr_regulator);
			goto out;
		}
		device_data->power_state = true;
	}

	if (device_data->restore_dev_ctx) {
		spin_lock(&device_data->ctx_lock);
		if (restore_device_context && device_data->current_ctx) {
			device_data->restore_dev_ctx = false;
			cryp_restore_device_context(device_data,
					&device_data->current_ctx->dev_ctx);
		}
		spin_unlock(&device_data->ctx_lock);
	}
out:
	spin_unlock(&device_data->power_state_spinlock);

	return ret;
}

Project: linux

static int cryp_get_device_data(struct cryp_ctx *ctx,
				struct cryp_device_data **device_data)
{
	int ret;
	struct klist_iter device_iterator;
	struct klist_node *device_node;
	struct cryp_device_data *local_device_data = NULL;
	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	/* Wait until a device is available */
	ret = down_interruptible(&driver_data.device_allocation);
	if (ret)
		return ret;  /* Interrupted */

	/* Select a device */
	klist_iter_init(&driver_data.device_list, &device_iterator);

	device_node = klist_next(&device_iterator);
	while (device_node) {
		local_device_data = container_of(device_node,
					   struct cryp_device_data, list_node);
		spin_lock(&local_device_data->ctx_lock);
		/* current_ctx allocates a device, NULL = unallocated */
		if (local_device_data->current_ctx) {
			device_node = klist_next(&device_iterator);
		} else {
			local_device_data->current_ctx = ctx;
			ctx->device = local_device_data;
			spin_unlock(&local_device_data->ctx_lock);
			break;
		}
		spin_unlock(&local_device_data->ctx_lock);
	}
	klist_iter_exit(&device_iterator);

	if (!device_node) {
		/**
		 * No free device found.
		 * Since we allocated a device with down_interruptible, this
		 * should not be able to happen.
		 * Number of available devices, which are contained in
		 * device_allocation, is therefore decremented by not doing
		 * an up(device_allocation).
		 */
		return -EBUSY;
	}

	*device_data = local_device_data;

	return 0;
}

Project: linux

static irqreturn_t cryp_interrupt_handler(int irq, void *param)
{
	struct cryp_ctx *ctx;
	int count;
	struct cryp_device_data *device_data;

	if (param == NULL) {
		BUG_ON(!param);
		return IRQ_HANDLED;
	}

	/* The device is coming from the one found in hw_crypt_noxts. */
	device_data = (struct cryp_device_data *)param;

	ctx = device_data->current_ctx;

	if (ctx == NULL) {
		BUG_ON(!ctx);
		return IRQ_HANDLED;
	}

	dev_dbg(ctx->device->dev, "[%s] (len: %d) %s, ", __func__, ctx->outlen,
		cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO) ?
		"out" : "in");

	if (cryp_pending_irq_src(device_data,
				 CRYP_IRQ_SRC_OUTPUT_FIFO)) {
		if (ctx->outlen / ctx->blocksize > 0) {
			count = ctx->blocksize / 4;

			readsl(&device_data->base->dout, ctx->outdata, count);
			ctx->outdata += count;
			ctx->outlen -= count;

			if (ctx->outlen == 0) {
				cryp_disable_irq_src(device_data,
						     CRYP_IRQ_SRC_OUTPUT_FIFO);
			}
		}
	} else if (cryp_pending_irq_src(device_data,
					CRYP_IRQ_SRC_INPUT_FIFO)) {
		if (ctx->datalen / ctx->blocksize > 0) {
			count = ctx->blocksize / 4;

			writesl(&device_data->base->din, ctx->indata, count);

			ctx->indata += count;
			ctx->datalen -= count;

			if (ctx->datalen == 0)
				cryp_disable_irq_src(device_data,
						   CRYP_IRQ_SRC_INPUT_FIFO);

			if (ctx->config.algomode == CRYP_ALGO_AES_XTS) {
				CRYP_PUT_BITS(&device_data->base->cr,
					      CRYP_START_ENABLE,
					      CRYP_CR_START_POS,
					      CRYP_CR_START_MASK);

				cryp_wait_until_done(device_data);
			}
		}
	}

	return IRQ_HANDLED;
}

Project: linux

static void cryp_polling_mode(struct cryp_ctx *ctx,
			      struct cryp_device_data *device_data)
{
	int len = ctx->blocksize / BYTES_PER_WORD;
	int remaining_length = ctx->datalen;
	u32 *indata = (u32 *)ctx->indata;
	u32 *outdata = (u32 *)ctx->outdata;

	while (remaining_length > 0) {
		writesl(&device_data->base->din, indata, len);
		indata += len;
		remaining_length -= (len * BYTES_PER_WORD);
		cryp_wait_until_done(device_data);

		readsl(&device_data->base->dout, outdata, len);
		outdata += len;
		cryp_wait_until_done(device_data);
	}
}

Project: linux

static int cryp_set_dma_transfer(struct cryp_ctx *ctx,
				 struct scatterlist *sg,
				 int len,
				 enum dma_data_direction direction)
{
	struct dma_async_tx_descriptor *desc;
	struct dma_chan *channel = NULL;
	dma_cookie_t cookie;

	dev_dbg(ctx->device->dev, "[%s]: ", __func__);

	if (unlikely(!IS_ALIGNED((u32)sg, 4))) {
		dev_err(ctx->device->dev, "[%s]: Data in sg list isn't "
			"aligned! Addr: 0x%08x", __func__, (u32)sg);
		return -EFAULT;
	}

	switch (direction) {
	case DMA_TO_DEVICE:
		channel = ctx->device->dma.chan_mem2cryp;
		ctx->device->dma.sg_src = sg;
		ctx->device->dma.sg_src_len = dma_map_sg(channel->device->dev,
						 ctx->device->dma.sg_src,
						 ctx->device->dma.nents_src,
						 direction);

		if (!ctx->device->dma.sg_src_len) {
			dev_dbg(ctx->device->dev,
				"[%s]: Could not map the sg list (TO_DEVICE)",
				__func__);
			return -EFAULT;
		}

		dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
			"(TO_DEVICE)", __func__);

		desc = dmaengine_prep_slave_sg(channel,
				ctx->device->dma.sg_src,
				ctx->device->dma.sg_src_len,
				direction, DMA_CTRL_ACK);
		break;

	case DMA_FROM_DEVICE:
		channel = ctx->device->dma.chan_cryp2mem;
		ctx->device->dma.sg_dst = sg;
		ctx->device->dma.sg_dst_len = dma_map_sg(channel->device->dev,
						 ctx->device->dma.sg_dst,
						 ctx->device->dma.nents_dst,
						 direction);

		if (!ctx->device->dma.sg_dst_len) {
			dev_dbg(ctx->device->dev,
				"[%s]: Could not map the sg list (FROM_DEVICE)",
				__func__);
			return -EFAULT;
		}

		dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer "
			"(FROM_DEVICE)", __func__);

		desc = dmaengine_prep_slave_sg(channel,
				ctx->device->dma.sg_dst,
				ctx->device->dma.sg_dst_len,
				direction,
				DMA_CTRL_ACK |
				DMA_PREP_INTERRUPT);

		desc->callback = cryp_dma_out_callback;
		desc->callback_param = ctx;
		break;

	default:
		dev_dbg(ctx->device->dev, "[%s]: Invalid DMA direction",
			__func__);
		return -EFAULT;
	}

	cookie = dmaengine_submit(desc);
	dma_async_issue_pending(channel);

	return 0;
}

Project: linux

static int cryp_setup_context(struct cryp_ctx *ctx,
			      struct cryp_device_data *device_data)
{
	u32 control_register = CRYP_CR_DEFAULT;

	switch (cryp_mode) {
	case CRYP_MODE_INTERRUPT:
		writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc);
		break;

	case CRYP_MODE_DMA:
		writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr);
		break;

	default:
		break;
	}

	if (ctx->updated == 0) {
		cryp_flush_inoutfifo(device_data);
		if (cfg_keys(ctx) != 0) {
			dev_err(ctx->device->dev, "[%s]: cfg_keys failed!",
				__func__);
			return -EINVAL;
		}

		if (ctx->iv &&
		    CRYP_ALGO_AES_ECB != ctx->config.algomode &&
		    CRYP_ALGO_DES_ECB != ctx->config.algomode &&
		    CRYP_ALGO_TDES_ECB != ctx->config.algomode) {
			if (cfg_ivs(device_data, ctx) != 0)
				return -EPERM;
		}

		cryp_set_configuration(device_data, &ctx->config,
				       &control_register);
		add_session_id(ctx);
	} else if (ctx->updated == 1 &&
		   ctx->session_id != atomic_read(&session_id)) {
		cryp_flush_inoutfifo(device_data);
		cryp_restore_device_context(device_data, &ctx->dev_ctx);

		add_session_id(ctx);
		control_register = ctx->dev_ctx.cr;
	} else
		control_register = ctx->dev_ctx.cr;

	writel(control_register |
	       (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS),
	       &device_data->base->cr);

	return 0;
}

Project: linux

static int des3_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
				  const u8 *key, unsigned int keylen)
{
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
	u32 *flags = &cipher->base.crt_flags;
	const u32 *K = (const u32 *)key;
	u32 tmp[DES3_EDE_EXPKEY_WORDS];
	int i, ret;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);
	if (keylen != DES3_EDE_KEY_SIZE) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
				__func__);
		return -EINVAL;
	}

	/* Checking key interdependency for weak key detection. */
	if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
				!((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
			(*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
		*flags |= CRYPTO_TFM_RES_WEAK_KEY;
		pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
				__func__);
		return -EINVAL;
	}
	for (i = 0; i < 3; i++) {
		ret = des_ekey(tmp, key + i*DES_KEY_SIZE);
		if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
			*flags |= CRYPTO_TFM_RES_WEAK_KEY;
			pr_debug(DEV_DBG_NAME " [%s]: "
					"CRYPTO_TFM_REQ_WEAK_KEY", __func__);
			return -EINVAL;
		}
	}

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;

	ctx->updated = 0;
	return 0;
}

Project: linux

static int des_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
				 const u8 *key, unsigned int keylen)
{
	struct cryp_ctx *ctx = crypto_ablkcipher_ctx(cipher);
	u32 *flags = &cipher->base.crt_flags;
	u32 tmp[DES_EXPKEY_WORDS];
	int ret;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);
	if (keylen != DES_KEY_SIZE) {
		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
		pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_RES_BAD_KEY_LEN",
				__func__);
		return -EINVAL;
	}

	ret = des_ekey(tmp, key);
	if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
		*flags |= CRYPTO_TFM_RES_WEAK_KEY;
		pr_debug(DEV_DBG_NAME " [%s]: CRYPTO_TFM_REQ_WEAK_KEY",
				__func__);
		return -EINVAL;
	}

	memcpy(ctx->key, key, keylen);
	ctx->keylen = keylen;

	ctx->updated = 0;
	return 0;
}

Project: linux

static int hw_crypt_noxts(struct cryp_ctx *ctx,
			  struct cryp_device_data *device_data)
{
	int ret = 0;

	const u8 *indata = ctx->indata;
	u8 *outdata = ctx->outdata;
	u32 datalen = ctx->datalen;
	u32 outlen = datalen;

	pr_debug(DEV_DBG_NAME " [%s]", __func__);

	ctx->outlen = ctx->datalen;

	if (unlikely(!IS_ALIGNED((u32)indata, 4))) {
		pr_debug(DEV_DBG_NAME " [%s]: Data isn't aligned! Addr: "
			 "0x%08x", __func__, (u32)indata);
		return -EINVAL;
	}

	ret = cryp_setup_context(ctx, device_data);

	if (ret)
		goto out;

	if (cryp_mode == CRYP_MODE_INTERRUPT) {
		cryp_enable_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO |
				    CRYP_IRQ_SRC_OUTPUT_FIFO);

		/*
		 * ctx->outlen is decremented in the cryp_interrupt_handler
		 * function. We had to add cpu_relax() (barrier) to make sure
		 * that gcc didn't optimze away this variable.
		 */
		while (ctx->outlen > 0)
			cpu_relax();
	} else if (cryp_mode == CRYP_MODE_POLLING ||
		   cryp_mode == CRYP_MODE_DMA) {
		/*
		 * The reason for having DMA in this if case is that if we are
		 * running cryp_mode = 2, then we separate DMA routines for
		 * handling cipher/plaintext > blocksize, except when
		 * running the normal CRYPTO_ALG_TYPE_CIPHER, then we still use
		 * the polling mode. Overhead of doing DMA setup eats up the
		 * benefits using it.
		 */
		cryp_polling_mode(ctx, device_data);
	} else {
		dev_err(ctx->device->dev, "[%s]: Invalid operation mode!",
			__func__);
		ret = -EPERM;
		goto out;
	}

	cryp_save_device_context(device_data, &ctx->dev_ctx, cryp_mode);
	ctx->updated = 1;

out:
	ctx->indata = indata;
	ctx->outdata = outdata;
	ctx->datalen = datalen;
	ctx->outlen = outlen;

	return ret;
}

Project: linux

static int mode_is_aes(enum cryp_algo_mode mode)
{
	return	CRYP_ALGO_AES_ECB == mode ||
		CRYP_ALGO_AES_CBC == mode ||
		CRYP_ALGO_AES_CTR == mode ||
		CRYP_ALGO_AES_XTS == mode;
}

Project: linux

static int set_key(struct cryp_device_data *device_data,
		   u32 left_key,
		   u32 right_key,
		   enum cryp_key_reg_index index)
{
	struct cryp_key_value key_value;
	int cryp_error;

	dev_dbg(device_data->dev, "[%s]", __func__);

	key_value.key_value_left = left_key;
	key_value.key_value_right = right_key;

	cryp_error = cryp_configure_key_values(device_data,
					       index,
					       key_value);
	if (cryp_error != 0)
		dev_err(device_data->dev, "[%s]: "
			"cryp_configure_key_values() failed!", __func__);

	return cryp_error;
}

Project: linux

static inline u8 swap_bits_in_byte(u8 b)
{
#define R_SHIFT_4_MASK  0xc0 /* Bits 6 and 7, right shift 4 */
#define R_SHIFT_2_MASK  0x28 /* (After right shift 4) Bits 3 and 5,
				  right shift 2 */
#define R_SHIFT_1_MASK  0x1e /* (After right shift 2) Bits 1-4,
				  right shift 1 */
#define L_SHIFT_4_MASK  0x03 /* Bits 0 and 1, left shift 4 */
#define L_SHIFT_2_MASK  0x14 /* (After left shift 4) Bits 2 and 4,
				  left shift 2 */
#define L_SHIFT_1_MASK  0x78 /* (After left shift 1) Bits 3-6,
				  left shift 1 */

	u8 n1;
	u8 n2;

	/* Swap most significant nibble */
	/* Right shift 4, bits 6 and 7 */
	n1 = ((b  & R_SHIFT_4_MASK) >> 4) | (b  & ~(R_SHIFT_4_MASK >> 4));
	/* Right shift 2, bits 3 and 5 */
	n1 = ((n1 & R_SHIFT_2_MASK) >> 2) | (n1 & ~(R_SHIFT_2_MASK >> 2));
	/* Right shift 1, bits 1-4 */
	n1 = (n1  & R_SHIFT_1_MASK) >> 1;

	/* Swap least significant nibble */
	/* Left shift 4, bits 0 and 1 */
	n2 = ((b  & L_SHIFT_4_MASK) << 4) | (b  & ~(L_SHIFT_4_MASK << 4));
	/* Left shift 2, bits 2 and 4 */
	n2 = ((n2 & L_SHIFT_2_MASK) << 2) | (n2 & ~(L_SHIFT_2_MASK << 2));
	/* Left shift 1, bits 3-6 */
	n2 = (n2  & L_SHIFT_1_MASK) << 1;

	return n1 | n2;
}

Project: linux

static inline void swap_words_in_key_and_bits_in_byte(const u8 *in,
						      u8 *out, u32 len)
{
	unsigned int i = 0;
	int j;
	int index = 0;

	j = len - BYTES_PER_WORD;
	while (j >= 0) {
		for (i = 0; i < BYTES_PER_WORD; i++) {
			index = len - j - BYTES_PER_WORD + i;
			out[j + i] =
				swap_bits_in_byte(in[index]);
		}
		j -= BYTES_PER_WORD;
	}
}

Project: linux

static inline u32 uint8p_to_uint32_be(u8 *in)
{
	u32 *data = (u32 *)in;

	return cpu_to_be32p(data);
}

Project: linux

static void __exit ux500_cryp_mod_fini(void)
{
	pr_debug("[%s] is called!", __func__);
	platform_driver_unregister(&cryp_driver);
	return;
}

Project: linux

static int __init ux500_cryp_mod_init(void)
{
	pr_debug("[%s] is called!", __func__);
	klist_init(&driver_data.device_list, NULL, NULL);
	/* Initialize the semaphore to 0 devices (locked state) */
	sema_init(&driver_data.device_allocation, 0);
	return platform_driver_register(&cryp_driver);
}

Project: linux

static int ux500_cryp_probe(struct platform_device *pdev)
{
	int ret;
	int cryp_error = 0;
	struct resource *res = NULL;
	struct resource *res_irq = NULL;
	struct cryp_device_data *device_data;
	struct cryp_protection_config prot = {
		.privilege_access = CRYP_STATE_ENABLE
	};
	struct device *dev = &pdev->dev;

	dev_dbg(dev, "[%s]", __func__);
	device_data = kzalloc(sizeof(struct cryp_device_data), GFP_ATOMIC);
	if (!device_data) {
		dev_err(dev, "[%s]: kzalloc() failed!", __func__);
		ret = -ENOMEM;
		goto out;
	}

	device_data->dev = dev;
	device_data->current_ctx = NULL;

	/* Grab the DMA configuration from platform data. */
	mem_to_engine = &((struct cryp_platform_data *)
			 dev->platform_data)->mem_to_engine;
	engine_to_mem = &((struct cryp_platform_data *)
			 dev->platform_data)->engine_to_mem;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!res) {
		dev_err(dev, "[%s]: platform_get_resource() failed",
				__func__);
		ret = -ENODEV;
		goto out_kfree;
	}

	res = request_mem_region(res->start, resource_size(res), pdev->name);
	if (res == NULL) {
		dev_err(dev, "[%s]: request_mem_region() failed",
				__func__);
		ret = -EBUSY;
		goto out_kfree;
	}

	device_data->phybase = res->start;
	device_data->base = ioremap(res->start, resource_size(res));
	if (!device_data->base) {
		dev_err(dev, "[%s]: ioremap failed!", __func__);
		ret = -ENOMEM;
		goto out_free_mem;
	}

	spin_lock_init(&device_data->ctx_lock);
	spin_lock_init(&device_data->power_state_spinlock);

	/* Enable power for CRYP hardware block */
	device_data->pwr_regulator = regulator_get(&pdev->dev, "v-ape");
	if (IS_ERR(device_data->pwr_regulator)) {
		dev_err(dev, "[%s]: could not get cryp regulator", __func__);
		ret = PTR_ERR(device_data->pwr_regulator);
		device_data->pwr_regulator = NULL;
		goto out_unmap;
	}

	/* Enable the clk for CRYP hardware block */
	device_data->clk = clk_get(&pdev->dev, NULL);
	if (IS_ERR(device_data->clk)) {
		dev_err(dev, "[%s]: clk_get() failed!", __func__);
		ret = PTR_ERR(device_data->clk);
		goto out_regulator;
	}

	ret = clk_prepare(device_data->clk);
	if (ret) {
		dev_err(dev, "[%s]: clk_prepare() failed!", __func__);
		goto out_clk;
	}

	/* Enable device power (and clock) */
	ret = cryp_enable_power(device_data->dev, device_data, false);
	if (ret) {
		dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
		goto out_clk_unprepare;
	}

	cryp_error = cryp_check(device_data);
	if (cryp_error != 0) {
		dev_err(dev, "[%s]: cryp_init() failed!", __func__);
		ret = -EINVAL;
		goto out_power;
	}

	cryp_error = cryp_configure_protection(device_data, &prot);
	if (cryp_error != 0) {
		dev_err(dev, "[%s]: cryp_configure_protection() failed!",
			__func__);
		ret = -EINVAL;
		goto out_power;
	}

	res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
	if (!res_irq) {
		dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
			__func__);
		ret = -ENODEV;
		goto out_power;
	}

	ret = request_irq(res_irq->start,
			  cryp_interrupt_handler,
			  0,
			  "cryp1",
			  device_data);
	if (ret) {
		dev_err(dev, "[%s]: Unable to request IRQ", __func__);
		goto out_power;
	}

	if (cryp_mode == CRYP_MODE_DMA)
		cryp_dma_setup_channel(device_data, dev);

	platform_set_drvdata(pdev, device_data);

	/* Put the new device into the device list... */
	klist_add_tail(&device_data->list_node, &driver_data.device_list);

	/* ... and signal that a new device is available. */
	up(&driver_data.device_allocation);

	atomic_set(&session_id, 1);

	ret = cryp_algs_register_all();
	if (ret) {
		dev_err(dev, "[%s]: cryp_algs_register_all() failed!",
			__func__);
		goto out_power;
	}

	dev_info(dev, "successfully registered\n");

	return 0;

out_power:
	cryp_disable_power(device_data->dev, device_data, false);

out_clk_unprepare:
	clk_unprepare(device_data->clk);

out_clk:
	clk_put(device_data->clk);

out_regulator:
	regulator_put(device_data->pwr_regulator);

out_unmap:
	iounmap(device_data->base);

out_free_mem:
	release_mem_region(res->start, resource_size(res));

out_kfree:
	kfree(device_data);
out:
	return ret;
}

Project: linux

static int ux500_cryp_remove(struct platform_device *pdev)
{
	struct resource *res = NULL;
	struct resource *res_irq = NULL;
	struct cryp_device_data *device_data;

	dev_dbg(&pdev->dev, "[%s]", __func__);
	device_data = platform_get_drvdata(pdev);
	if (!device_data) {
		dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
			__func__);
		return -ENOMEM;
	}

	/* Try to decrease the number of available devices. */
	if (down_trylock(&driver_data.device_allocation))
		return -EBUSY;

	/* Check that the device is free */
	spin_lock(&device_data->ctx_lock);
	/* current_ctx allocates a device, NULL = unallocated */
	if (device_data->current_ctx) {
		/* The device is busy */
		spin_unlock(&device_data->ctx_lock);
		/* Return the device to the pool. */
		up(&driver_data.device_allocation);
		return -EBUSY;
	}

	spin_unlock(&device_data->ctx_lock);

	/* Remove the device from the list */
	if (klist_node_attached(&device_data->list_node))
		klist_remove(&device_data->list_node);

	/* If this was the last device, remove the services */
	if (list_empty(&driver_data.device_list.k_list))
		cryp_algs_unregister_all();

	res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
	if (!res_irq)
		dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable",
			__func__);
	else {
		disable_irq(res_irq->start);
		free_irq(res_irq->start, device_data);
	}

	if (cryp_disable_power(&pdev->dev, device_data, false))
		dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
			__func__);

	clk_unprepare(device_data->clk);
	clk_put(device_data->clk);
	regulator_put(device_data->pwr_regulator);

	iounmap(device_data->base);

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (res)
		release_mem_region(res->start, resource_size(res));

	kfree(device_data);

	return 0;
}

Project: linux

static int ux500_cryp_resume(struct device *dev)
{
	int ret = 0;
	struct platform_device *pdev = to_platform_device(dev);
	struct cryp_device_data *device_data;
	struct resource *res_irq;
	struct cryp_ctx *temp_ctx = NULL;

	dev_dbg(dev, "[%s]", __func__);

	device_data = platform_get_drvdata(pdev);
	if (!device_data) {
		dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
		return -ENOMEM;
	}

	spin_lock(&device_data->ctx_lock);
	if (device_data->current_ctx == ++temp_ctx)
		device_data->current_ctx = NULL;
	spin_unlock(&device_data->ctx_lock);


	if (!device_data->current_ctx)
		up(&driver_data.device_allocation);
	else
		ret = cryp_enable_power(dev, device_data, true);

	if (ret)
		dev_err(dev, "[%s]: cryp_enable_power() failed!", __func__);
	else {
		res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
		if (res_irq)
			enable_irq(res_irq->start);
	}

	return ret;
}

Project: linux

static void ux500_cryp_shutdown(struct platform_device *pdev)
{
	struct resource *res_irq = NULL;
	struct cryp_device_data *device_data;

	dev_dbg(&pdev->dev, "[%s]", __func__);

	device_data = platform_get_drvdata(pdev);
	if (!device_data) {
		dev_err(&pdev->dev, "[%s]: platform_get_drvdata() failed!",
			__func__);
		return;
	}

	/* Check that the device is free */
	spin_lock(&device_data->ctx_lock);
	/* current_ctx allocates a device, NULL = unallocated */
	if (!device_data->current_ctx) {
		if (down_trylock(&driver_data.device_allocation))
			dev_dbg(&pdev->dev, "[%s]: Cryp still in use!"
				"Shutting down anyway...", __func__);
		/**
		 * (Allocate the device)
		 * Need to set this to non-null (dummy) value,
		 * to avoid usage if context switching.
		 */
		device_data->current_ctx++;
	}
	spin_unlock(&device_data->ctx_lock);

	/* Remove the device from the list */
	if (klist_node_attached(&device_data->list_node))
		klist_remove(&device_data->list_node);

	/* If this was the last device, remove the services */
	if (list_empty(&driver_data.device_list.k_list))
		cryp_algs_unregister_all();

	res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
	if (!res_irq)
		dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable",
			__func__);
	else {
		disable_irq(res_irq->start);
		free_irq(res_irq->start, device_data);
	}

	if (cryp_disable_power(&pdev->dev, device_data, false))
		dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed",
			__func__);

}

Project: linux

static int ux500_cryp_suspend(struct device *dev)
{
	int ret;
	struct platform_device *pdev = to_platform_device(dev);
	struct cryp_device_data *device_data;
	struct resource *res_irq;
	struct cryp_ctx *temp_ctx = NULL;

	dev_dbg(dev, "[%s]", __func__);

	/* Handle state? */
	device_data = platform_get_drvdata(pdev);
	if (!device_data) {
		dev_err(dev, "[%s]: platform_get_drvdata() failed!", __func__);
		return -ENOMEM;
	}

	res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
	if (!res_irq)
		dev_err(dev, "[%s]: IORESOURCE_IRQ, unavailable", __func__);
	else
		disable_irq(res_irq->start);

	spin_lock(&device_data->ctx_lock);
	if (!device_data->current_ctx)
		device_data->current_ctx++;
	spin_unlock(&device_data->ctx_lock);

	if (device_data->current_ctx == ++temp_ctx) {
		if (down_interruptible(&driver_data.device_allocation))
			dev_dbg(dev, "[%s]: down_interruptible() failed",
				__func__);
		ret = cryp_disable_power(dev, device_data, false);

	} else
		ret = cryp_disable_power(dev, device_data, true);

	if (ret)
		dev_err(dev, "[%s]: cryp_disable_power()", __func__);

	return ret;
}

Project: linux

static int ahash_algs_register_all(struct hash_device_data *device_data)
{
	int ret;
	int i;
	int count;

	for (i = 0; i < ARRAY_SIZE(hash_algs); i++) {
		ret = crypto_register_ahash(&hash_algs[i].hash);
		if (ret) {
			count = i;
			dev_err(device_data->dev, "%s: alg registration failed\n",
				hash_algs[i].hash.halg.base.cra_driver_name);
			goto unreg;
		}
	}
	return 0;
unreg:
	for (i = 0; i < count; i++)
		crypto_unregister_ahash(&hash_algs[i].hash);
	return ret;
}

Project: linux

static void ahash_algs_unregister_all(struct hash_device_data *device_data)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(hash_algs); i++)
		crypto_unregister_ahash(&hash_algs[i].hash);
}

Project: linux

static int ahash_final(struct ahash_request *req)
{
	int ret = 0;
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);

	pr_debug("%s: data size: %d\n", __func__, req->nbytes);

	if ((hash_mode == HASH_MODE_DMA) && req_ctx->dma_mode)
		ret = hash_dma_final(req);
	else
		ret = hash_hw_final(req);

	if (ret) {
		pr_err("%s: hash_hw/dma_final() failed\n", __func__);
	}

	return ret;
}

Project: linux

static int ahash_sha1_digest(struct ahash_request *req)
{
	int ret2, ret1;

	ret1 = ahash_sha1_init(req);
	if (ret1)
		goto out;

	ret1 = ahash_update(req);
	ret2 = ahash_final(req);

out:
	return ret1 ? ret1 : ret2;
}

Project: linux

static int ahash_sha1_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);

	ctx->config.data_format = HASH_DATA_8_BITS;
	ctx->config.algorithm = HASH_ALGO_SHA1;
	ctx->config.oper_mode = HASH_OPER_MODE_HASH;
	ctx->digestsize = SHA1_DIGEST_SIZE;

	return hash_init(req);
}

Project: linux

static int ahash_sha256_digest(struct ahash_request *req)
{
	int ret2, ret1;

	ret1 = ahash_sha256_init(req);
	if (ret1)
		goto out;

	ret1 = ahash_update(req);
	ret2 = ahash_final(req);

out:
	return ret1 ? ret1 : ret2;
}

Project: linux

static int ahash_update(struct ahash_request *req)
{
	int ret = 0;
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);

	if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode)
		ret = hash_hw_update(req);
	/* Skip update for DMA, all data will be passed to DMA in final */

	if (ret) {
		pr_err("%s: hash_hw_update() failed!\n", __func__);
	}

	return ret;
}

Project: linux

static int get_empty_message_digest(
		struct hash_device_data *device_data,
		u8 *zero_hash, u32 *zero_hash_size, bool *zero_digest)
{
	int ret = 0;
	struct hash_ctx *ctx = device_data->current_ctx;
	*zero_digest = false;

	/**
	 * Caller responsible for ctx != NULL.
	 */

	if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) {
		if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
			memcpy(zero_hash, &zero_message_hash_sha1[0],
			       SHA1_DIGEST_SIZE);
			*zero_hash_size = SHA1_DIGEST_SIZE;
			*zero_digest = true;
		} else if (HASH_ALGO_SHA256 ==
				ctx->config.algorithm) {
			memcpy(zero_hash, &zero_message_hash_sha256[0],
			       SHA256_DIGEST_SIZE);
			*zero_hash_size = SHA256_DIGEST_SIZE;
			*zero_digest = true;
		} else {
			dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
				__func__);
			ret = -EINVAL;
			goto out;
		}
	} else if (HASH_OPER_MODE_HMAC == ctx->config.oper_mode) {
		if (!ctx->keylen) {
			if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
				memcpy(zero_hash, &zero_message_hmac_sha1[0],
				       SHA1_DIGEST_SIZE);
				*zero_hash_size = SHA1_DIGEST_SIZE;
				*zero_digest = true;
			} else if (HASH_ALGO_SHA256 == ctx->config.algorithm) {
				memcpy(zero_hash, &zero_message_hmac_sha256[0],
				       SHA256_DIGEST_SIZE);
				*zero_hash_size = SHA256_DIGEST_SIZE;
				*zero_digest = true;
			} else {
				dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
					__func__);
				ret = -EINVAL;
				goto out;
			}
		} else {
			dev_dbg(device_data->dev,
				"%s: Continue hash calculation, since hmac key available\n",
				__func__);
		}
	}
out:

	return ret;
}

Project: linux

void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx)
{
	/* HW and SW initializations */
	/* Note: there is no need to initialize buffer and digest members */

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();

	/*
	 * INIT bit. Set this bit to 0b1 to reset the HASH processor core and
	 * prepare the initialize the HASH accelerator to compute the message
	 * digest of a new message.
	 */
	HASH_INITIALIZE;

	/*
	 * NBLW bits. Reset the number of bits in last word (NBLW).
	 */
	HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);
}

Project: linux

int hash_check_hw(struct hash_device_data *device_data)
{
	/* Checking Peripheral Ids  */
	if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0) &&
	    HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1) &&
	    HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2) &&
	    HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3) &&
	    HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0) &&
	    HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1) &&
	    HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2) &&
	    HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)) {
		return 0;
	}

	dev_err(device_data->dev, "%s: HASH_UNSUPPORTED_HW!\n", __func__);
	return -ENOTSUPP;
}

Project: linux

static int hash_cra_init(struct crypto_tfm *tfm)
{
	struct hash_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_alg *alg = tfm->__crt_alg;
	struct hash_algo_template *hash_alg;

	hash_alg = container_of(__crypto_ahash_alg(alg),
			struct hash_algo_template,
			hash);

	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
				 sizeof(struct hash_req_ctx));

	ctx->config.data_format = HASH_DATA_8_BITS;
	ctx->config.algorithm = hash_alg->conf.algorithm;
	ctx->config.oper_mode = hash_alg->conf.oper_mode;

	ctx->digestsize = hash_alg->hash.halg.digestsize;

	return 0;
}

Project: linux

static int hash_disable_power(struct hash_device_data *device_data,
			      bool save_device_state)
{
	int ret = 0;
	struct device *dev = device_data->dev;

	spin_lock(&device_data->power_state_lock);
	if (!device_data->power_state)
		goto out;

	if (save_device_state) {
		hash_save_state(device_data,
				&device_data->state);
		device_data->restore_dev_state = true;
	}

	clk_disable(device_data->clk);
	ret = regulator_disable(device_data->regulator);
	if (ret)
		dev_err(dev, "%s: regulator_disable() failed!\n", __func__);

	device_data->power_state = false;

out:
	spin_unlock(&device_data->power_state_lock);

	return ret;
}

Project: linux

static void hash_dma_done(struct hash_ctx *ctx)
{
	struct dma_chan *chan;

	chan = ctx->device->dma.chan_mem2hash;
	dmaengine_device_control(chan, DMA_TERMINATE_ALL, 0);
	dma_unmap_sg(chan->device->dev, ctx->device->dma.sg,
		     ctx->device->dma.sg_len, DMA_TO_DEVICE);
}

Project: linux

static int hash_dma_final(struct ahash_request *req)
{
	int ret = 0;
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
	struct hash_device_data *device_data;
	u8 digest[SHA256_DIGEST_SIZE];
	int bytes_written = 0;

	ret = hash_get_device_data(ctx, &device_data);
	if (ret)
		return ret;

	dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);

	if (req_ctx->updated) {
		ret = hash_resume_state(device_data, &device_data->state);

		if (ret) {
			dev_err(device_data->dev, "%s: hash_resume_state() failed!\n",
				__func__);
			goto out;
		}
	}

	if (!req_ctx->updated) {
		ret = hash_setconfiguration(device_data, &ctx->config);
		if (ret) {
			dev_err(device_data->dev,
				"%s: hash_setconfiguration() failed!\n",
				__func__);
			goto out;
		}

		/* Enable DMA input */
		if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode) {
			HASH_CLEAR_BITS(&device_data->base->cr,
					HASH_CR_DMAE_MASK);
		} else {
			HASH_SET_BITS(&device_data->base->cr,
				      HASH_CR_DMAE_MASK);
			HASH_SET_BITS(&device_data->base->cr,
				      HASH_CR_PRIVN_MASK);
		}

		HASH_INITIALIZE;

		if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC)
			hash_hw_write_key(device_data, ctx->key, ctx->keylen);

		/* Number of bits in last word = (nbytes * 8) % 32 */
		HASH_SET_NBLW((req->nbytes * 8) % 32);
		req_ctx->updated = 1;
	}

	/* Store the nents in the dma struct. */
	ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL);
	if (!ctx->device->dma.nents) {
		dev_err(device_data->dev, "%s: ctx->device->dma.nents = 0\n",
			__func__);
		ret = ctx->device->dma.nents;
		goto out;
	}

	bytes_written = hash_dma_write(ctx, req->src, req->nbytes);
	if (bytes_written != req->nbytes) {
		dev_err(device_data->dev, "%s: hash_dma_write() failed!\n",
			__func__);
		ret = bytes_written;
		goto out;
	}

	wait_for_completion(&ctx->device->dma.complete);
	hash_dma_done(ctx);

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();

	if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) {
		unsigned int keylen = ctx->keylen;
		u8 *key = ctx->key;

		dev_dbg(device_data->dev, "%s: keylen: %d\n",
			__func__, ctx->keylen);
		hash_hw_write_key(device_data, key, keylen);
	}

	hash_get_digest(device_data, digest, ctx->config.algorithm);
	memcpy(req->result, digest, ctx->digestsize);

out:
	release_hash_device(device_data);

	/**
	 * Allocated in setkey, and only used in HMAC.
	 */
	kfree(ctx->key);

	return ret;
}

Project: linux

static void hash_dma_setup_channel(struct hash_device_data *device_data,
				   struct device *dev)
{
	struct hash_platform_data *platform_data = dev->platform_data;
	struct dma_slave_config conf = {
		.direction = DMA_MEM_TO_DEV,
		.dst_addr = device_data->phybase + HASH_DMA_FIFO,
		.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
		.dst_maxburst = 16,
	};

	dma_cap_zero(device_data->dma.mask);
	dma_cap_set(DMA_SLAVE, device_data->dma.mask);

	device_data->dma.cfg_mem2hash = platform_data->mem_to_engine;
	device_data->dma.chan_mem2hash =
		dma_request_channel(device_data->dma.mask,
				    platform_data->dma_filter,
				    device_data->dma.cfg_mem2hash);

	dmaengine_slave_config(device_data->dma.chan_mem2hash, &conf);

	init_completion(&device_data->dma.complete);
}

Project: linux

static bool hash_dma_valid_data(struct scatterlist *sg, int datasize)
{
	bool aligned;

	/* Need to include at least one nent, else error */
	if (hash_get_nents(sg, datasize, &aligned) < 1)
		return false;

	return aligned;
}

Project: linux

static int hash_dma_write(struct hash_ctx *ctx,
			  struct scatterlist *sg, int len)
{
	int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
	if (error) {
		dev_dbg(ctx->device->dev,
			"%s: hash_set_dma_transfer() failed\n", __func__);
		return error;
	}

	return len;
}

Project: linux

static int hash_enable_power(struct hash_device_data *device_data,
			     bool restore_device_state)
{
	int ret = 0;
	struct device *dev = device_data->dev;

	spin_lock(&device_data->power_state_lock);
	if (!device_data->power_state) {
		ret = regulator_enable(device_data->regulator);
		if (ret) {
			dev_err(dev, "%s: regulator_enable() failed!\n",
				__func__);
			goto out;
		}
		ret = clk_enable(device_data->clk);
		if (ret) {
			dev_err(dev, "%s: clk_enable() failed!\n", __func__);
			ret = regulator_disable(
					device_data->regulator);
			goto out;
		}
		device_data->power_state = true;
	}

	if (device_data->restore_dev_state) {
		if (restore_device_state) {
			device_data->restore_dev_state = false;
			hash_resume_state(device_data, &device_data->state);
		}
	}
out:
	spin_unlock(&device_data->power_state_lock);

	return ret;
}

Project: linux

static int hash_get_device_data(struct hash_ctx *ctx,
				struct hash_device_data **device_data)
{
	int			ret;
	struct klist_iter	device_iterator;
	struct klist_node	*device_node;
	struct hash_device_data *local_device_data = NULL;

	/* Wait until a device is available */
	ret = down_interruptible(&driver_data.device_allocation);
	if (ret)
		return ret;  /* Interrupted */

	/* Select a device */
	klist_iter_init(&driver_data.device_list, &device_iterator);
	device_node = klist_next(&device_iterator);
	while (device_node) {
		local_device_data = container_of(device_node,
					   struct hash_device_data, list_node);
		spin_lock(&local_device_data->ctx_lock);
		/* current_ctx allocates a device, NULL = unallocated */
		if (local_device_data->current_ctx) {
			device_node = klist_next(&device_iterator);
		} else {
			local_device_data->current_ctx = ctx;
			ctx->device = local_device_data;
			spin_unlock(&local_device_data->ctx_lock);
			break;
		}
		spin_unlock(&local_device_data->ctx_lock);
	}
	klist_iter_exit(&device_iterator);

	if (!device_node) {
		/**
		 * No free device found.
		 * Since we allocated a device with down_interruptible, this
		 * should not be able to happen.
		 * Number of available devices, which are contained in
		 * device_allocation, is therefore decremented by not doing
		 * an up(device_allocation).
		 */
		return -EBUSY;
	}

	*device_data = local_device_data;

	return 0;
}

Project: linux

static int hash_hw_final(struct ahash_request *req)
{
	int ret = 0;
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
	struct hash_device_data *device_data;
	u8 digest[SHA256_DIGEST_SIZE];

	ret = hash_get_device_data(ctx, &device_data);
	if (ret)
		return ret;

	dev_dbg(device_data->dev, "%s: (ctx=0x%x)!\n", __func__, (u32) ctx);

	if (req_ctx->updated) {
		ret = hash_resume_state(device_data, &device_data->state);

		if (ret) {
			dev_err(device_data->dev,
				"%s: hash_resume_state() failed!\n", __func__);
			goto out;
		}
	} else if (req->nbytes == 0 && ctx->keylen == 0) {
		u8 zero_hash[SHA256_DIGEST_SIZE];
		u32 zero_hash_size = 0;
		bool zero_digest = false;
		/**
		 * Use a pre-calculated empty message digest
		 * (workaround since hw return zeroes, hw bug!?)
		 */
		ret = get_empty_message_digest(device_data, &zero_hash[0],
				&zero_hash_size, &zero_digest);
		if (!ret && likely(zero_hash_size == ctx->digestsize) &&
		    zero_digest) {
			memcpy(req->result, &zero_hash[0], ctx->digestsize);
			goto out;
		} else if (!ret && !zero_digest) {
			dev_dbg(device_data->dev,
				"%s: HMAC zero msg with key, continue...\n",
				__func__);
		} else {
			dev_err(device_data->dev,
				"%s: ret=%d, or wrong digest size? %s\n",
				__func__, ret,
				zero_hash_size == ctx->digestsize ?
				"true" : "false");
			/* Return error */
			goto out;
		}
	} else if (req->nbytes == 0 && ctx->keylen > 0) {
		dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n",
			__func__);
		goto out;
	}

	if (!req_ctx->updated) {
		ret = init_hash_hw(device_data, ctx);
		if (ret) {
			dev_err(device_data->dev,
				"%s: init_hash_hw() failed!\n", __func__);
			goto out;
		}
	}

	if (req_ctx->state.index) {
		hash_messagepad(device_data, req_ctx->state.buffer,
				req_ctx->state.index);
	} else {
		HASH_SET_DCAL;
		while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
			cpu_relax();
	}

	if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) {
		unsigned int keylen = ctx->keylen;
		u8 *key = ctx->key;

		dev_dbg(device_data->dev, "%s: keylen: %d\n",
			__func__, ctx->keylen);
		hash_hw_write_key(device_data, key, keylen);
	}

	hash_get_digest(device_data, digest, ctx->config.algorithm);
	memcpy(req->result, digest, ctx->digestsize);

out:
	release_hash_device(device_data);

	/**
	 * Allocated in setkey, and only used in HMAC.
	 */
	kfree(ctx->key);

	return ret;
}

Project: linux

int hash_hw_update(struct ahash_request *req)
{
	int ret = 0;
	u8 index = 0;
	u8 *buffer;
	struct hash_device_data *device_data;
	u8 *data_buffer;
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
	struct crypto_hash_walk walk;
	int msg_length = crypto_hash_walk_first(req, &walk);

	/* Empty message ("") is correct indata */
	if (msg_length == 0)
		return ret;

	index = req_ctx->state.index;
	buffer = (u8 *)req_ctx->state.buffer;

	/* Check if ctx->state.length + msg_length
	   overflows */
	if (msg_length > (req_ctx->state.length.low_word + msg_length) &&
	    HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) {
		pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__);
		return -EPERM;
	}

	ret = hash_get_device_data(ctx, &device_data);
	if (ret)
		return ret;

	/* Main loop */
	while (0 != msg_length) {
		data_buffer = walk.data;
		ret = hash_process_data(device_data, ctx, req_ctx, msg_length,
				data_buffer, buffer, &index);

		if (ret) {
			dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n",
				__func__);
			goto out;
		}

		msg_length = crypto_hash_walk_done(&walk, 0);
	}

	req_ctx->state.index = index;
	dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n",
		__func__, req_ctx->state.index, req_ctx->state.bit_index);

out:
	release_hash_device(device_data);

	return ret;
}

Project: linux

static void hash_hw_write_key(struct hash_device_data *device_data,
			      const u8 *key, unsigned int keylen)
{
	u32 word = 0;
	int nwords = 1;

	HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);

	while (keylen >= 4) {
		u32 *key_word = (u32 *)key;

		HASH_SET_DIN(key_word, nwords);
		keylen -= 4;
		key += 4;
	}

	/* Take care of the remaining bytes in the last word */
	if (keylen) {
		word = 0;
		while (keylen) {
			word |= (key[keylen - 1] << (8 * (keylen - 1)));
			keylen--;
		}

		HASH_SET_DIN(&word, nwords);
	}

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();

	HASH_SET_DCAL;

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();
}

Project: linux

static void hash_incrementlength(struct hash_req_ctx *ctx, u32 incr)
{
	ctx->state.length.low_word += incr;

	/* Check for wrap-around */
	if (ctx->state.length.low_word < incr)
		ctx->state.length.high_word++;
}

Project: linux

static int hash_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
	struct hash_req_ctx *req_ctx = ahash_request_ctx(req);

	if (!ctx->key)
		ctx->keylen = 0;

	memset(&req_ctx->state, 0, sizeof(struct hash_state));
	req_ctx->updated = 0;
	if (hash_mode == HASH_MODE_DMA) {
		if (req->nbytes < HASH_DMA_ALIGN_SIZE) {
			req_ctx->dma_mode = false; /* Don't use DMA */

			pr_debug("%s: DMA mode, but direct to CPU mode for data size < %d\n",
				 __func__, HASH_DMA_ALIGN_SIZE);
		} else {
			if (req->nbytes >= HASH_DMA_PERFORMANCE_MIN_SIZE &&
			    hash_dma_valid_data(req->src, req->nbytes)) {
				req_ctx->dma_mode = true;
			} else {
				req_ctx->dma_mode = false;
				pr_debug("%s: DMA mode, but use CPU mode for datalength < %d or non-aligned data, except in last nent\n",
					 __func__,
					 HASH_DMA_PERFORMANCE_MIN_SIZE);
			}
		}
	}
	return 0;
}

Project: linux

static void hash_messagepad(struct hash_device_data *device_data,
			    const u32 *message, u8 index_bytes)
{
	int nwords = 1;

	/*
	 * Clear hash str register, only clear NBLW
	 * since DCAL will be reset by hardware.
	 */
	HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);

	/* Main loop */
	while (index_bytes >= 4) {
		HASH_SET_DIN(message, nwords);
		index_bytes -= 4;
		message++;
	}

	if (index_bytes)
		HASH_SET_DIN(message, nwords);

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();

	/* num_of_bytes == 0 => NBLW <- 0 (32 bits valid in DATAIN) */
	HASH_SET_NBLW(index_bytes * 8);
	dev_dbg(device_data->dev, "%s: DIN=0x%08x NBLW=%lu\n",
		__func__, readl_relaxed(&device_data->base->din),
		readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
	HASH_SET_DCAL;
	dev_dbg(device_data->dev, "%s: after dcal -> DIN=0x%08x NBLW=%lu\n",
		__func__, readl_relaxed(&device_data->base->din),
		readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);

	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();
}

Project: linux

static int hash_process_data(struct hash_device_data *device_data,
			     struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
			     int msg_length, u8 *data_buffer, u8 *buffer,
			     u8 *index)
{
	int ret = 0;
	u32 count;

	do {
		if ((*index + msg_length) < HASH_BLOCK_SIZE) {
			for (count = 0; count < msg_length; count++) {
				buffer[*index + count] =
					*(data_buffer + count);
			}
			*index += msg_length;
			msg_length = 0;
		} else {
			if (req_ctx->updated) {
				ret = hash_resume_state(device_data,
						&device_data->state);
				memmove(req_ctx->state.buffer,
					device_data->state.buffer,
					HASH_BLOCK_SIZE / sizeof(u32));
				if (ret) {
					dev_err(device_data->dev,
						"%s: hash_resume_state() failed!\n",
						__func__);
					goto out;
				}
			} else {
				ret = init_hash_hw(device_data, ctx);
				if (ret) {
					dev_err(device_data->dev,
						"%s: init_hash_hw() failed!\n",
						__func__);
					goto out;
				}
				req_ctx->updated = 1;
			}
			/*
			 * If 'data_buffer' is four byte aligned and
			 * local buffer does not have any data, we can
			 * write data directly from 'data_buffer' to
			 * HW peripheral, otherwise we first copy data
			 * to a local buffer
			 */
			if ((0 == (((u32)data_buffer) % 4)) &&
			    (0 == *index))
				hash_processblock(device_data,
						  (const u32 *)data_buffer,
						  HASH_BLOCK_SIZE);
			else {
				for (count = 0;
				     count < (u32)(HASH_BLOCK_SIZE - *index);
				     count++) {
					buffer[*index + count] =
						*(data_buffer + count);
				}
				hash_processblock(device_data,
						  (const u32 *)buffer,
						  HASH_BLOCK_SIZE);
			}
			hash_incrementlength(req_ctx, HASH_BLOCK_SIZE);
			data_buffer += (HASH_BLOCK_SIZE - *index);

			msg_length -= (HASH_BLOCK_SIZE - *index);
			*index = 0;

			ret = hash_save_state(device_data,
					&device_data->state);

			memmove(device_data->state.buffer,
				req_ctx->state.buffer,
				HASH_BLOCK_SIZE / sizeof(u32));
			if (ret) {
				dev_err(device_data->dev, "%s: hash_save_state() failed!\n",
					__func__);
				goto out;
			}
		}
	} while (msg_length != 0);
out:

	return ret;
}

Project: linux

static void hash_processblock(struct hash_device_data *device_data,
			      const u32 *message, int length)
{
	int len = length / HASH_BYTES_PER_WORD;
	/*
	 * NBLW bits. Reset the number of bits in last word (NBLW).
	 */
	HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);

	/*
	 * Write message data to the HASH_DIN register.
	 */
	HASH_SET_DIN(message, len);
}

Project: linux

int hash_resume_state(struct hash_device_data *device_data,
		      const struct hash_state *device_state)
{
	u32 temp_cr;
	s32 count;
	int hash_mode = HASH_OPER_MODE_HASH;

	if (NULL == device_state) {
		dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
			__func__);
		return -EPERM;
	}

	/* Check correctness of index and length members */
	if (device_state->index > HASH_BLOCK_SIZE ||
	    (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
		dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
			__func__);
		return -EPERM;
	}

	/*
	 * INIT bit. Set this bit to 0b1 to reset the HASH processor core and
	 * prepare the initialize the HASH accelerator to compute the message
	 * digest of a new message.
	 */
	HASH_INITIALIZE;

	temp_cr = device_state->temp_cr;
	writel_relaxed(temp_cr & HASH_CR_RESUME_MASK, &device_data->base->cr);

	if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK)
		hash_mode = HASH_OPER_MODE_HMAC;
	else
		hash_mode = HASH_OPER_MODE_HASH;

	for (count = 0; count < HASH_CSR_COUNT; count++) {
		if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH))
			break;

		writel_relaxed(device_state->csr[count],
			       &device_data->base->csrx[count]);
	}

	writel_relaxed(device_state->csfull, &device_data->base->csfull);
	writel_relaxed(device_state->csdatain, &device_data->base->csdatain);

	writel_relaxed(device_state->str_reg, &device_data->base->str);
	writel_relaxed(temp_cr, &device_data->base->cr);

	return 0;
}

Project: linux

int hash_save_state(struct hash_device_data *device_data,
		    struct hash_state *device_state)
{
	u32 temp_cr;
	u32 count;
	int hash_mode = HASH_OPER_MODE_HASH;

	if (NULL == device_state) {
		dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
			__func__);
		return -ENOTSUPP;
	}

	/* Write dummy value to force digest intermediate calculation. This
	 * actually makes sure that there isn't any ongoing calculation in the
	 * hardware.
	 */
	while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
		cpu_relax();

	temp_cr = readl_relaxed(&device_data->base->cr);

	device_state->str_reg = readl_relaxed(&device_data->base->str);

	device_state->din_reg = readl_relaxed(&device_data->base->din);

	if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK)
		hash_mode = HASH_OPER_MODE_HMAC;
	else
		hash_mode = HASH_OPER_MODE_HASH;

	for (count = 0; count < HASH_CSR_COUNT; count++) {
		if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH))
			break;

		device_state->csr[count] =
			readl_relaxed(&device_data->base->csrx[count]);
	}

	device_state->csfull = readl_relaxed(&device_data->base->csfull);
	device_state->csdatain = readl_relaxed(&device_data->base->csdatain);

	device_state->temp_cr = temp_cr;

	return 0;
}

Project: linux

static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
				 int len, enum dma_data_direction direction)
{
	struct dma_async_tx_descriptor *desc = NULL;
	struct dma_chan *channel = NULL;
	dma_cookie_t cookie;

	if (direction != DMA_TO_DEVICE) {
		dev_err(ctx->device->dev, "%s: Invalid DMA direction\n",
			__func__);
		return -EFAULT;
	}

	sg->length = ALIGN(sg->length, HASH_DMA_ALIGN_SIZE);

	channel = ctx->device->dma.chan_mem2hash;
	ctx->device->dma.sg = sg;
	ctx->device->dma.sg_len = dma_map_sg(channel->device->dev,
			ctx->device->dma.sg, ctx->device->dma.nents,
			direction);

	if (!ctx->device->dma.sg_len) {
		dev_err(ctx->device->dev, "%s: Could not map the sg list (TO_DEVICE)\n",
			__func__);
		return -EFAULT;
	}

	dev_dbg(ctx->device->dev, "%s: Setting up DMA for buffer (TO_DEVICE)\n",
		__func__);
	desc = dmaengine_prep_slave_sg(channel,
			ctx->device->dma.sg, ctx->device->dma.sg_len,
			direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
	if (!desc) {
		dev_err(ctx->device->dev,
			"%s: device_prep_slave_sg() failed!\n", __func__);
		return -EFAULT;
	}

	desc->callback = hash_dma_callback;
	desc->callback_param = ctx;

	cookie = dmaengine_submit(desc);
	dma_async_issue_pending(channel);

	return 0;
}

Project: linux

int hash_setconfiguration(struct hash_device_data *device_data,
			  struct hash_config *config)
{
	int ret = 0;

	if (config->algorithm != HASH_ALGO_SHA1 &&
	    config->algorithm != HASH_ALGO_SHA256)
		return -EPERM;

	/*
	 * DATAFORM bits. Set the DATAFORM bits to 0b11, which means the data
	 * to be written to HASH_DIN is considered as 32 bits.
	 */
	HASH_SET_DATA_FORMAT(config->data_format);

	/*
	 * ALGO bit. Set to 0b1 for SHA-1 and 0b0 for SHA-256
	 */
	switch (config->algorithm) {
	case HASH_ALGO_SHA1:
		HASH_SET_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK);
		break;

	case HASH_ALGO_SHA256:
		HASH_CLEAR_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK);
		break;

	default:
		dev_err(device_data->dev, "%s: Incorrect algorithm\n",
			__func__);
		return -EPERM;
	}

	/*
	 * MODE bit. This bit selects between HASH or HMAC mode for the
	 * selected algorithm. 0b0 = HASH and 0b1 = HMAC.
	 */
	if (HASH_OPER_MODE_HASH == config->oper_mode)
		HASH_CLEAR_BITS(&device_data->base->cr,
				HASH_CR_MODE_MASK);
	else if (HASH_OPER_MODE_HMAC == config->oper_mode) {
		HASH_SET_BITS(&device_data->base->cr, HASH_CR_MODE_MASK);
		if (device_data->current_ctx->keylen > HASH_BLOCK_SIZE) {
			/* Truncate key to blocksize */
			dev_dbg(device_data->dev, "%s: LKEY set\n", __func__);
			HASH_SET_BITS(&device_data->base->cr,
				      HASH_CR_LKEY_MASK);
		} else {
			dev_dbg(device_data->dev, "%s: LKEY cleared\n",
				__func__);
			HASH_CLEAR_BITS(&device_data->base->cr,
					HASH_CR_LKEY_MASK);
		}
	} else {	/* Wrong hash mode */
		ret = -EPERM;
		dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
			__func__);
	}
	return ret;
}

Project: linux

static int hash_setkey(struct crypto_ahash *tfm,
		       const u8 *key, unsigned int keylen, int alg)
{
	int ret = 0;
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);

	/**
	 * Freed in final.
	 */
	ctx->key = kmemdup(key, keylen, GFP_KERNEL);
	if (!ctx->key) {
		pr_err("%s: Failed to allocate ctx->key for %d\n",
		       __func__, alg);
		return -ENOMEM;
	}
	ctx->keylen = keylen;

	return ret;
}

Project: linux

static int hmac_sha1_digest(struct ahash_request *req)
{
	int ret2, ret1;

	ret1 = hmac_sha1_init(req);
	if (ret1)
		goto out;

	ret1 = ahash_update(req);
	ret2 = ahash_final(req);

out:
	return ret1 ? ret1 : ret2;
}

Project: linux

static int hmac_sha1_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);

	ctx->config.data_format	= HASH_DATA_8_BITS;
	ctx->config.algorithm	= HASH_ALGO_SHA1;
	ctx->config.oper_mode	= HASH_OPER_MODE_HMAC;
	ctx->digestsize		= SHA1_DIGEST_SIZE;

	return hash_init(req);
}

Project: linux

static int hmac_sha1_setkey(struct crypto_ahash *tfm,
			    const u8 *key, unsigned int keylen)
{
	return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1);
}

Project: linux

static int hmac_sha256_digest(struct ahash_request *req)
{
	int ret2, ret1;

	ret1 = hmac_sha256_init(req);
	if (ret1)
		goto out;

	ret1 = ahash_update(req);
	ret2 = ahash_final(req);

out:
	return ret1 ? ret1 : ret2;
}

Project: linux

static int hmac_sha256_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct hash_ctx *ctx = crypto_ahash_ctx(tfm);

	ctx->config.data_format	= HASH_DATA_8_BITS;
	ctx->config.algorithm	= HASH_ALGO_SHA256;
	ctx->config.oper_mode	= HASH_OPER_MODE_HMAC;
	ctx->digestsize		= SHA256_DIGEST_SIZE;

	return hash_init(req);
}

Project: linux

static int hmac_sha256_setkey(struct crypto_ahash *tfm,
			      const u8 *key, unsigned int keylen)
{
	return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256);
}

Project: linux

static int init_hash_hw(struct hash_device_data *device_data,
			struct hash_ctx *ctx)
{
	int ret = 0;

	ret = hash_setconfiguration(device_data, &ctx->config);
	if (ret) {
		dev_err(device_data->dev, "%s: hash_setconfiguration() failed!\n",
			__func__);
		return ret;
	}

	hash_begin(device_data, ctx);

	if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC)
		hash_hw_write_key(device_data, ctx->key, ctx->keylen);

	return ret;
}

Project: linux

static void release_hash_device(struct hash_device_data *device_data)
{
	spin_lock(&device_data->ctx_lock);
	device_data->current_ctx->device = NULL;
	device_data->current_ctx = NULL;
	spin_unlock(&device_data->ctx_lock);

	/*
	 * The down_interruptible part for this semaphore is called in
	 * cryp_get_device_data.
	 */
	up(&driver_data.device_allocation);
}

Project: linux

static void __exit ux500_hash_mod_fini(void)
{
	platform_driver_unregister(&hash_driver);
}

Project: linux

static int __init ux500_hash_mod_init(void)
{
	klist_init(&driver_data.device_list, NULL, NULL);
	/* Initialize the semaphore to 0 devices (locked state) */
	sema_init(&driver_data.device_allocation, 0);

	return platform_driver_register(&hash_driver);
}

Project: linux

static int ux500_hash_probe(struct platform_device *pdev)
{
	int			ret = 0;
	struct resource		*res = NULL;
	struct hash_device_data *device_data;
	struct device		*dev = &pdev->dev;

	device_data = kzalloc(sizeof(*device_data), GFP_ATOMIC);
	if (!device_data) {
		ret = -ENOMEM;
		goto out;
	}

	device_data->dev = dev;
	device_data->current_ctx = NULL;

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!res) {
		dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__);
		ret = -ENODEV;
		goto out_kfree;
	}

	res = request_mem_region(res->start, resource_size(res), pdev->name);
	if (res == NULL) {
		dev_dbg(dev, "%s: request_mem_region() failed!\n", __func__);
		ret = -EBUSY;
		goto out_kfree;
	}

	device_data->phybase = res->start;
	device_data->base = ioremap(res->start, resource_size(res));
	if (!device_data->base) {
		dev_err(dev, "%s: ioremap() failed!\n", __func__);
		ret = -ENOMEM;
		goto out_free_mem;
	}
	spin_lock_init(&device_data->ctx_lock);
	spin_lock_init(&device_data->power_state_lock);

	/* Enable power for HASH1 hardware block */
	device_data->regulator = regulator_get(dev, "v-ape");
	if (IS_ERR(device_data->regulator)) {
		dev_err(dev, "%s: regulator_get() failed!\n", __func__);
		ret = PTR_ERR(device_data->regulator);
		device_data->regulator = NULL;
		goto out_unmap;
	}

	/* Enable the clock for HASH1 hardware block */
	device_data->clk = clk_get(dev, NULL);
	if (IS_ERR(device_data->clk)) {
		dev_err(dev, "%s: clk_get() failed!\n", __func__);
		ret = PTR_ERR(device_data->clk);
		goto out_regulator;
	}

	ret = clk_prepare(device_data->clk);
	if (ret) {
		dev_err(dev, "%s: clk_prepare() failed!\n", __func__);
		goto out_clk;
	}

	/* Enable device power (and clock) */
	ret = hash_enable_power(device_data, false);
	if (ret) {
		dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
		goto out_clk_unprepare;
	}

	ret = hash_check_hw(device_data);
	if (ret) {
		dev_err(dev, "%s: hash_check_hw() failed!\n", __func__);
		goto out_power;
	}

	if (hash_mode == HASH_MODE_DMA)
		hash_dma_setup_channel(device_data, dev);

	platform_set_drvdata(pdev, device_data);

	/* Put the new device into the device list... */
	klist_add_tail(&device_data->list_node, &driver_data.device_list);
	/* ... and signal that a new device is available. */
	up(&driver_data.device_allocation);

	ret = ahash_algs_register_all(device_data);
	if (ret) {
		dev_err(dev, "%s: ahash_algs_register_all() failed!\n",
			__func__);
		goto out_power;
	}

	dev_info(dev, "successfully registered\n");
	return 0;

out_power:
	hash_disable_power(device_data, false);

out_clk_unprepare:
	clk_unprepare(device_data->clk);

out_clk:
	clk_put(device_data->clk);

out_regulator:
	regulator_put(device_data->regulator);

out_unmap:
	iounmap(device_data->base);

out_free_mem:
	release_mem_region(res->start, resource_size(res));

out_kfree:
	kfree(device_data);
out:
	return ret;
}

Project: linux

static int ux500_hash_remove(struct platform_device *pdev)
{
	struct resource		*res;
	struct hash_device_data *device_data;
	struct device		*dev = &pdev->dev;

	device_data = platform_get_drvdata(pdev);
	if (!device_data) {
		dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
		return -ENOMEM;
	}

	/* Try to decrease the number of available devices. */
	if (down_trylock(&driver_data.device_allocation))
		return -EBUSY;

	/* Check that the device is free */
	spin_lock(&device_data->ctx_lock);
	/* current_ctx allocates a device, NULL = unallocated */
	if (device_data->current_ctx) {
		/* The device is busy */
		spin_unlock(&device_data->ctx_lock);
		/* Return the device to the pool. */
		up(&driver_data.device_allocation);
		return -EBUSY;
	}

	spin_unlock(&device_data->ctx_lock);

	/* Remove the device from the list */
	if (klist_node_attached(&device_data->list_node))
		klist_remove(&device_data->list_node);

	/* If this was the last device, remove the services */
	if (list_empty(&driver_data.device_list.k_list))
		ahash_algs_unregister_all(device_data);

	if (hash_disable_power(device_data, false))
		dev_err(dev, "%s: hash_disable_power() failed\n",
			__func__);

	clk_unprepare(device_data->clk);
	clk_put(device_data->clk);
	regulator_put(device_data->regulator);

	iounmap(device_data->base);

	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (res)
		release_mem_region(res->start, resource_size(res));

	kfree(device_data);

	return 0;
}

Project: linux

static int ux500_hash_resume(struct device *dev)
{
	int ret = 0;
	struct hash_device_data *device_data;
	struct hash_ctx *temp_ctx = NULL;

	device_data = dev_get_drvdata(dev);
	if (!device_data) {
		dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
		return -ENOMEM;
	}

	spin_lock(&device_data->ctx_lock);
	if (device_data->current_ctx == ++temp_ctx)
		device_data->current_ctx = NULL;
	spin_unlock(&device_data->ctx_lock);

	if (!device_data->current_ctx)
		up(&driver_data.device_allocation);
	else
		ret = hash_enable_power(device_data, true);

	if (ret)
		dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);

	return ret;
}

Project: linux

static int ux500_hash_suspend(struct device *dev)
{
	int ret;
	struct hash_device_data *device_data;
	struct hash_ctx *temp_ctx = NULL;

	device_data = dev_get_drvdata(dev);
	if (!device_data) {
		dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
		return -ENOMEM;
	}

	spin_lock(&device_data->ctx_lock);
	if (!device_data->current_ctx)
		device_data->current_ctx++;
	spin_unlock(&device_data->ctx_lock);

	if (device_data->current_ctx == ++temp_ctx) {
		if (down_interruptible(&driver_data.device_allocation))
			dev_dbg(dev, "%s: down_interruptible() failed\n",
				__func__);
		ret = hash_disable_power(device_data, false);

	} else {
		ret = hash_disable_power(device_data, true);
	}

	if (ret)
		dev_err(dev, "%s: hash_disable_power()\n", __func__);

	return ret;
}

Project: linux

static void __ap_flush_queue(struct ap_device *ap_dev)
{
	struct ap_message *ap_msg, *next;

	list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
		list_del_init(&ap_msg->list);
		ap_dev->pendingq_count--;
		ap_msg->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
	}
	list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
		list_del_init(&ap_msg->list);
		ap_dev->requestq_count--;
		ap_msg->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
	}
}

Project: linux

static int __ap_match_all(struct device *dev, void *data)
{
	return 1;
}

Project: linux

static int __ap_poll_device(struct ap_device *ap_dev, unsigned long *flags)
{
	if (!ap_dev->unregistered) {
		if (ap_poll_queue(ap_dev, flags))
			ap_dev->unregistered = 1;
		if (ap_dev->reset == AP_RESET_DO)
			ap_reset(ap_dev);
	}
	return 0;
}

Project: linux

static inline int __ap_query_configuration(struct ap_config_info *config)
{
	register unsigned long reg0 asm ("0") = 0x04000000UL;
	register unsigned long reg1 asm ("1") = -EINVAL;
	register unsigned char *reg2 asm ("2") = (unsigned char *)config;

	asm volatile(
		".long 0xb2af0000\n"		/* PQAP(QCI) */
		"0: la    %1,0\n"
		"1:\n"
		EX_TABLE(0b, 1b)
		: "+d" (reg0), "+d" (reg1), "+d" (reg2)
		:
		: "cc");

	return reg1;
}

Project: linux

__ap_query_functions(ap_qid_t qid, unsigned int *functions)
{
	register unsigned long reg0 asm ("0") = 0UL | qid | (1UL << 23);
	register struct ap_queue_status reg1 asm ("1") = AP_QUEUE_STATUS_INVALID;
	register unsigned long reg2 asm ("2");

	asm volatile(
		".long 0xb2af0000\n"		/* PQAP(TAPQ) */
		"0:\n"
		EX_TABLE(0b, 0b)
		: "+d" (reg0), "+d" (reg1), "=d" (reg2)
		:
		: "cc");

	*functions = (unsigned int)(reg2 >> 32);
	return reg1;
}

Project: linux

static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
{
	struct ap_queue_status status;

	if (list_empty(&ap_dev->requestq) &&
	    ap_dev->queue_count < ap_dev->queue_depth) {
		status = __ap_send(ap_dev->qid, ap_msg->psmid,
				   ap_msg->message, ap_msg->length,
				   ap_msg->special);
		switch (status.response_code) {
		case AP_RESPONSE_NORMAL:
			list_add_tail(&ap_msg->list, &ap_dev->pendingq);
			atomic_inc(&ap_poll_requests);
			ap_dev->pendingq_count++;
			ap_increase_queue_count(ap_dev);
			ap_dev->total_request_count++;
			break;
		case AP_RESPONSE_Q_FULL:
		case AP_RESPONSE_RESET_IN_PROGRESS:
			list_add_tail(&ap_msg->list, &ap_dev->requestq);
			ap_dev->requestq_count++;
			ap_dev->total_request_count++;
			return -EBUSY;
		case AP_RESPONSE_REQ_FAC_NOT_INST:
		case AP_RESPONSE_MESSAGE_TOO_BIG:
			ap_msg->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
			return -EINVAL;
		default:	/* Device is gone. */
			ap_msg->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
			return -ENODEV;
		}
	} else {
		list_add_tail(&ap_msg->list, &ap_dev->requestq);
		ap_dev->requestq_count++;
		ap_dev->total_request_count++;
		return -EBUSY;
	}
	ap_schedule_poll_timer();
	return 0;
}

Project: linux

__ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
{
	typedef struct { char _[length]; } msgblock;
	register unsigned long reg0 asm("0") = qid | 0x80000000UL;
	register struct ap_queue_status reg1 asm ("1");
	register unsigned long reg2 asm("2") = 0UL;
	register unsigned long reg4 asm("4") = (unsigned long) msg;
	register unsigned long reg5 asm("5") = (unsigned long) length;
	register unsigned long reg6 asm("6") = 0UL;
	register unsigned long reg7 asm("7") = 0UL;


	asm volatile(
		"0: .long 0xb2ae0064\n"		/* DQAP */
		"   brc   6,0b\n"
		: "+d" (reg0), "=d" (reg1), "+d" (reg2),
		"+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
		"=m" (*(msgblock *) msg) : : "cc" );
	*psmid = (((unsigned long long) reg6) << 32) + reg7;
	return reg1;
}

Project: linux

static int __ap_scan_bus(struct device *dev, void *data)
{
	return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
}

Project: linux

void ap_bus_force_rescan(void)
{
	/* reconfigure the AP bus rescan timer. */
	mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ);
	/* processing a asynchronous bus rescan */
	queue_work(ap_work_queue, &ap_config_work);
	flush_work(&ap_config_work);
}

Project: linux

static int ap_bus_match(struct device *dev, struct device_driver *drv)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	struct ap_driver *ap_drv = to_ap_drv(drv);
	struct ap_device_id *id;

	/*
	 * Compare device type of the device with the list of
	 * supported types of the device_driver.
	 */
	for (id = ap_drv->ids; id->match_flags; id++) {
		if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
		    (id->dev_type != ap_dev->device_type))
			continue;
		return 1;
	}
	return 0;
}

Project: linux

static int ap_bus_resume(struct device *dev)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	int rc;

	if (ap_suspend_flag) {
		ap_suspend_flag = 0;
		if (ap_interrupts_available()) {
			if (!ap_using_interrupts()) {
				rc = register_adapter_interrupt(&ap_airq);
				ap_airq_flag = (rc == 0);
			}
		} else {
			if (ap_using_interrupts()) {
				unregister_adapter_interrupt(&ap_airq);
				ap_airq_flag = 0;
			}
		}
		ap_query_configuration();
		if (!user_set_domain) {
			ap_domain_index = -1;
			ap_select_domain();
		}
		init_timer(&ap_config_timer);
		ap_config_timer.function = ap_config_timeout;
		ap_config_timer.data = 0;
		ap_config_timer.expires = jiffies + ap_config_time * HZ;
		add_timer(&ap_config_timer);
		ap_work_queue = create_singlethread_workqueue("kapwork");
		if (!ap_work_queue)
			return -ENOMEM;
		tasklet_enable(&ap_tasklet);
		if (!ap_using_interrupts())
			ap_schedule_poll_timer();
		else
			tasklet_schedule(&ap_tasklet);
		if (ap_thread_flag)
			rc = ap_poll_thread_start();
		else
			rc = 0;
	} else
		rc = 0;
	if (AP_QID_QUEUE(ap_dev->qid) != ap_domain_index) {
		spin_lock_bh(&ap_dev->lock);
		ap_dev->qid = AP_MKQID(AP_QID_DEVICE(ap_dev->qid),
				       ap_domain_index);
		spin_unlock_bh(&ap_dev->lock);
	}
	queue_work(ap_work_queue, &ap_config_work);

	return rc;
}

Project: linux

void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
{
	struct ap_message *tmp;

	spin_lock_bh(&ap_dev->lock);
	if (!list_empty(&ap_msg->list)) {
		list_for_each_entry(tmp, &ap_dev->pendingq, list)
			if (tmp->psmid == ap_msg->psmid) {
				ap_dev->pendingq_count--;
				goto found;
			}
		ap_dev->requestq_count--;
	found:
		list_del_init(&ap_msg->list);
	}
	spin_unlock_bh(&ap_dev->lock);
}

Project: linux

static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
{
	return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
}

Project: linux

static ssize_t ap_config_time_store(struct bus_type *bus,
				    const char *buf, size_t count)
{
	int time;

	if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
		return -EINVAL;
	ap_config_time = time;
	if (!timer_pending(&ap_config_timer) ||
	    !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
		ap_config_timer.expires = jiffies + ap_config_time * HZ;
		add_timer(&ap_config_timer);
	}
	return count;
}

Project: linux

ap_config_timeout(unsigned long ptr)
{
	queue_work(ap_work_queue, &ap_config_work);
	ap_config_timer.expires = jiffies + ap_config_time * HZ;
	add_timer(&ap_config_timer);
}

Project: linux

static int ap_configuration_available(void)
{
	return test_facility(2) && test_facility(12);
}

Project: linux

static ssize_t ap_control_domain_mask_show(struct bus_type *bus, char *buf)
{
	if (ap_configuration != NULL) { /* QCI not supported */
		if (test_facility(76)) { /* format 1 - 256 bit domain field */
			return snprintf(buf, PAGE_SIZE,
				"0x%08x%08x%08x%08x%08x%08x%08x%08x\n",
			ap_configuration->adm[0], ap_configuration->adm[1],
			ap_configuration->adm[2], ap_configuration->adm[3],
			ap_configuration->adm[4], ap_configuration->adm[5],
			ap_configuration->adm[6], ap_configuration->adm[7]);
		} else { /* format 0 - 16 bit domain field */
			return snprintf(buf, PAGE_SIZE, "%08x%08x\n",
			ap_configuration->adm[0], ap_configuration->adm[1]);
		  }
	} else {
		return snprintf(buf, PAGE_SIZE, "not supported\n");
	  }
}

Project: linux

static void ap_decrease_queue_count(struct ap_device *ap_dev)
{
	int timeout = ap_dev->drv->request_timeout;

	ap_dev->queue_count--;
	if (ap_dev->queue_count > 0)
		mod_timer(&ap_dev->timeout, jiffies + timeout);
	else
		/*
		 * The timeout timer should to be disabled now - since
		 * del_timer_sync() is very expensive, we just tell via the
		 * reset flag to ignore the pending timeout timer.
		 */
		ap_dev->reset = AP_RESET_IGNORE;
}

Project: linux

static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
}

Project: linux

static int ap_device_probe(struct device *dev)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
	int rc;

	ap_dev->drv = ap_drv;

	spin_lock_bh(&ap_device_list_lock);
	list_add(&ap_dev->list, &ap_device_list);
	spin_unlock_bh(&ap_device_list_lock);

	rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
	if (rc) {
		spin_lock_bh(&ap_device_list_lock);
		list_del_init(&ap_dev->list);
		spin_unlock_bh(&ap_device_list_lock);
	}
	return rc;
}

Project: linux

static void ap_device_release(struct device *dev)
{
	struct ap_device *ap_dev = to_ap_dev(dev);

	kfree(ap_dev);
}

Project: linux

static int ap_device_remove(struct device *dev)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	struct ap_driver *ap_drv = ap_dev->drv;

	ap_flush_queue(ap_dev);
	del_timer_sync(&ap_dev->timeout);
	spin_lock_bh(&ap_device_list_lock);
	list_del_init(&ap_dev->list);
	spin_unlock_bh(&ap_device_list_lock);
	if (ap_drv->remove)
		ap_drv->remove(ap_dev);
	spin_lock_bh(&ap_dev->lock);
	atomic_sub(ap_dev->queue_count, &ap_poll_requests);
	spin_unlock_bh(&ap_dev->lock);
	return 0;
}

Project: linux

static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
{
	return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
}

Project: linux

int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
		       char *name)
{
	struct device_driver *drv = &ap_drv->driver;

	drv->bus = &ap_bus_type;
	drv->probe = ap_device_probe;
	drv->remove = ap_device_remove;
	drv->owner = owner;
	drv->name = name;
	return driver_register(drv);
}

Project: linux

void ap_flush_queue(struct ap_device *ap_dev)
{
	spin_lock_bh(&ap_dev->lock);
	__ap_flush_queue(ap_dev);
	spin_unlock_bh(&ap_dev->lock);
}

Project: linux

static ssize_t ap_hwtype_show(struct device *dev,
			      struct device_attribute *attr, char *buf)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
}

Project: linux

static void ap_increase_queue_count(struct ap_device *ap_dev)
{
	int timeout = ap_dev->drv->request_timeout;

	ap_dev->queue_count++;
	if (ap_dev->queue_count == 1) {
		mod_timer(&ap_dev->timeout, jiffies + timeout);
		ap_dev->reset = AP_RESET_ARMED;
	}
}

Project: linux

static int ap_init_queue(ap_qid_t qid)
{
	struct ap_queue_status status;
	int rc, dummy, i;

	rc = -ENODEV;
	status = ap_reset_queue(qid);
	for (i = 0; i < AP_MAX_RESET; i++) {
		switch (status.response_code) {
		case AP_RESPONSE_NORMAL:
			if (status.queue_empty)
				rc = 0;
			break;
		case AP_RESPONSE_Q_NOT_AVAIL:
		case AP_RESPONSE_DECONFIGURED:
		case AP_RESPONSE_CHECKSTOPPED:
			i = AP_MAX_RESET;	/* return with -ENODEV */
			break;
		case AP_RESPONSE_RESET_IN_PROGRESS:
			rc = -EBUSY;
		case AP_RESPONSE_BUSY:
		default:
			break;
		}
		if (rc != -ENODEV && rc != -EBUSY)
			break;
		if (i < AP_MAX_RESET - 1) {
			/* Time we are waiting until we give up (0.7sec * 90).
			 * Since the actual request (in progress) will not
			 * interrupted immediately for the reset command,
			 * we have to be patient. In worst case we have to
			 * wait 60sec + reset time (some msec).
			 */
			schedule_timeout(AP_RESET_TIMEOUT);
			status = ap_test_queue(qid, &dummy, &dummy);
		}
	}
	if (rc == 0 && ap_using_interrupts()) {
		rc = ap_queue_enable_interruption(qid, ap_airq.lsi_ptr);
		/* If interruption mode is supported by the machine,
		* but an AP can not be enabled for interruption then
		* the AP will be discarded.    */
		if (rc)
			pr_err("Registering adapter interrupts for "
			       "AP %d failed\n", AP_QID_DEVICE(qid));
	}
	return rc;
}

Project: linux

static inline int ap_instructions_available(void)
{
	register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
	register unsigned long reg1 asm ("1") = -ENODEV;
	register unsigned long reg2 asm ("2") = 0UL;

	asm volatile(
		"   .long 0xb2af0000\n"		/* PQAP(TAPQ) */
		"0: la    %1,0\n"
		"1:\n"
		EX_TABLE(0b, 1b)
		: "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
	return reg1;
}

Project: linux

static int ap_interrupts_available(void)
{
	return test_facility(2) && test_facility(65);
}

Project: linux

static ssize_t ap_interrupts_show(struct bus_type *bus, char *buf)
{
	return snprintf(buf, PAGE_SIZE, "%d\n",
			ap_using_interrupts() ? 1 : 0);
}

Project: linux

static ssize_t ap_modalias_show(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
}

Project: linux

void ap_module_exit(void)
{
	int i;
	struct device *dev;

	ap_reset_domain();
	ap_poll_thread_stop();
	del_timer_sync(&ap_config_timer);
	hrtimer_cancel(&ap_poll_timer);
	destroy_workqueue(ap_work_queue);
	tasklet_kill(&ap_tasklet);
	root_device_unregister(ap_root_device);
	while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
		    __ap_match_all)))
	{
		device_unregister(dev);
		put_device(dev);
	}
	for (i = 0; ap_bus_attrs[i]; i++)
		bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
	bus_unregister(&ap_bus_type);
	unregister_reset_call(&ap_reset_call);
	if (ap_using_interrupts())
		unregister_adapter_interrupt(&ap_airq);
}

Project: linux

int __init ap_module_init(void)
{
	int rc, i;

	if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
		pr_warning("%d is not a valid cryptographic domain\n",
			   ap_domain_index);
		return -EINVAL;
	}
	/* In resume callback we need to know if the user had set the domain.
	 * If so, we can not just reset it.
	 */
	if (ap_domain_index >= 0)
		user_set_domain = 1;

	if (ap_instructions_available() != 0) {
		pr_warning("The hardware system does not support "
			   "AP instructions\n");
		return -ENODEV;
	}
	if (ap_interrupts_available()) {
		rc = register_adapter_interrupt(&ap_airq);
		ap_airq_flag = (rc == 0);
	}

	register_reset_call(&ap_reset_call);

	/* Create /sys/bus/ap. */
	rc = bus_register(&ap_bus_type);
	if (rc)
		goto out;
	for (i = 0; ap_bus_attrs[i]; i++) {
		rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
		if (rc)
			goto out_bus;
	}

	/* Create /sys/devices/ap. */
	ap_root_device = root_device_register("ap");
	rc = PTR_RET(ap_root_device);
	if (rc)
		goto out_bus;

	ap_work_queue = create_singlethread_workqueue("kapwork");
	if (!ap_work_queue) {
		rc = -ENOMEM;
		goto out_root;
	}

	ap_query_configuration();
	if (ap_select_domain() == 0)
		ap_scan_bus(NULL);

	/* Setup the AP bus rescan timer. */
	init_timer(&ap_config_timer);
	ap_config_timer.function = ap_config_timeout;
	ap_config_timer.data = 0;
	ap_config_timer.expires = jiffies + ap_config_time * HZ;
	add_timer(&ap_config_timer);

	/* Setup the high resultion poll timer.
	 * If we are running under z/VM adjust polling to z/VM polling rate.
	 */
	if (MACHINE_IS_VM)
		poll_timeout = 1500000;
	spin_lock_init(&ap_poll_timer_lock);
	hrtimer_init(&ap_poll_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
	ap_poll_timer.function = ap_poll_timeout;

	/* Start the low priority AP bus poll thread. */
	if (ap_thread_flag) {
		rc = ap_poll_thread_start();
		if (rc)
			goto out_work;
	}

	return 0;

out_work:
	del_timer_sync(&ap_config_timer);
	hrtimer_cancel(&ap_poll_timer);
	destroy_workqueue(ap_work_queue);
out_root:
	root_device_unregister(ap_root_device);
out_bus:
	while (i--)
		bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
	bus_unregister(&ap_bus_type);
out:
	unregister_reset_call(&ap_reset_call);
	if (ap_using_interrupts())
		unregister_adapter_interrupt(&ap_airq);
	return rc;
}

Project: linux

static ssize_t ap_pendingq_count_show(struct device *dev,
				      struct device_attribute *attr, char *buf)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	int rc;

	spin_lock_bh(&ap_dev->lock);
	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->pendingq_count);
	spin_unlock_bh(&ap_dev->lock);
	return rc;
}

Project: linux

static void ap_poll_all(unsigned long dummy)
{
	unsigned long flags;
	struct ap_device *ap_dev;

	/* Reset the indicator if interrupts are used. Thus new interrupts can
	 * be received. Doing it in the beginning of the tasklet is therefor
	 * important that no requests on any AP get lost.
	 */
	if (ap_using_interrupts())
		xchg(ap_airq.lsi_ptr, 0);
	do {
		flags = 0;
		spin_lock(&ap_device_list_lock);
		list_for_each_entry(ap_dev, &ap_device_list, list) {
			spin_lock(&ap_dev->lock);
			__ap_poll_device(ap_dev, &flags);
			spin_unlock(&ap_dev->lock);
		}
		spin_unlock(&ap_device_list_lock);
	} while (flags & 1);
	if (flags & 2)
		ap_schedule_poll_timer();
}

Project: linux

static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
{
	int rc;

	rc = ap_poll_read(ap_dev, flags);
	if (rc)
		return rc;
	return ap_poll_write(ap_dev, flags);
}

Project: linux

static int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
{
	struct ap_queue_status status;
	struct ap_message *ap_msg;

	if (ap_dev->queue_count <= 0)
		return 0;
	status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
			   ap_dev->reply->message, ap_dev->reply->length);
	switch (status.response_code) {
	case AP_RESPONSE_NORMAL:
		atomic_dec(&ap_poll_requests);
		ap_decrease_queue_count(ap_dev);
		list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
			if (ap_msg->psmid != ap_dev->reply->psmid)
				continue;
			list_del_init(&ap_msg->list);
			ap_dev->pendingq_count--;
			ap_msg->receive(ap_dev, ap_msg, ap_dev->reply);
			break;
		}
		if (ap_dev->queue_count > 0)
			*flags |= 1;
		break;
	case AP_RESPONSE_NO_PENDING_REPLY:
		if (status.queue_empty) {
			/* The card shouldn't forget requests but who knows. */
			atomic_sub(ap_dev->queue_count, &ap_poll_requests);
			ap_dev->queue_count = 0;
			list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
			ap_dev->requestq_count += ap_dev->pendingq_count;
			ap_dev->pendingq_count = 0;
		} else
			*flags |= 2;
		break;
	default:
		return -ENODEV;
	}
	return 0;
}

Project: linux

static int ap_poll_thread(void *data)
{
	DECLARE_WAITQUEUE(wait, current);
	unsigned long flags;
	int requests;
	struct ap_device *ap_dev;

	set_user_nice(current, MAX_NICE);
	while (1) {
		if (ap_suspend_flag)
			return 0;
		if (need_resched()) {
			schedule();
			continue;
		}
		add_wait_queue(&ap_poll_wait, &wait);
		set_current_state(TASK_INTERRUPTIBLE);
		if (kthread_should_stop())
			break;
		requests = atomic_read(&ap_poll_requests);
		if (requests <= 0)
			schedule();
		set_current_state(TASK_RUNNING);
		remove_wait_queue(&ap_poll_wait, &wait);

		flags = 0;
		spin_lock_bh(&ap_device_list_lock);
		list_for_each_entry(ap_dev, &ap_device_list, list) {
			spin_lock(&ap_dev->lock);
			__ap_poll_device(ap_dev, &flags);
			spin_unlock(&ap_dev->lock);
		}
		spin_unlock_bh(&ap_device_list_lock);
	}
	set_current_state(TASK_RUNNING);
	remove_wait_queue(&ap_poll_wait, &wait);
	return 0;
}

Project: linux

static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
{
	return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
}

Project: linux

static int ap_poll_thread_start(void)
{
	int rc;

	if (ap_using_interrupts() || ap_suspend_flag)
		return 0;
	mutex_lock(&ap_poll_thread_mutex);
	if (!ap_poll_kthread) {
		ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
		rc = PTR_RET(ap_poll_kthread);
		if (rc)
			ap_poll_kthread = NULL;
	}
	else
		rc = 0;
	mutex_unlock(&ap_poll_thread_mutex);
	return rc;
}

Project: linux

static void ap_poll_thread_stop(void)
{
	mutex_lock(&ap_poll_thread_mutex);
	if (ap_poll_kthread) {
		kthread_stop(ap_poll_kthread);
		ap_poll_kthread = NULL;
	}
	mutex_unlock(&ap_poll_thread_mutex);
}

Project: linux

static ssize_t ap_poll_thread_store(struct bus_type *bus,
				    const char *buf, size_t count)
{
	int flag, rc;

	if (sscanf(buf, "%d\n", &flag) != 1)
		return -EINVAL;
	if (flag) {
		rc = ap_poll_thread_start();
		if (rc)
			return rc;
	}
	else
		ap_poll_thread_stop();
	return count;
}

Project: linux

static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused)
{
	tasklet_schedule(&ap_tasklet);
	return HRTIMER_NORESTART;
}

Project: linux

static int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
{
	struct ap_queue_status status;
	struct ap_message *ap_msg;

	if (ap_dev->requestq_count <= 0 ||
	    ap_dev->queue_count >= ap_dev->queue_depth)
		return 0;
	/* Start the next request on the queue. */
	ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
	status = __ap_send(ap_dev->qid, ap_msg->psmid,
			   ap_msg->message, ap_msg->length, ap_msg->special);
	switch (status.response_code) {
	case AP_RESPONSE_NORMAL:
		atomic_inc(&ap_poll_requests);
		ap_increase_queue_count(ap_dev);
		list_move_tail(&ap_msg->list, &ap_dev->pendingq);
		ap_dev->requestq_count--;
		ap_dev->pendingq_count++;
		if (ap_dev->queue_count < ap_dev->queue_depth &&
		    ap_dev->requestq_count > 0)
			*flags |= 1;
		*flags |= 2;
		break;
	case AP_RESPONSE_RESET_IN_PROGRESS:
		__ap_schedule_poll_timer();
	case AP_RESPONSE_Q_FULL:
		*flags |= 2;
		break;
	case AP_RESPONSE_MESSAGE_TOO_BIG:
	case AP_RESPONSE_REQ_FAC_NOT_INST:
		return -EINVAL;
	default:
		return -ENODEV;
	}
	return 0;
}

Project: linux

static int ap_probe_device_type(struct ap_device *ap_dev)
{
	static unsigned char msg[] = {
		0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
		0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
		0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
		0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
		0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
		0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
		0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
		0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
		0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
		0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
		0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
		0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
		0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
		0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
		0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
		0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
		0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
		0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
		0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
		0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
		0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
		0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
		0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
		0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
		0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
		0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
	};
	struct ap_queue_status status;
	unsigned long long psmid;
	char *reply;
	int rc, i;

	reply = (void *) get_zeroed_page(GFP_KERNEL);
	if (!reply) {
		rc = -ENOMEM;
		goto out;
	}

	status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
			   msg, sizeof(msg), 0);
	if (status.response_code != AP_RESPONSE_NORMAL) {
		rc = -ENODEV;
		goto out_free;
	}

	/* Wait for the test message to complete. */
	for (i = 0; i < 6; i++) {
		mdelay(300);
		status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
		if (status.response_code == AP_RESPONSE_NORMAL &&
		    psmid == 0x0102030405060708ULL)
			break;
	}
	if (i < 6) {
		/* Got an answer. */
		if (reply[0] == 0x00 && reply[1] == 0x86)
			ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
		else
			ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
		rc = 0;
	} else
		rc = -ENODEV;

out_free:
	free_page((unsigned long) reply);
out:
	return rc;
}

Project: linux

static void ap_query_configuration(void)
{
#ifdef CONFIG_64BIT
	if (ap_configuration_available()) {
		if (!ap_configuration)
			ap_configuration =
				kzalloc(sizeof(struct ap_config_info),
					GFP_KERNEL);
		if (ap_configuration)
			__ap_query_configuration(ap_configuration);
	} else
		ap_configuration = NULL;
#else
	ap_configuration = NULL;
#endif
}

Project: linux

static int ap_query_functions(ap_qid_t qid, unsigned int *functions)
{
#ifdef CONFIG_64BIT
	struct ap_queue_status status;
	int i;
	status = __ap_query_functions(qid, functions);

	for (i = 0; i < AP_MAX_RESET; i++) {
		if (ap_queue_status_invalid_test(&status))
			return -ENODEV;

		switch (status.response_code) {
		case AP_RESPONSE_NORMAL:
			return 0;
		case AP_RESPONSE_RESET_IN_PROGRESS:
		case AP_RESPONSE_BUSY:
			break;
		case AP_RESPONSE_Q_NOT_AVAIL:
		case AP_RESPONSE_DECONFIGURED:
		case AP_RESPONSE_CHECKSTOPPED:
		case AP_RESPONSE_INVALID_ADDRESS:
			return -ENODEV;
		case AP_RESPONSE_OTHERWISE_CHANGED:
			break;
		default:
			break;
		}
		if (i < AP_MAX_RESET - 1) {
			udelay(5);
			status = __ap_query_functions(qid, functions);
		}
	}
	return -EBUSY;
#else
	return -EINVAL;
#endif
}

Project: linux

static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
{
	struct ap_queue_status status;
	int t_depth, t_device_type, rc, i;

	rc = -EBUSY;
	for (i = 0; i < AP_MAX_RESET; i++) {
		status = ap_test_queue(qid, &t_depth, &t_device_type);
		switch (status.response_code) {
		case AP_RESPONSE_NORMAL:
			*queue_depth = t_depth + 1;
			*device_type = t_device_type;
			rc = 0;
			break;
		case AP_RESPONSE_Q_NOT_AVAIL:
			rc = -ENODEV;
			break;
		case AP_RESPONSE_RESET_IN_PROGRESS:
			break;
		case AP_RESPONSE_DECONFIGURED:
			rc = -ENODEV;
			break;
		case AP_RESPONSE_CHECKSTOPPED:
			rc = -ENODEV;
			break;
		case AP_RESPONSE_INVALID_ADDRESS:
			rc = -ENODEV;
			break;
		case AP_RESPONSE_OTHERWISE_CHANGED:
			break;
		case AP_RESPONSE_BUSY:
			break;
		default:
			BUG();
		}
		if (rc != -EBUSY)
			break;
		if (i < AP_MAX_RESET - 1)
			udelay(5);
	}
	return rc;
}

Project: linux

ap_queue_interruption_control(ap_qid_t qid, void *ind)
{
	register unsigned long reg0 asm ("0") = qid | 0x03000000UL;
	register unsigned long reg1_in asm ("1") = 0x0000800000000000UL | AP_ISC;
	register struct ap_queue_status reg1_out asm ("1");
	register void *reg2 asm ("2") = ind;
	asm volatile(
		".long 0xb2af0000"		/* PQAP(AQIC) */
		: "+d" (reg0), "+d" (reg1_in), "=d" (reg1_out), "+d" (reg2)
		:
		: "cc" );
	return reg1_out;
}

Project: linux

void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
{
	unsigned long flags;
	int rc;

	/* For asynchronous message handling a valid receive-callback
	 * is required. */
	BUG_ON(!ap_msg->receive);

	spin_lock_bh(&ap_dev->lock);
	if (!ap_dev->unregistered) {
		/* Make room on the queue by polling for finished requests. */
		rc = ap_poll_queue(ap_dev, &flags);
		if (!rc)
			rc = __ap_queue_message(ap_dev, ap_msg);
		if (!rc)
			wake_up(&ap_poll_wait);
		if (rc == -ENODEV)
			ap_dev->unregistered = 1;
	} else {
		ap_msg->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
		rc = -ENODEV;
	}
	spin_unlock_bh(&ap_dev->lock);
	if (rc == -ENODEV)
		device_unregister(&ap_dev->device);
}

Project: linux

static ssize_t ap_raw_hwtype_show(struct device *dev,
			      struct device_attribute *attr, char *buf)
{
	struct ap_device *ap_dev = to_ap_dev(dev);

	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->raw_hwtype);
}

Project: linux

static ssize_t ap_request_count_show(struct device *dev,
				     struct device_attribute *attr,
				     char *buf)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	int rc;

	spin_lock_bh(&ap_dev->lock);
	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
	spin_unlock_bh(&ap_dev->lock);
	return rc;
}

Project: linux

static void ap_request_timeout(unsigned long data)
{
	struct ap_device *ap_dev = (struct ap_device *) data;

	if (ap_dev->reset == AP_RESET_ARMED) {
		ap_dev->reset = AP_RESET_DO;

		if (ap_using_interrupts())
			tasklet_schedule(&ap_tasklet);
	}
}

Project: linux

static void ap_reset(struct ap_device *ap_dev)
{
	int rc;

	ap_dev->reset = AP_RESET_IGNORE;
	atomic_sub(ap_dev->queue_count, &ap_poll_requests);
	ap_dev->queue_count = 0;
	list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
	ap_dev->requestq_count += ap_dev->pendingq_count;
	ap_dev->pendingq_count = 0;
	rc = ap_init_queue(ap_dev->qid);
	if (rc == -ENODEV)
		ap_dev->unregistered = 1;
	else
		__ap_schedule_poll_timer();
}

Project: linux

static void ap_reset_all(void)
{
	int i, j;

	for (i = 0; i < AP_DOMAINS; i++) {
		if (!ap_test_config_domain(i))
			continue;
		for (j = 0; j < AP_DEVICES; j++) {
			if (!ap_test_config_card_id(j))
				continue;
			ap_reset_queue(AP_MKQID(j, i));
		}
	}
}

Project: linux

static void ap_reset_domain(void)
{
	int i;

	if (ap_domain_index != -1)
		for (i = 0; i < AP_DEVICES; i++)
			ap_reset_queue(AP_MKQID(i, ap_domain_index));
}

Project: linux

static void ap_scan_bus(struct work_struct *unused)
{
	struct ap_device *ap_dev;
	struct device *dev;
	ap_qid_t qid;
	int queue_depth, device_type;
	unsigned int device_functions;
	int rc, i;

	ap_query_configuration();
	if (ap_select_domain() != 0) {
		return;
	}
	for (i = 0; i < AP_DEVICES; i++) {
		qid = AP_MKQID(i, ap_domain_index);
		dev = bus_find_device(&ap_bus_type, NULL,
				      (void *)(unsigned long)qid,
				      __ap_scan_bus);
		if (ap_test_config_card_id(i))
			rc = ap_query_queue(qid, &queue_depth, &device_type);
		else
			rc = -ENODEV;
		if (dev) {
			if (rc == -EBUSY) {
				set_current_state(TASK_UNINTERRUPTIBLE);
				schedule_timeout(AP_RESET_TIMEOUT);
				rc = ap_query_queue(qid, &queue_depth,
						    &device_type);
			}
			ap_dev = to_ap_dev(dev);
			spin_lock_bh(&ap_dev->lock);
			if (rc || ap_dev->unregistered) {
				spin_unlock_bh(&ap_dev->lock);
				if (ap_dev->unregistered)
					i--;
				device_unregister(dev);
				put_device(dev);
				continue;
			}
			spin_unlock_bh(&ap_dev->lock);
			put_device(dev);
			continue;
		}
		if (rc)
			continue;
		rc = ap_init_queue(qid);
		if (rc)
			continue;
		ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
		if (!ap_dev)
			break;
		ap_dev->qid = qid;
		ap_dev->queue_depth = queue_depth;
		ap_dev->unregistered = 1;
		spin_lock_init(&ap_dev->lock);
		INIT_LIST_HEAD(&ap_dev->pendingq);
		INIT_LIST_HEAD(&ap_dev->requestq);
		INIT_LIST_HEAD(&ap_dev->list);
		setup_timer(&ap_dev->timeout, ap_request_timeout,
			    (unsigned long) ap_dev);
		switch (device_type) {
		case 0:
			/* device type probing for old cards */
			if (ap_probe_device_type(ap_dev)) {
				kfree(ap_dev);
				continue;
			}
			break;
		case 11:
			ap_dev->device_type = 10;
			break;
		default:
			ap_dev->device_type = device_type;
		}
		ap_dev->raw_hwtype = device_type;

		rc = ap_query_functions(qid, &device_functions);
		if (!rc)
			ap_dev->functions = device_functions;
		else
			ap_dev->functions = 0u;

		ap_dev->device.bus = &ap_bus_type;
		ap_dev->device.parent = ap_root_device;
		if (dev_set_name(&ap_dev->device, "card%02x",
				 AP_QID_DEVICE(ap_dev->qid))) {
			kfree(ap_dev);
			continue;
		}
		ap_dev->device.release = ap_device_release;
		rc = device_register(&ap_dev->device);
		if (rc) {
			put_device(&ap_dev->device);
			continue;
		}
		/* Add device attributes. */
		rc = sysfs_create_group(&ap_dev->device.kobj,
					&ap_dev_attr_group);
		if (!rc) {
			spin_lock_bh(&ap_dev->lock);
			ap_dev->unregistered = 0;
			spin_unlock_bh(&ap_dev->lock);
		}
		else
			device_unregister(&ap_dev->device);
	}
}

Project: linux

static inline void ap_schedule_poll_timer(void)
{
	if (ap_using_interrupts())
		return;
	__ap_schedule_poll_timer();
}

Project: linux

static inline int ap_test_config(unsigned int *field, unsigned int nr)
{
	if (nr > 0xFFu)
		return 0;
	return ap_test_bit((field + (nr >> 5)), (nr & 0x1f));
}

Project: linux

static inline int ap_test_config_card_id(unsigned int id)
{
	if (!ap_configuration)
		return 1;
	return ap_test_config(ap_configuration->apm, id);
}

Project: linux

static inline int ap_test_config_domain(unsigned int domain)
{
	if (!ap_configuration)
		return 1;
	return ap_test_config(ap_configuration->aqm, domain);
}

Project: linux

ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
{
	register unsigned long reg0 asm ("0") = qid;
	register struct ap_queue_status reg1 asm ("1");
	register unsigned long reg2 asm ("2") = 0UL;

	asm volatile(".long 0xb2af0000"		/* PQAP(TAPQ) */
		     : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
	*device_type = (int) (reg2 >> 24);
	*queue_depth = (int) (reg2 & 0xff);
	return reg1;
}

Project: linux

static int ap_uevent (struct device *dev, struct kobj_uevent_env *env)
{
	struct ap_device *ap_dev = to_ap_dev(dev);
	int retval = 0;

	if (!ap_dev)
		return -ENODEV;

	/* Set up DEV_TYPE environment variable. */
	retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type);
	if (retval)
		return retval;

	/* Add MODALIAS= */
	retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type);

	return retval;
}

Project: linux

static inline int ap_using_interrupts(void)
{
	return ap_airq_flag;
}

Project: linux

static ssize_t poll_timeout_store(struct bus_type *bus, const char *buf,
				  size_t count)
{
	unsigned long long time;
	ktime_t hr_time;

	/* 120 seconds = maximum poll interval */
	if (sscanf(buf, "%llu\n", &time) != 1 || time < 1 ||
	    time > 120000000000ULL)
		return -EINVAL;
	poll_timeout = time;
	hr_time = ktime_set(0, poll_timeout);

	if (!hrtimer_is_queued(&ap_poll_timer) ||
	    !hrtimer_forward(&ap_poll_timer, hrtimer_get_expires(&ap_poll_timer), hr_time)) {
		hrtimer_set_expires(&ap_poll_timer, hr_time);
		hrtimer_start_expires(&ap_poll_timer, HRTIMER_MODE_ABS);
	}
	return count;
}