fio

# Commands

https://qiita.com/Taroi_Japanista/items/16b6b6a6e64ecc6dfa44

## dd command

<pre>
SeqR: time dd if=/nvme/testfile of=/dev/null bs=1024k iflag=direct
SeqW: time dd if=/dev/zero of=/nvme/testfile bs=1024k count=8192 oflag=direct
</pre>

https://developers.cyberagent.co.jp/blog/archives/6623/

https://www.na3.jp/entry/20150831/p1

https://software.intel.com/content/www/us/en/develop/articles/evaluate-performance-for-storage-performance-development-kit-spdk-based-nvme-ssd.html

<pre>
[Global]
ioengine=libaio
direct=1
rw=randrw
rwmixread=100 (100% reads), 70 (70% reads 30% writes), 0 (100 writes)
thread=1
norandommap=1
time_base=1
runtime=300s
ramp_time=10s
bs=4k
iodepth=32
Numjobs=1
[test]
filename=/dev/nvme0n1
</pre>

https://qiita.com/toshihirock/items/fa4d310115e6921ab0ac

## Block size: 4k
<pre>
SeqR: fio -filename=/tmp/test2g -direct=1 -rw=read -bs=4k -size=2G -numjobs=64 -runtime=10 -group_reporting -name=file1
SeqW: fio -filename=/tmp/test2g -direct=1 -rw=write -bs=4k -size=2G -numjobs=64 -runtime=10 -group_reporting -name=file1
RndR: fio -filename=/tmp/test2g -direct=1 -rw=randread -bs=4k -size=2G -numjobs=64 -runtime=10 -group_reporting -name=file1
RndW: fio -filename=/tmp/test2g -direct=1 -rw=randwrite -bs=4k -size=2G -numjobs=64 -runtime=10 -group_reporting -name=file1
</pre>

## Block size: 32m

<pre>
SeqR: fio -filename=/tmp/test2g -direct=1 -rw=read -bs=32m -size=2G -numjobs=16 -runtime=10 -group_reporting -name=file1
SeqW: fio -filename=/tmp/test2g -direct=1 -rw=write -bs=32m -size=2G -numjobs=16 -runtime=10 -group_reporting -name=file1
RndR: fio -filename=/tmp/test2g -direct=1 -rw=randread -bs=32m -size=2G -numjobs=16 -runtime=10 -group_reporting -name=file1
RndW: fio -filename=/tmp/test2g -direct=1 -rw=randwrite -bs=32m -size=2G -numjobs=16 -runtime=10 -group_reporting -name=file1
</pre>

# Links

https://github.com/axboe/fio

https://linux.die.net/man/1/fio

https://docs.oracle.com/en-us/iaas/Content/Block/References/samplefiocommandslinux.htm

https://developers.cyberagent.co.jp/blog/archives/6623/

# linux storage stack diagram

https://nvmexpress.org/education/drivers/linux-driver-information/
https://www.thomas-krenn.com/en/wiki/Linux_Storage_Stack_Diagram
![Linux-storage-stack-diagram_v4.10-e1575939041721.png](https://qiita-image-store.s3.ap-northeast-1.amazonaws.com/0/632906/37bf457c-d8c4-572e-2fac-d173641bb912.png)

# aio (linux)

## fs/notify/fsnotify.c

<pre>
__fsnotify_parent()
  fsnotify()
    send_to_group()
      fsnotify_handle_event()
        fsnotify_handle_inode_event()
          return ops->handle_inode_event(inode_mark, mask, inode, dir, name, cookie);
</pre>

## include/linux/fsnotify.h

<pre>
fsnotify_perm()
  fsnotify_file()
    fsnotify_parent()
      __fsnotify_parent()
</pre>

## fs/read_write.c

<pre>
rw_verify_area()
  security_file_permission()
    fsnotify_perm()
</pre>

## fs/aio.c

<pre>
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
		struct iocb __user * __user *, iocbpp)
  io_submit_one()
    __io_submit_one()
      aio_read()
        rw_verify_area()
        aio_rw_done()
      aio_write()
        rw_verify_area()
        aio_rw_done()
</pre>

## include/linux/syscalls.h

<pre>
#define SYSCALL_METADATA(sname, nb, ...)			\
	static const char *types_##sname[] = {			\
		__MAP(nb,__SC_STR_TDECL,__VA_ARGS__)		\
	};							\
	static const char *args_##sname[] = {			\
		__MAP(nb,__SC_STR_ADECL,__VA_ARGS__)		\
	};							\
	SYSCALL_TRACE_ENTER_EVENT(sname);			\
	SYSCALL_TRACE_EXIT_EVENT(sname);			\
	static struct syscall_metadata __used			\
	  __syscall_meta_##sname = {				\
		.name 		= "sys"#sname,			\
		.syscall_nr	= -1,	/* Filled in at boot */	\
		.nb_args 	= nb,				\
		.types		= nb ? types_##sname : NULL,	\
		.args		= nb ? args_##sname : NULL,	\
		.enter_event	= &event_enter_##sname,		\
		.exit_event	= &event_exit_##sname,		\
		.enter_fields	= LIST_HEAD_INIT(__syscall_meta_##sname.enter_fields), \
	};							\
	static struct syscall_metadata __used			\
	  __section("__syscalls_metadata")			\
	 *__p_syscall_meta_##sname = &__syscall_meta_##sname;

#define SYSCALL_DEFINE3(name, ...) SYSCALL_DEFINEx(3, _##name, __VA_ARGS__)

#define SYSCALL_DEFINEx(x, sname, ...)				\
	SYSCALL_METADATA(sname, x, __VA_ARGS__)			\
	__SYSCALL_DEFINEx(x, sname, __VA_ARGS__)

#define __SYSCALL_DEFINEx(x, name, ...)					\
	__diag_push();							\
	__diag_ignore(GCC, 8, "-Wattribute-alias",			\
		      "Type aliasing is used to sanitize syscall arguments");\
	asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))	\
		__attribute__((alias(__stringify(__se_sys##name))));	\
	ALLOW_ERROR_INJECTION(sys##name, ERRNO);			\
	static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));\
	asmlinkage long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__));	\
	asmlinkage long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__))	\
	{								\
		long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__));\
		__MAP(x,__SC_TEST,__VA_ARGS__);				\
		__PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__));	\
		return ret;						\
	}								\
	__diag_pop();							\
	static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
</pre>

# Latency

## ioengines.c
<pre>
td_io_queue()
  fio_gettime(&io_u->issue_time, NULL);
</pre>

<pre>
td_io_commit()
  td->io_ops->commit() -> fio_libaio_commit()
</pre>

## engines\libaio.c
<pre>
fio_libaio_commit()
  io_submit()
  fio_libaio_queued()
    fio_gettime(&now, NULL);
    memcpy(&io_u->issue_time, &now, sizeof(now));
</pre>

## backend.c
<pre>
do_verify()
  fio_gettime(&io_u->start_time, NULL);
</pre>

<pre>
io_u_submit()
  td_io_queue()

fio_io_sync()
  td_io_queue()
</pre>

## io_u.c
<pre>
get_io_u()
  fio_gettime(&io_u->start_time, NULL);
</pre>

<pre>
io_u_queued()
  slat_time = ntime_since(&io_u->start_time, &io_u->issue_time);
  add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen, io_u->offset, io_u_is_prio(io_u));
</pre>

<pre>
io_u_queued_complete()
  init_icd()
    fio_gettime(&icd->time, NULL);
  ios_completed()
    account_io_completion()
      llnsec = ntime_since(&io_u->issue_time, &icd->time);
      tnsec = ntime_since(&io_u->start_time, &icd->time);
      add_lat_sample(td, idx, tnsec, bytes, io_u->offset, io_u_is_prio(io_u));
      add_clat_sample(td, idx, llnsec, bytes, io_u->offset, io_u_is_prio(io_u));
      io_u_mark_latency(td, llnsec);
</pre>

## rate-submit.c
<pre>
io_workqueue_fn()
  td_io_queue()
</pre>