Repository: opencontainers/runc
Pull request #2407 (2020-05-15 12:27:24)

Update blkio GetStats
Updates blkio GetStats, as proposed in [2365](https://github.com/opencontainers/runc/issues/2365)

Signed-off-by: Piotr Wagner <piotr.wagner@intel.com>

piowag: What do you think about changes introduced in this pull request?
piowag: @AkihiroSuda Could you take a look at this PR?
piowag: @crosbymichael, @mrunalp, @dqminh, @hqhq, @cyphar Could you take a look at this pull request?
crosbymichael: Overall, I think the ability to handle the correct blkio stats is good.

It would be good to update the code in this PR to handle these various sources in a loop.  Also the whole move next logic could be replaced by a typed error, which is the common way to handle this in Go.  But I think you can get away from all of that error handling by just trying to read all the sources in a loop and just continuing on error. 
piowag: Is this close to what you had in mind @crosbymichael ?
Creatone: lgtm
harche: I ran into similar issue where `blkio.throttle.io_service_bytes_recursive` and `blkio.throttle.io_serviced_recursive` contained accurate information compared to `blkio.throttle.io_service_bytes` and `blkio.throttle.io_serviced` respectively. 

This PR solves that issue by reading  `blkio.throttle.io_service_bytes_recursive` and `blkio.throttle.io_serviced_recursive` over `blkio.throttle.io_service_bytes` and `blkio.throttle.io_serviced` respectively. 

@piowag thanks for this PR. :) 
harche: Just to add, I have verified the proposed fix of preferring to use `_recursive` file by this rudimentary diff,

```diff
diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go
index 52c118d68c1..eb25d4af61b 100644
--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go
+++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go
@@ -224,15 +224,42 @@ func getStats(path string, stats *cgroups.Stats) error {
        var blkioStats []cgroups.BlkioStatEntry
        var err error
 
-       if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_service_bytes")); err != nil {
+       ioServiceBytesRecursiveFileName, err := getRecursiveFileName(filepath.Join(path, "blkio.throttle.io_service_bytes"))
+       if err != nil {
+               return err
+       }
+
+       if blkioStats, err = getBlkioStat(ioServiceBytesRecursiveFileName); err != nil {
                return err
        }
        stats.BlkioStats.IoServiceBytesRecursive = blkioStats
 
-       if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_serviced")); err != nil {
+       ioServicedRecursiveFileName, err := getRecursiveFileName(filepath.Join(path, "blkio.throttle.io_serviced"))
+       if err != nil {
+               return err
+       }
+
+       if blkioStats, err = getBlkioStat(ioServicedRecursiveFileName); err != nil {
                return err
        }
        stats.BlkioStats.IoServicedRecursive = blkioStats
 
        return nil
 }
+
+func getRecursiveFileName(path string) (string, error) {
+       var cgroupFile string
+
+       recursiveFileName := filepath.Join(path, "_recursive")
+       if _, err := os.Stat(recursiveFileName); err == nil {
+               cgroupFile = recursiveFileName
+
+       } else if os.IsNotExist(err) {
+               cgroupFile = path
+
+       } else {
+               return "", err
+       }
+
+       return cgroupFile, nil
+}
```

but I think this PR handles it more gracefully. 
Creatone: @kolyshkin @AkihiroSuda 
kolyshkin: @piowag PTAL
Creatone: @kolyshkin you gave comments to outdated code
Creatone: > @kolyshkin you gave comments to outdated code

Ok, now I understand. Nits fixed.

AkihiroSuda: @kolyshkin LGTY?


Patch:
diff --git a/libcontainer/cgroups/fs/blkio.go b/libcontainer/cgroups/fs/blkio.go
index 4d5e5beb..dca2a622 100644
--- a/libcontainer/cgroups/fs/blkio.go
+++ b/libcontainer/cgroups/fs/blkio.go
@@ -157,73 +157,134 @@ func getBlkioStat(dir, file string) ([]cgroups.BlkioStatEntry, error) {
 }
 
 func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error {
-	// Try to read CFQ stats available on all CFQ enabled kernels first
-	if blkioStats, err := getBlkioStat(path, "blkio.io_serviced_recursive"); err == nil && blkioStats != nil {
-		return getCFQStats(path, stats)
+	type blkioStatInfo struct {
+		filename            string
+		blkioStatEntriesPtr *[]cgroups.BlkioStatEntry
 	}
-	return getStats(path, stats) // Use generic stats as fallback
-}
-
-func getCFQStats(path string, stats *cgroups.Stats) error {
-	var blkioStats []cgroups.BlkioStatEntry
-	var err error
-
-	if blkioStats, err = getBlkioStat(path, "blkio.sectors_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.SectorsRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_service_bytes_recursive"); err != nil {
-		return err
+	var bfqDebugStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
 	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_serviced_recursive"); err != nil {
-		return err
+	var bfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
 	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_queued_recursive"); err != nil {
-		return err
+	var cfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
 	}
-	stats.BlkioStats.IoQueuedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_service_time_recursive"); err != nil {
-		return err
+	var throttleRecursiveStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
 	}
-	stats.BlkioStats.IoServiceTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_wait_time_recursive"); err != nil {
-		return err
+	var baseStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
 	}
-	stats.BlkioStats.IoWaitTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.io_merged_recursive"); err != nil {
-		return err
+	var orderedStats = [][]blkioStatInfo{
+		bfqDebugStats,
+		bfqStats,
+		cfqStats,
+		throttleRecursiveStats,
+		baseStats,
 	}
-	stats.BlkioStats.IoMergedRecursive = blkioStats
 
-	if blkioStats, err = getBlkioStat(path, "blkio.time_recursive"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoTimeRecursive = blkioStats
-
-	return nil
-}
-
-func getStats(path string, stats *cgroups.Stats) error {
 	var blkioStats []cgroups.BlkioStatEntry
 	var err error
 
-	if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_service_bytes"); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(path, "blkio.throttle.io_serviced"); err != nil {
-		return err
+	for _, statGroup := range orderedStats {
+		for i, statInfo := range statGroup {
+			if blkioStats, err = getBlkioStat(path, statInfo.filename); err != nil || blkioStats == nil {
+				// if error occurs on first file, move to next group
+				if i == 0 {
+					break
+				}
+				return err
+			}
+			*statInfo.blkioStatEntriesPtr = blkioStats
+			//finish if all stats are gathered
+			if i == len(statGroup)-1 {
+				return nil
+			}
+		}
 	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
 	return nil
 }


Test patch:
diff --git a/libcontainer/cgroups/fs/blkio_test.go b/libcontainer/cgroups/fs/blkio_test.go
index 5ba60fa7..23564805 100644
--- a/libcontainer/cgroups/fs/blkio_test.go
+++ b/libcontainer/cgroups/fs/blkio_test.go
@@ -3,6 +3,7 @@
 package fs
 
 import (
+	"fmt"
 	"strconv"
 	"testing"
 
@@ -13,43 +14,81 @@ import (
 
 const (
 	sectorsRecursiveContents      = `8:0 1024`
+	sectorsRecursiveContentsBFQ   = `8:0 2048`
 	serviceBytesRecursiveContents = `8:0 Read 100
 8:0 Write 200
 8:0 Sync 300
 8:0 Async 500
 8:0 Total 500
 Total 500`
+
+	serviceBytesRecursiveContentsBFQ = `8:0 Read 1100
+8:0 Write 1200
+8:0 Sync 1300
+8:0 Async 1500
+8:0 Total 1500
+Total 1500`
 	servicedRecursiveContents = `8:0 Read 10
 8:0 Write 40
 8:0 Sync 20
 8:0 Async 30
 8:0 Total 50
 Total 50`
+	servicedRecursiveContentsBFQ = `8:0 Read 11
+8:0 Write 41
+8:0 Sync 21
+8:0 Async 31
+8:0 Total 51
+Total 51`
 	queuedRecursiveContents = `8:0 Read 1
 8:0 Write 4
 8:0 Sync 2
 8:0 Async 3
 8:0 Total 5
 Total 5`
+	queuedRecursiveContentsBFQ = `8:0 Read 2
+8:0 Write 3
+8:0 Sync 4
+8:0 Async 5
+8:0 Total 6
+Total 6`
 	serviceTimeRecursiveContents = `8:0 Read 173959
 8:0 Write 0
 8:0 Sync 0
 8:0 Async 173959
 8:0 Total 17395
 Total 17395`
+	serviceTimeRecursiveContentsBFQ = `8:0 Read 173959
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 173
+8:0 Total 174
+Total 174`
 	waitTimeRecursiveContents = `8:0 Read 15571
 8:0 Write 0
 8:0 Sync 0
 8:0 Async 15571
 8:0 Total 15571`
+	waitTimeRecursiveContentsBFQ = `8:0 Read 1557
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 1557
+8:0 Total 1557`
 	mergedRecursiveContents = `8:0 Read 5
 8:0 Write 10
 8:0 Sync 0
 8:0 Async 0
 8:0 Total 15
 Total 15`
-	timeRecursiveContents = `8:0 8`
-	throttleServiceBytes  = `8:0 Read 11030528
+	mergedRecursiveContentsBFQ = `8:0 Read 51
+8:0 Write 101
+8:0 Sync 0
+8:0 Async 0
+8:0 Total 151
+Total 151`
+	timeRecursiveContents    = `8:0 8`
+	timeRecursiveContentsBFQ = `8:0 16`
+	throttleServiceBytes     = `8:0 Read 11030528
 8:0 Write 23
 8:0 Sync 42
 8:0 Async 11030528
@@ -60,6 +99,17 @@ Total 15`
 252:0 Async 11030528
 252:0 Total 11030528
 Total 22061056`
+	throttleServiceBytesRecursive = `8:0 Read 110305281
+8:0 Write 231
+8:0 Sync 421
+8:0 Async 110305281
+8:0 Total 110305281
+252:0 Read 110305281
+252:0 Write 231
+252:0 Sync 421
+252:0 Async 110305281
+252:0 Total 110305281
+Total 220610561`
 	throttleServiced = `8:0 Read 164
 8:0 Write 23
 8:0 Sync 42
@@ -71,8 +121,51 @@ Total 22061056`
 252:0 Async 164
 252:0 Total 164
 Total 328`
+	throttleServicedRecursive = `8:0 Read 1641
+8:0 Write 231
+8:0 Sync 421
+8:0 Async 1641
+8:0 Total 1641
+252:0 Read 1641
+252:0 Write 231
+252:0 Sync 421
+252:0 Async 1641
+252:0 Total 1641
+Total 3281`
 )
 
+var blkioBFQDebugStatsTestFiles = map[string]string{
+	"blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ,
+	"blkio.bfq.io_serviced_recursive":      servicedRecursiveContentsBFQ,
+	"blkio.bfq.io_queued_recursive":        queuedRecursiveContentsBFQ,
+	"blkio.bfq.io_service_time_recursive":  serviceTimeRecursiveContentsBFQ,
+	"blkio.bfq.io_wait_time_recursive":     waitTimeRecursiveContentsBFQ,
+	"blkio.bfq.io_merged_recursive":        mergedRecursiveContentsBFQ,
+	"blkio.bfq.time_recursive":             timeRecursiveContentsBFQ,
+	"blkio.bfq.sectors_recursive":          sectorsRecursiveContentsBFQ,
+}
+
+var blkioBFQStatsTestFiles = map[string]string{
+	"blkio.bfq.io_service_bytes_recursive": serviceBytesRecursiveContentsBFQ,
+	"blkio.bfq.io_serviced_recursive":      servicedRecursiveContentsBFQ,
+}
+
+var blkioCFQStatsTestFiles = map[string]string{
+	"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+	"blkio.io_serviced_recursive":      servicedRecursiveContents,
+	"blkio.io_queued_recursive":        queuedRecursiveContents,
+	"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+	"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+	"blkio.io_merged_recursive":        mergedRecursiveContents,
+	"blkio.time_recursive":             timeRecursiveContents,
+	"blkio.sectors_recursive":          sectorsRecursiveContents,
+}
+
+type blkioStatFailureTestCase struct {
+	desc     string
+	filename string
+}
+
 func appendBlkioStatEntry(blkioStatEntries *[]cgroups.BlkioStatEntry, major, minor, value uint64, op string) {
 	*blkioStatEntries = append(*blkioStatEntries, cgroups.BlkioStatEntry{Major: major, Minor: minor, Value: value, Op: op})
 }
@@ -174,20 +267,10 @@ func TestBlkioSetMultipleWeightDevice(t *testing.T) {
 	}
 }
 
-func TestBlkioStats(t *testing.T) {
+func TestBlkioBFQDebugStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
-
+	helper.writeFileContents(blkioBFQDebugStatsTestFiles)
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
@@ -195,236 +278,338 @@ func TestBlkioStats(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Verify expected stats.
 	expectedStats := cgroups.BlkioStats{}
-	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 1024, "")
-
-	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 100, "Read")
-	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 200, "Write")
-	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 300, "Sync")
-	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Async")
-	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Total")
-
-	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 10, "Read")
-	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 40, "Write")
-	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 20, "Sync")
-	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 30, "Async")
-	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 50, "Total")
-
-	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 1, "Read")
-	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Write")
-	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Sync")
-	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Async")
-	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Total")
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total")
 
 	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
 	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
 	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
-	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Async")
-	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 17395, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total")
 
-	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read")
 	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
 	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
-	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Async")
-	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Total")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total")
 
-	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 5, "Read")
-	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 10, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write")
 	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
 	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
-	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 15, "Total")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total")
 
-	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 8, "")
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "")
 
 	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }
 
-func TestBlkioStatsNoSectorsFile(t *testing.T) {
+func TestBlkioMultipleStatsFiles(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
-
+	helper.writeFileContents(blkioBFQDebugStatsTestFiles)
+	helper.writeFileContents(blkioCFQStatsTestFiles)
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
 	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+		t.Fatal(err)
 	}
-}
 
-func TestBlkioStatsNoServiceBytesFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_serviced_recursive":     servicedRecursiveContents,
-		"blkio.io_queued_recursive":       queuedRecursiveContents,
-		"blkio.sectors_recursive":         sectorsRecursiveContents,
-		"blkio.io_service_time_recursive": serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":    waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":       mergedRecursiveContents,
-		"blkio.time_recursive":            timeRecursiveContents,
-	})
+	expectedStats := cgroups.BlkioStats{}
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 2048, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 6, "Total")
 
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
-}
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 174, "Total")
 
-func TestBlkioStatsNoServicedFile(t *testing.T) {
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 1557, "Total")
 
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
-	}
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 51, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 101, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 151, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 16, "")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }
 
-func TestBlkioStatsNoQueuedFile(t *testing.T) {
+func TestBlkioBFQStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-	})
-
+	helper.writeFileContents(blkioBFQStatsTestFiles)
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
 	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+		t.Fatal(err)
 	}
+
+	expectedStats := cgroups.BlkioStats{}
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 1500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 11, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 41, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 21, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 31, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 51, "Total")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }
 
-func TestBlkioStatsNoServiceTimeFile(t *testing.T) {
+func TestBlkioStatsNoFilesBFQDebug(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+	testCases := []blkioStatFailureTestCase{
+		{
+			desc:     "missing blkio.bfq.io_service_bytes_recursive file",
+			filename: "blkio.bfq.io_service_bytes_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_serviced_recursive file",
+			filename: "blkio.bfq.io_serviced_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_queued_recursive file",
+			filename: "blkio.bfq.io_queued_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.sectors_recursive file",
+			filename: "blkio.bfq.sectors_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_service_time_recursive file",
+			filename: "blkio.bfq.io_service_time_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_wait_time_recursive file",
+			filename: "blkio.bfq.io_wait_time_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.io_merged_recursive file",
+			filename: "blkio.bfq.io_merged_recursive",
+		},
+		{
+			desc:     "missing blkio.bfq.time_recursive file",
+			filename: "blkio.bfq.time_recursive",
+		},
+	}
+
+	for _, testCase := range testCases {
+		helper := NewCgroupTestUtil("cpuset", t)
+		defer helper.cleanup()
+
+		tempBlkioTestFiles := map[string]string{}
+		for i, v := range blkioBFQDebugStatsTestFiles {
+			tempBlkioTestFiles[i] = v
+		}
+		delete(tempBlkioTestFiles, testCase.filename)
+
+		helper.writeFileContents(tempBlkioTestFiles)
+		cpuset := &CpusetGroup{}
+		actualStats := *cgroups.NewStats()
+		err := cpuset.GetStats(helper.CgroupPath, &actualStats)
+
+		if err != nil {
+			t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err))
+		}
 	}
 }
 
-func TestBlkioStatsNoWaitTimeFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
+func TestBlkioCFQStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
+	helper.writeFileContents(blkioCFQStatsTestFiles)
 
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
 	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+		t.Fatal(err)
 	}
+
+	// Verify expected stats.
+	expectedStats := cgroups.BlkioStats{}
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 1024, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 10, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 40, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 20, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 30, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 50, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 1, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 17395, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 5, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 10, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 15, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 8, "")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }
 
-func TestBlkioStatsNoMergedFile(t *testing.T) {
+func TestBlkioStatsNoFilesCFQ(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode.")
 	}
-	helper := NewCgroupTestUtil("blkio", t)
-	defer helper.cleanup()
-	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-	})
-
-	blkio := &BlkioGroup{}
-	actualStats := *cgroups.NewStats()
-	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+	testCases := []blkioStatFailureTestCase{
+		{
+			desc:     "missing blkio.io_service_bytes_recursive file",
+			filename: "blkio.io_service_bytes_recursive",
+		},
+		{
+			desc:     "missing blkio.io_serviced_recursive file",
+			filename: "blkio.io_serviced_recursive",
+		},
+		{
+			desc:     "missing blkio.io_queued_recursive file",
+			filename: "blkio.io_queued_recursive",
+		},
+		{
+			desc:     "missing blkio.sectors_recursive file",
+			filename: "blkio.sectors_recursive",
+		},
+		{
+			desc:     "missing blkio.io_service_time_recursive file",
+			filename: "blkio.io_service_time_recursive",
+		},
+		{
+			desc:     "missing blkio.io_wait_time_recursive file",
+			filename: "blkio.io_wait_time_recursive",
+		},
+		{
+			desc:     "missing blkio.io_merged_recursive file",
+			filename: "blkio.io_merged_recursive",
+		},
+		{
+			desc:     "missing blkio.time_recursive file",
+			filename: "blkio.time_recursive",
+		},
+	}
+
+	for _, testCase := range testCases {
+		helper := NewCgroupTestUtil("cpuset", t)
+		defer helper.cleanup()
+
+		tempBlkioTestFiles := map[string]string{}
+		for i, v := range blkioCFQStatsTestFiles {
+			tempBlkioTestFiles[i] = v
+		}
+		delete(tempBlkioTestFiles, testCase.filename)
+
+		helper.writeFileContents(tempBlkioTestFiles)
+		cpuset := &CpusetGroup{}
+		actualStats := *cgroups.NewStats()
+		err := cpuset.GetStats(helper.CgroupPath, &actualStats)
+
+		if err != nil {
+			t.Errorf(fmt.Sprintf("test case '%s' failed unexpectedly: %s", testCase.desc, err))
+		}
 	}
 }
 
-func TestBlkioStatsNoTimeFile(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
+func TestBlkioStatsUnexpectedNumberOfFields(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
 	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_service_bytes_recursive": "8:0 Read 100 100",
 		"blkio.io_serviced_recursive":      servicedRecursiveContents,
 		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
 		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
 		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
 		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
 	})
 
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err != nil {
-		t.Fatalf("Failed unexpectedly: %s", err)
+	if err == nil {
+		t.Fatal("Expected to fail, but did not")
 	}
 }
 
-func TestBlkioStatsUnexpectedNumberOfFields(t *testing.T) {
+func TestBlkioStatsUnexpectedFieldType(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
 	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": "8:0 Read 100 100",
+		"blkio.io_service_bytes_recursive": "8:0 Read Write",
 		"blkio.io_serviced_recursive":      servicedRecursiveContents,
 		"blkio.io_queued_recursive":        queuedRecursiveContents,
 		"blkio.sectors_recursive":          sectorsRecursiveContents,
@@ -442,29 +627,58 @@ func TestBlkioStatsUnexpectedNumberOfFields(t *testing.T) {
 	}
 }
 
-func TestBlkioStatsUnexpectedFieldType(t *testing.T) {
+func TestThrottleRecursiveBlkioStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
 	helper.writeFileContents(map[string]string{
-		"blkio.io_service_bytes_recursive": "8:0 Read Write",
-		"blkio.io_serviced_recursive":      servicedRecursiveContents,
-		"blkio.io_queued_recursive":        queuedRecursiveContents,
-		"blkio.sectors_recursive":          sectorsRecursiveContents,
-		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
-		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
-		"blkio.io_merged_recursive":        mergedRecursiveContents,
-		"blkio.time_recursive":             timeRecursiveContents,
+		"blkio.io_service_bytes_recursive":          "",
+		"blkio.io_serviced_recursive":               "",
+		"blkio.io_queued_recursive":                 "",
+		"blkio.sectors_recursive":                   "",
+		"blkio.io_service_time_recursive":           "",
+		"blkio.io_wait_time_recursive":              "",
+		"blkio.io_merged_recursive":                 "",
+		"blkio.time_recursive":                      "",
+		"blkio.throttle.io_service_bytes_recursive": throttleServiceBytesRecursive,
+		"blkio.throttle.io_serviced_recursive":      throttleServicedRecursive,
 	})
 
 	blkio := &BlkioGroup{}
 	actualStats := *cgroups.NewStats()
 	err := blkio.GetStats(helper.CgroupPath, &actualStats)
-	if err == nil {
-		t.Fatal("Expected to fail, but did not")
+	if err != nil {
+		t.Fatal(err)
 	}
+
+	// Verify expected stats.
+	expectedStats := cgroups.BlkioStats{}
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 110305281, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 110305281, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 1641, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 231, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 421, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 1641, "Total")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
 }
 
-func TestNonCFQBlkioStats(t *testing.T) {
+func TestThrottleBlkioStats(t *testing.T) {
 	helper := NewCgroupTestUtil("blkio", t)
 	defer helper.cleanup()
 	helper.writeFileContents(map[string]string{