func (n *node) GetMemoryType() MemoryType {
	return n.memoryType
}
GetMemoryType returns the memory type for this node.
package sysfs
import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
	logger "github.com/intel/cri-resource-manager/pkg/log"
	"github.com/intel/cri-resource-manager/pkg/utils"
	"github.com/intel/goresctrl/pkg/sst"
	idset "github.com/intel/goresctrl/pkg/utils"
)
const (
	SysfsRootPath = "/sys"
	sysfsCPUPath = "devices/system/cpu"
	sysfsNumaNodePath = "devices/system/node"
)
type DiscoveryFlag uint
const (
	DiscoverCPUTopology DiscoveryFlag = 1 << iota
	DiscoverMemTopology
	DiscoverCache
	DiscoverSst
	DiscoverNone DiscoveryFlag = 0
	DiscoverAll DiscoveryFlag = 0xffffffff
	DiscoverDefault DiscoveryFlag = (DiscoverCPUTopology | DiscoverMemTopology | DiscoverSst)
)
type MemoryType int
const (
	MemoryTypeDRAM MemoryType = iota
	MemoryTypePMEM
	MemoryTypeHBM
)
type System interface {
	Discover(flags DiscoveryFlag) error
	SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error)
	SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error
	PackageIDs() []idset.ID
	NodeIDs() []idset.ID
	CPUIDs() []idset.ID
	PackageCount() int
	SocketCount() int
	CPUCount() int
	NUMANodeCount() int
	ThreadCount() int
	CPUSet() cpuset.CPUSet
	Package(id idset.ID) CPUPackage
	Node(id idset.ID) Node
	NodeDistance(from, to idset.ID) int
	CPU(id idset.ID) CPU
	Offlined() cpuset.CPUSet
	Isolated() cpuset.CPUSet
}
type system struct {
	logger.Logger                           
	flags         DiscoveryFlag             
	path          string                    
	packages      map[idset.ID]*cpuPackage  
	nodes         map[idset.ID]*node        
	cpus          map[idset.ID]*cpu         
	cache         map[idset.ID]*Cache       
	offline       idset.IDSet               
	isolated      idset.IDSet               
	threads       int                       
}
type CPUPackage interface {
	ID() idset.ID
	CPUSet() cpuset.CPUSet
	DieIDs() []idset.ID
	NodeIDs() []idset.ID
	DieNodeIDs(idset.ID) []idset.ID
	DieCPUSet(idset.ID) cpuset.CPUSet
	SstInfo() sst.SstPackageInfo
}
type cpuPackage struct {
	id       idset.ID                  
	cpus     idset.IDSet               
	nodes    idset.IDSet               
	dies     idset.IDSet               
	dieCPUs  map[idset.ID]idset.IDSet  
	dieNodes map[idset.ID]idset.IDSet  
	sstInfo  sst.SstPackageInfo        
}
type Node interface {
	ID() idset.ID
	PackageID() idset.ID
	DieID() idset.ID
	CPUSet() cpuset.CPUSet
	Distance() []int
	DistanceFrom(id idset.ID) int
	MemoryInfo() (*MemInfo, error)
	GetMemoryType() MemoryType
	HasNormalMemory() bool
}
type node struct {
	path       string       
	id         idset.ID     
	pkg        idset.ID     
	die        idset.ID     
	cpus       idset.IDSet  
	memoryType MemoryType   
	normalMem  bool         
	distance   []int        
}
type CPU interface {
	ID() idset.ID
	PackageID() idset.ID
	DieID() idset.ID
	NodeID() idset.ID
	CoreID() idset.ID
	ThreadCPUSet() cpuset.CPUSet
	BaseFrequency() uint64
	FrequencyRange() CPUFreq
	EPP() EPP
	Online() bool
	Isolated() bool
	SetFrequencyLimits(min, max uint64) error
	SstClos() int
}
type cpu struct {
	path     string       
	id       idset.ID     
	pkg      idset.ID     
	die      idset.ID     
	node     idset.ID     
	core     idset.ID     
	threads  idset.IDSet  
	baseFreq uint64       
	freq     CPUFreq      
	epp      EPP          
	online   bool         
	isolated bool         
	sstClos  int          
}
type CPUFreq struct {
	min uint64    
	max uint64    
	all []uint64  
}
type EPP int
const (
	EPPPerformance EPP = iota
	EPPBalancePerformance
	EPPBalancePower
	EPPPower
	EPPUnknown
)
type MemInfo struct {
	MemTotal uint64
	MemFree  uint64
	MemUsed  uint64
}
type CacheType string
const (
	DataCache CacheType = "Data"
	InstructionCache CacheType = "Instruction"
	UnifiedCache CacheType = "Unified"
)
type Cache struct {
	id    idset.ID     
	kind  CacheType    
	size  uint64       
	level uint8        
	cpus  idset.IDSet  
}
func DiscoverSystem(args ...DiscoveryFlag) (System, error) {
	return DiscoverSystemAt(SysfsRootPath, args...)
}
func DiscoverSystemAt(path string, args ...DiscoveryFlag) (System, error) {
	var flags DiscoveryFlag
	if len(args) < 1 {
		flags = DiscoverDefault
	} else {
		flags = DiscoverNone
		for _, flag := range args {
			flags |= flag
		}
	}
	sys := &system{
		Logger:  logger.NewLogger("sysfs"),
		path:    path,
		offline: idset.NewIDSet(),
	}
	if err := sys.Discover(flags); err != nil {
		return nil, err
	}
	return sys, nil
}
func (sys *system) Discover(flags DiscoveryFlag) error {
	sys.flags |= (flags &^ DiscoverCache)
	if (sys.flags & (DiscoverCPUTopology | DiscoverCache | DiscoverSst)) != 0 {
		if err := sys.discoverCPUs(); err != nil {
			return err
		}
		if err := sys.discoverNodes(); err != nil {
			return err
		}
		if err := sys.discoverPackages(); err != nil {
			return err
		}
	}
	if (sys.flags & DiscoverSst) != 0 {
		if err := sys.discoverSst(); err != nil {
			sys.Warn("%v", err)
		}
	}
	if (sys.flags & DiscoverMemTopology) != 0 {
		if err := sys.discoverNodes(); err != nil {
			return err
		}
	}
	if len(sys.nodes) > 0 {
		for _, pkg := range sys.packages {
			for _, nodeID := range pkg.nodes.SortedMembers() {
				if node, ok := sys.nodes[nodeID]; ok {
					node.pkg = pkg.id
				} else {
					return sysfsError("NUMA nodes", "can't find NUMA node for ID %d", nodeID)
				}
			}
			for _, dieID := range pkg.DieIDs() {
				for _, nodeID := range pkg.DieNodeIDs(dieID) {
					if node, ok := sys.nodes[nodeID]; ok {
						node.die = dieID
					} else {
						return sysfsError("NUMA nodes", "can't find NUMA node for ID %d", nodeID)
					}
				}
			}
		}
	}
	if sys.DebugEnabled() {
		for id, pkg := range sys.packages {
			sys.Info("package #%d:", id)
			sys.Debug("   cpus: %s", pkg.cpus)
			sys.Debug("  nodes: %s", pkg.nodes)
			sys.Debug("   dies: %s", pkg.dies)
			for die := range pkg.dies {
				sys.Debug("    die #%v nodes: %v", die, pkg.DieNodeIDs(die))
				sys.Debug("    die #%v cpus: %s", die, pkg.DieCPUSet(die).String())
			}
		}
		for id, node := range sys.nodes {
			sys.Debug("node #%d:", id)
			sys.Debug("      cpus: %s", node.cpus)
			sys.Debug("  distance: %v", node.distance)
			sys.Debug("   package: #%d", node.pkg)
			sys.Debug("       die: #%d", node.die)
		}
		for id, cpu := range sys.cpus {
			sys.Debug("CPU #%d:", id)
			sys.Debug("        pkg: %d", cpu.pkg)
			sys.Debug("        die: %d", cpu.die)
			sys.Debug("       node: %d", cpu.node)
			sys.Debug("       core: %d", cpu.core)
			sys.Debug("    threads: %s", cpu.threads)
			sys.Debug("  base freq: %d", cpu.baseFreq)
			sys.Debug("       freq: %d - %d", cpu.freq.min, cpu.freq.max)
			sys.Debug("        epp: %d", cpu.epp)
		}
		sys.Debug("offline CPUs: %s", sys.offline)
		sys.Debug("isolated CPUs: %s", sys.isolated)
		for id, cch := range sys.cache {
			sys.Debug("cache #%d:", id)
			sys.Debug("   type: %v", cch.kind)
			sys.Debug("   size: %d", cch.size)
			sys.Debug("  level: %d", cch.level)
			sys.Debug("   CPUs: %s", cch.cpus)
		}
	}
	return nil
}
func (sys *system) SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) {
	var entries []string
	if cpus == nil {
		entries, _ = filepath.Glob(filepath.Join(sys.path, sysfsCPUPath, "cpu[0-9]*"))
	} else {
		entries = make([]string, cpus.Size())
		for idx, id := range cpus.Members() {
			entries[idx] = sys.path + "/" + sysfsCPUPath + "/cpu" + strconv.Itoa(int(id))
		}
	}
	desired := map[bool]int{false: 0, true: 1}[online]
	changed := idset.NewIDSet()
	for _, entry := range entries {
		var current int
		id := getEnumeratedID(entry)
		if id <= 0 {
			continue
		}
		if _, err := writeSysfsEntry(entry, "online", desired, &current); err != nil {
			return nil, sysfsError(entry, "failed to set online to %d: %v", desired, err)
		}
		if desired != current {
			changed.Add(id)
			if cpu, found := sys.cpus[id]; found {
				cpu.online = online
				if online {
					sys.offline.Del(id)
				} else {
					sys.offline.Add(id)
				}
			}
		}
	}
	return changed, nil
}
func (sys *system) SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error {
	if cpus == nil {
		cpus = idset.NewIDSet(sys.CPUIDs()...)
	}
	for _, id := range cpus.Members() {
		if cpu, ok := sys.cpus[id]; ok {
			if err := cpu.SetFrequencyLimits(min, max); err != nil {
				return err
			}
		}
	}
	return nil
}
func (sys *system) PackageIDs() []idset.ID {
	ids := make([]idset.ID, len(sys.packages))
	idx := 0
	for id := range sys.packages {
		ids[idx] = id
		idx++
	}
	sort.Slice(ids, func(i, j int) bool {
		return int(ids[i]) < int(ids[j])
	})
	return ids
}
func (sys *system) NodeIDs() []idset.ID {
	ids := make([]idset.ID, len(sys.nodes))
	idx := 0
	for id := range sys.nodes {
		ids[idx] = id
		idx++
	}
	sort.Slice(ids, func(i, j int) bool {
		return int(ids[i]) < int(ids[j])
	})
	return ids
}
func (sys *system) CPUIDs() []idset.ID {
	ids := make([]idset.ID, len(sys.cpus))
	idx := 0
	for id := range sys.cpus {
		ids[idx] = id
		idx++
	}
	sort.Slice(ids, func(i, j int) bool {
		return int(ids[i]) < int(ids[j])
	})
	return ids
}
func (sys *system) PackageCount() int {
	return len(sys.packages)
}
func (sys *system) SocketCount() int {
	return len(sys.packages)
}
func (sys *system) CPUCount() int {
	return len(sys.cpus)
}
func (sys *system) NUMANodeCount() int {
	cnt := len(sys.nodes)
	if cnt < 1 {
		cnt = 1
	}
	return cnt
}
func (sys *system) ThreadCount() int {
	return sys.threads
}
func (sys *system) CPUSet() cpuset.CPUSet {
	return CPUSetFromIDSet(idset.NewIDSet(sys.CPUIDs()...))
}
func (sys *system) Package(id idset.ID) CPUPackage {
	return sys.packages[id]
}
func (sys *system) Node(id idset.ID) Node {
	return sys.nodes[id]
}
func (sys *system) NodeDistance(from, to idset.ID) int {
	return sys.nodes[from].DistanceFrom(to)
}
func (sys *system) CPU(id idset.ID) CPU {
	return sys.cpus[id]
}
func (sys *system) Offlined() cpuset.CPUSet {
	return CPUSetFromIDSet(sys.offline)
}
func (sys *system) Isolated() cpuset.CPUSet {
	return CPUSetFromIDSet(sys.isolated)
}
func (sys *system) discoverCPUs() error {
	if sys.cpus != nil {
		return nil
	}
	sys.cpus = make(map[idset.ID]*cpu)
	_, err := readSysfsEntry(sys.path, filepath.Join(sysfsCPUPath, "isolated"), &sys.isolated, ",")
	if err != nil {
		sys.Error("failed to get set of isolated cpus: %v", err)
	}
	entries, _ := filepath.Glob(filepath.Join(sys.path, sysfsCPUPath, "cpu[0-9]*"))
	for _, entry := range entries {
		if err := sys.discoverCPU(entry); err != nil {
			return fmt.Errorf("failed to discover cpu for entry %s: %v", entry, err)
		}
	}
	return nil
}
func (sys *system) discoverCPU(path string) error {
	cpu := &cpu{path: path, id: getEnumeratedID(path), online: true, sstClos: -1}
	cpu.isolated = sys.isolated.Has(cpu.id)
	if online, err := readSysfsEntry(path, "online", nil); err == nil {
		cpu.online = (online != "" && online[0] != '0')
	}
	if cpu.online {
		if _, err := readSysfsEntry(path, "topology/physical_package_id", &cpu.pkg); err != nil {
			return err
		}
		readSysfsEntry(path, "topology/die_id", &cpu.die)
		if _, err := readSysfsEntry(path, "topology/core_id", &cpu.core); err != nil {
			return err
		}
		if _, err := readSysfsEntry(path, "topology/thread_siblings_list", &cpu.threads, ","); err != nil {
			return err
		}
	} else {
		sys.offline.Add(cpu.id)
	}
	if _, err := readSysfsEntry(path, "cpufreq/base_frequency", &cpu.baseFreq); err != nil {
		cpu.baseFreq = 0
	}
	if _, err := readSysfsEntry(path, "cpufreq/cpuinfo_min_freq", &cpu.freq.min); err != nil {
		cpu.freq.min = 0
	}
	if _, err := readSysfsEntry(path, "cpufreq/cpuinfo_max_freq", &cpu.freq.max); err != nil {
		cpu.freq.max = 0
	}
	if _, err := readSysfsEntry(path, "cpufreq/energy_performance_preference", &cpu.epp); err != nil {
		cpu.epp = EPPUnknown
	}
	if node, _ := filepath.Glob(filepath.Join(path, "node[0-9]*")); len(node) == 1 {
		cpu.node = getEnumeratedID(node[0])
	} else {
		return fmt.Errorf("exactly one node per cpu allowed")
	}
	if sys.threads < 1 {
		sys.threads = 1
	}
	if cpu.threads.Size() > sys.threads {
		sys.threads = cpu.threads.Size()
	}
	sys.cpus[cpu.id] = cpu
	if (sys.flags & DiscoverCache) != 0 {
		entries, _ := filepath.Glob(filepath.Join(path, "cache/index[0-9]*"))
		for _, entry := range entries {
			if err := sys.discoverCache(entry); err != nil {
				return err
			}
		}
	}
	return nil
}
func (c *cpu) ID() idset.ID {
	return c.id
}
func (c *cpu) PackageID() idset.ID {
	return c.pkg
}
func (c *cpu) DieID() idset.ID {
	return c.die
}
func (c *cpu) NodeID() idset.ID {
	return c.node
}
func (c *cpu) CoreID() idset.ID {
	return c.core
}
func (c *cpu) ThreadCPUSet() cpuset.CPUSet {
	return CPUSetFromIDSet(c.threads)
}
func (c *cpu) BaseFrequency() uint64 {
	return c.baseFreq
}
func (c *cpu) FrequencyRange() CPUFreq {
	return c.freq
}
func (c *cpu) EPP() EPP {
	return c.epp
}
func (c *cpu) Online() bool {
	return c.online
}
func (c *cpu) Isolated() bool {
	return c.isolated
}
func (c *cpu) SstClos() int {
	return c.sstClos
}
func (c *cpu) SetFrequencyLimits(min, max uint64) error {
	if c.freq.min == 0 {
		return nil
	}
	min /= 1000
	max /= 1000
	if min < c.freq.min && min != 0 {
		min = c.freq.min
	}
	if min > c.freq.max {
		min = c.freq.max
	}
	if max < c.freq.min && max != 0 {
		max = c.freq.min
	}
	if max > c.freq.max {
		max = c.freq.max
	}
	if _, err := writeSysfsEntry(c.path, "cpufreq/scaling_min_freq", min, nil); err != nil {
		return err
	}
	if _, err := writeSysfsEntry(c.path, "cpufreq/scaling_max_freq", max, nil); err != nil {
		return err
	}
	return nil
}
func readCPUsetFile(base, entry string) (cpuset.CPUSet, error) {
	path := filepath.Join(base, entry)
	blob, err := ioutil.ReadFile(path)
	if err != nil {
		return cpuset.NewCPUSet(), sysfsError(path, "failed to read sysfs entry: %v", err)
	}
	return cpuset.Parse(strings.Trim(string(blob), "\n"))
}
func (sys *system) discoverNodes() error {
	if sys.nodes != nil {
		return nil
	}
	sysNodesPath := filepath.Join(sys.path, sysfsNumaNodePath)
	sys.nodes = make(map[idset.ID]*node)
	entries, _ := filepath.Glob(filepath.Join(sysNodesPath, "node[0-9]*"))
	for _, entry := range entries {
		if err := sys.discoverNode(entry); err != nil {
			return fmt.Errorf("failed to discover node for entry %s: %v", entry, err)
		}
	}
	normalMemNodeIDs, err := readSysfsEntry(sysNodesPath, "has_normal_memory", nil)
	if err != nil {
		return fmt.Errorf("failed to discover nodes with normal memory: %v", err)
	}
	normalMemNodes, err := cpuset.Parse(normalMemNodeIDs)
	if err != nil {
		return fmt.Errorf("failed to parse nodes with normal memory (%q): %v",
			normalMemNodes, err)
	}
	memoryNodeIDs, err := readSysfsEntry(sysNodesPath, "has_memory", nil)
	if err != nil {
		return fmt.Errorf("failed to discover nodes with memory: %v", err)
	}
	memoryNodes, err := cpuset.Parse(memoryNodeIDs)
	if err != nil {
		return fmt.Errorf("failed to parse nodes with memory (%q): %v",
			memoryNodeIDs, err)
	}
	cpuNodesBuilder := cpuset.NewBuilder()
	for id, node := range sys.nodes {
		if node.cpus.Size() > 0 {
			cpuNodesBuilder.Add(int(id))
		}
		if normalMemNodes.Contains(int(id)) {
			node.normalMem = true
		}
	}
	cpuNodes := cpuNodesBuilder.Result()
	sys.Logger.Info("NUMA nodes with CPUs: %s", cpuNodes.String())
	sys.Logger.Info("NUMA nodes with (any) memory: %s", memoryNodes.String())
	sys.Logger.Info("NUMA nodes with normal memory: %s", normalMemNodes.String())
	dramNodes := memoryNodes.Intersection(cpuNodes)
	pmemOrHbmNodes := memoryNodes.Difference(dramNodes)
	dramNodeIds := IDSetFromCPUSet(dramNodes)
	pmemOrHbmNodeIds := IDSetFromCPUSet(pmemOrHbmNodes)
	infos := make(map[idset.ID]*MemInfo)
	dramAvg := uint64(0)
	if len(pmemOrHbmNodeIds) > 0 && len(dramNodeIds) > 0 {
		dramTotal := uint64(0)
		for _, node := range sys.nodes {
			info, err := node.MemoryInfo()
			if err != nil {
				return fmt.Errorf("failed to get memory info for node %v: %s", node, err)
			}
			infos[node.id] = info
			if _, ok := dramNodeIds[node.id]; ok {
				dramTotal += info.MemTotal
			}
		}
		dramAvg = dramTotal / uint64(len(dramNodeIds))
		if dramAvg == 0 {
			return fmt.Errorf("no dram in the system, cannot determine special memory types")
		}
	}
	for _, node := range sys.nodes {
		if _, ok := pmemOrHbmNodeIds[node.id]; ok {
			mem, ok := infos[node.id]
			if !ok {
				return fmt.Errorf("not able to determine system special memory types")
			}
			if mem.MemTotal < dramAvg {
				sys.Logger.Info("node %d has HBM memory", node.id)
				node.memoryType = MemoryTypeHBM
			} else {
				sys.Logger.Info("node %d has PMEM memory", node.id)
				node.memoryType = MemoryTypePMEM
			}
		} else if _, ok := dramNodeIds[node.id]; ok {
			sys.Logger.Info("node %d has DRAM memory", node.id)
			node.memoryType = MemoryTypeDRAM
		} else {
			return fmt.Errorf("Unknown memory type for node %v (pmem nodes: %s, dram nodes: %s)", node, pmemOrHbmNodes, dramNodes)
		}
	}
	return nil
}
func (sys *system) discoverNode(path string) error {
	node := &node{path: path, id: getEnumeratedID(path)}
	if _, err := readSysfsEntry(path, "cpulist", &node.cpus, ","); err != nil {
		return err
	}
	if _, err := readSysfsEntry(path, "distance", &node.distance); err != nil {
		return err
	}
	sys.nodes[node.id] = node
	return nil
}
func (n *node) ID() idset.ID {
	return n.id
}
func (n *node) PackageID() idset.ID {
	return n.pkg
}
func (n *node) DieID() idset.ID {
	return n.die
}
func (n *node) CPUSet() cpuset.CPUSet {
	return CPUSetFromIDSet(n.cpus)
}
func (n *node) Distance() []int {
	return n.distance
}
func (n *node) DistanceFrom(id idset.ID) int {
	if int(id) < len(n.distance) {
		return n.distance[int(id)]
	}
	return -1
}
func (n *node) MemoryInfo() (*MemInfo, error) {
	meminfo := filepath.Join(n.path, "meminfo")
	buf := &MemInfo{}
	err := ParseFileEntries(meminfo,
		map[string]interface{}{
			"MemTotal:": &buf.MemTotal,
			"MemFree:":  &buf.MemFree,
			"MemUsed:":  &buf.MemUsed,
		},
		func(line string) (string, string, error) {
			fields := strings.Fields(strings.TrimSpace(line))
			if len(fields) < 4 {
				return "", "", sysfsError(meminfo, "failed to parse entry: '%s'", line)
			}
			key := fields[2]
			val := fields[3]
			if len(fields) == 5 {
				val += " " + fields[4]
			}
			return key, val, nil
		},
	)
	if err != nil {
		return nil, err
	}
	return buf, nil
}