Repository: bacalhau-project/bacalhau (language: python)

Issue:
Bacalhau only supports the use of 1 GPU
Only 1 GPU can be used at a time: https://github.com/filecoin-project/bacalhau/blob/dev/pkg/executor/docker/executor.go#L225
The [Docker API expects](https://docs.docker.com/config/containers/resource_constraints/#expose-gpus-for-use) you to specify the exact GPU you want to run on by ID or by device number.

At the time of writing this code I didn’t know how to or whether we wanted to support maintaining the state of what GPUs are in use by that particular node. Maybe there’s a way now, but that was considered future work. Hence the todo.

In summary, only GPU 0 can be used and the docs are correct.

In the future, we need to support both running multiple jobs on multiple GPUs and single jobs on multiple GPUs. But this is very tied to how GPU jobs are specified and state is maintained so there's probably some dependent work that is required first. See the main GPU improvement issue for this.


Discussion / hints:
We need this to support compute providers who are showing up with multi-GPU servers

Fix patch:
diff --git a/cmd/cli/config/auto.go b/cmd/cli/config/auto.go
index 8136fc49cf..02eb4885bd 100644
--- a/cmd/cli/config/auto.go
+++ b/cmd/cli/config/auto.go
@@ -1,6 +1,7 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 
@@ -97,7 +98,7 @@ func newAutoResourceCmd() *cobra.Command {
 			if err := settings.validate(); err != nil {
 				return err
 			}
-			return autoConfig(settings)
+			return autoConfig(cmd.Context(), settings)
 		},
 	}
 
@@ -106,9 +107,9 @@ func newAutoResourceCmd() *cobra.Command {
 	return autoCmd
 }
 
-func autoConfig(settings *autoSettings) error {
+func autoConfig(ctx context.Context, settings *autoSettings) error {
 	pp := system.NewPhysicalCapacityProvider()
-	physicalResources, err := pp.GetTotalCapacity()
+	physicalResources, err := pp.GetTotalCapacity(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to calculate system physical resources: %w", err)
 	}
diff --git a/cmd/cli/serve/serve.go b/cmd/cli/serve/serve.go
index 5b36a69928..78cbef8f2c 100644
--- a/cmd/cli/serve/serve.go
+++ b/cmd/cli/serve/serve.go
@@ -1,6 +1,7 @@
 package serve
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
@@ -433,10 +434,11 @@ func AutoOutputLabels() map[string]string {
 	arch := runtime.GOARCH
 	m["Architecture"] = arch
 
-	gpus, err := system_capacity.GetSystemGPUs()
+	provider := system_capacity.NewPhysicalCapacityProvider()
+	resources, err := provider.GetAvailableCapacity(context.Background())
 	if err == nil {
 		// Print the GPU names
-		for i, gpu := range gpus {
+		for i, gpu := range resources.GPUs {
 			// Model label e.g. GPU-0: Tesla-T1
 			key := fmt.Sprintf("GPU-%d", gpu.Index)
 			name := strings.Replace(gpu.Name, " ", "-", -1) // Replace spaces with dashes
diff --git a/cmd/cli/serve/util.go b/cmd/cli/serve/util.go
index 3d3288c1fe..720c89c3b8 100644
--- a/cmd/cli/serve/util.go
+++ b/cmd/cli/serve/util.go
@@ -8,6 +8,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"go.uber.org/multierr"
 
 	"github.com/bacalhau-project/bacalhau/cmd/util/flags/configflags"
 	"github.com/bacalhau-project/bacalhau/pkg/orchestrator/transformer"
@@ -18,8 +19,6 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/ipfs"
 	bac_libp2p "github.com/bacalhau-project/bacalhau/pkg/libp2p"
 	"github.com/bacalhau-project/bacalhau/pkg/libp2p/rcmgr"
-	"github.com/bacalhau-project/bacalhau/pkg/model"
-	"github.com/bacalhau-project/bacalhau/pkg/models"
 	"github.com/bacalhau-project/bacalhau/pkg/node"
 	"github.com/bacalhau-project/bacalhau/pkg/system"
 )
@@ -29,27 +28,20 @@ func GetComputeConfig() (node.ComputeConfig, error) {
 	if err := config.ForKey(types.NodeCompute, &cfg); err != nil {
 		return node.ComputeConfig{}, err
 	}
-	total, err := model.ParseResourceUsageConfig(cfg.Capacity.TotalResourceLimits)
-	if err != nil {
-		return node.ComputeConfig{}, err
-	}
-	queue, err := model.ParseResourceUsageConfig(cfg.Capacity.QueueResourceLimits)
-	if err != nil {
-		return node.ComputeConfig{}, err
-	}
-	job, err := model.ParseResourceUsageConfig(cfg.Capacity.JobResourceLimits)
-	if err != nil {
-		return node.ComputeConfig{}, err
-	}
-	def, err := model.ParseResourceUsageConfig(cfg.Capacity.DefaultJobResourceLimits)
-	if err != nil {
+
+	totalResources, totalErr := cfg.Capacity.TotalResourceLimits.ToResources()
+	queueResources, queueErr := cfg.Capacity.QueueResourceLimits.ToResources()
+	jobResources, jobErr := cfg.Capacity.JobResourceLimits.ToResources()
+	defaultResources, defaultErr := cfg.Capacity.DefaultJobResourceLimits.ToResources()
+	if err := multierr.Combine(totalErr, queueErr, jobErr, defaultErr); err != nil {
 		return node.ComputeConfig{}, err
 	}
+
 	return node.NewComputeConfigWith(node.ComputeConfigParams{
-		TotalResourceLimits:                   models.Resources(total),
-		QueueResourceLimits:                   models.Resources(queue),
-		JobResourceLimits:                     models.Resources(job),
-		DefaultJobResourceLimits:              models.Resources(def),
+		TotalResourceLimits:                   *totalResources,
+		QueueResourceLimits:                   *queueResources,
+		JobResourceLimits:                     *jobResources,
+		DefaultJobResourceLimits:              *defaultResources,
 		IgnorePhysicalResourceLimits:          cfg.Capacity.IgnorePhysicalResourceLimits,
 		JobNegotiationTimeout:                 time.Duration(cfg.JobTimeouts.JobNegotiationTimeout),
 		MinJobExecutionTimeout:                time.Duration(cfg.JobTimeouts.MinJobExecutionTimeout),
diff --git a/pkg/compute/capacity/system/gpu/amd.go b/pkg/compute/capacity/system/gpu/amd.go
new file mode 100644
index 0000000000..837de5cf53
--- /dev/null
+++ b/pkg/compute/capacity/system/gpu/amd.go
@@ -0,0 +1,65 @@
+package gpu
+
+import (
+	"encoding/json"
+	"io"
+	"strconv"
+
+	"github.com/bacalhau-project/bacalhau/pkg/compute/capacity"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+)
+
+const rocmCommand = "rocm-smi"
+const bytesPerMebibyte = 1048576
+
+var rocmArgs = []string{"--showproductname", "--showmeminfo", "vram", "--json"}
+
+// {"card0": {"VRAM Total Memory (B)": "68702699520", "VRAM Total Used Memory
+// (B)": "10960896", "Card series": "Instinct MI210", "Card model": "0x0c34",
+// "Card vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU":
+// "D67301"}}
+type rocmGPU struct {
+	TotalMemory string `json:"VRAM Total Memory (B)"`
+	UsedMemory  string `json:"VRAM Total Used Memory (B)"`
+	Series      string `json:"Card series"`
+	Model       string `json:"Card model"`
+	Vendor      string `json:"Card vendor"`
+	SKU         string `json:"Card SKU"`
+}
+
+type rocmGPUList map[string]rocmGPU
+
+func parseRocmSMIOutput(output io.Reader) (models.Resources, error) {
+	var records rocmGPUList
+	err := json.NewDecoder(output).Decode(&records)
+	if err != nil {
+		return models.Resources{}, err
+	}
+
+	gpus := make([]models.GPU, len(records))
+	for cardNumber, record := range records {
+		index, err := strconv.ParseUint(cardNumber[4:], 10, 64)
+		if err != nil {
+			return models.Resources{}, err
+		}
+		gpus[index].Index = index
+		gpus[index].Name = record.Series
+		memBytes, err := strconv.ParseUint(record.TotalMemory, 10, 64)
+		if err != nil {
+			return models.Resources{}, err
+		}
+		gpus[index].Memory = memBytes / bytesPerMebibyte // convert to mebibytes
+		gpus[index].Vendor = models.GPUVendorAMDATI
+	}
+
+	return models.Resources{GPU: uint64(len(gpus)), GPUs: gpus}, nil
+}
+
+func NewAMDGPUProvider() capacity.Provider {
+	return &capacity.ToolBasedProvider{
+		Command:  rocmCommand,
+		Provides: "AMD GPUs",
+		Args:     rocmArgs,
+		Parser:   parseRocmSMIOutput,
+	}
+}
diff --git a/pkg/compute/capacity/system/gpu/nvidia.go b/pkg/compute/capacity/system/gpu/nvidia.go
new file mode 100644
index 0000000000..542b2ec766
--- /dev/null
+++ b/pkg/compute/capacity/system/gpu/nvidia.go
@@ -0,0 +1,55 @@
+package gpu
+
+import (
+	"encoding/csv"
+	"io"
+	"strconv"
+
+	"github.com/bacalhau-project/bacalhau/pkg/compute/capacity"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+)
+
+// TODO(forrest) consider switching to: https://github.com/NVIDIA/gpu-monitoring-tools
+const (
+	// nvidiaCLI is the path to the Nvidia helper binary
+	nvidiaCLI = "nvidia-smi"
+	// nvidiaCLIArgs is the args we pass the nvidiaCLI
+	nvidiaCLIQueryArg  = "--query-gpu=index,gpu_name,memory.total"
+	nvidiaCLIFormatArg = "--format=csv,noheader,nounits"
+)
+
+func NewNvidiaGPUProvider() capacity.Provider {
+	return &capacity.ToolBasedProvider{
+		Command:  nvidiaCLI,
+		Provides: "Nvidia GPUs",
+		Args:     []string{nvidiaCLIQueryArg, nvidiaCLIFormatArg},
+		Parser:   parseNvidiaCliOutput,
+	}
+}
+
+func parseNvidiaCliOutput(resp io.Reader) (models.Resources, error) {
+	reader := csv.NewReader(resp)
+	reader.TrimLeadingSpace = true
+
+	records, err := reader.ReadAll()
+	if err != nil {
+		return models.Resources{}, err
+	}
+
+	gpus := make([]models.GPU, len(records))
+	for index, record := range records {
+		gpus[index].Index, err = strconv.ParseUint(record[0], 10, 64)
+		if err != nil {
+			return models.Resources{}, err
+		}
+
+		gpus[index].Vendor = models.GPUVendorNvidia
+		gpus[index].Name = record[1]
+		gpus[index].Memory, err = strconv.ParseUint(record[2], 10, 64)
+		if err != nil {
+			return models.Resources{}, err
+		}
+	}
+
+	return models.Resources{GPU: uint64(len(gpus)), GPUs: gpus}, nil
+}
diff --git a/pkg/compute/capacity/system/provider.go b/pkg/compute/capacity/system/provider.go
index 3e47b38149..d04ba5beee 100644
--- a/pkg/compute/capacity/system/provider.go
+++ b/pkg/compute/capacity/system/provider.go
@@ -1,77 +1,70 @@
 package system
 
 import (
-	"bytes"
 	"context"
-	"encoding/csv"
 	"fmt"
-	"io"
-	"os/exec"
 	"runtime"
-	"strconv"
 
 	"github.com/pbnjay/memory"
 	"github.com/ricochet2200/go-disk-usage/du"
-	"github.com/rs/zerolog/log"
 
 	"github.com/bacalhau-project/bacalhau/pkg/compute/capacity"
+	"github.com/bacalhau-project/bacalhau/pkg/compute/capacity/system/gpu"
 	"github.com/bacalhau-project/bacalhau/pkg/config"
 	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
-type GPU struct {
-	// Self-reported index of the device in the system
-	Index uint64
-	// Model name of the GPU e.g. Tesla T4
-	Name string
-	// Total GPU memory in mebibytes (MiB)
-	Memory uint64
-}
-
 type PhysicalCapacityProvider struct {
+	gpuCapacityProviders []capacity.Provider
 }
 
 func NewPhysicalCapacityProvider() *PhysicalCapacityProvider {
-	return &PhysicalCapacityProvider{}
+	return &PhysicalCapacityProvider{
+		gpuCapacityProviders: []capacity.Provider{
+			gpu.NewNvidiaGPUProvider(),
+			gpu.NewAMDGPUProvider(),
+		},
+	}
 }
 
 func (p *PhysicalCapacityProvider) GetAvailableCapacity(ctx context.Context) (models.Resources, error) {
-	diskSpace, err := getFreeDiskSpace(config.GetStoragePath())
+	totalCapacity, err := p.GetTotalCapacity(ctx)
 	if err != nil {
-		return models.Resources{}, err
-	}
-	gpus, err := numSystemGPUs()
-	if err != nil {
-		return models.Resources{}, err
+		return totalCapacity, err
 	}
 
-	// the actual resources we have
 	return models.Resources{
-		CPU:    float64(runtime.NumCPU()) * 0.8,
-		Memory: memory.TotalMemory() * 80 / 100,
-		Disk:   diskSpace * 80 / 100,
-		GPU:    gpus,
+		CPU:    totalCapacity.CPU * 0.8,
+		Memory: totalCapacity.Memory * 80 / 100,
+		Disk:   totalCapacity.Disk * 80 / 100,
+		GPU:    totalCapacity.GPU,
+		GPUs:   totalCapacity.GPUs,
 	}, nil
 }
 
-func (p *PhysicalCapacityProvider) GetTotalCapacity() (models.Resources, error) {
+func (p *PhysicalCapacityProvider) GetTotalCapacity(ctx context.Context) (models.Resources, error) {
 	diskSpace, err := getFreeDiskSpace(config.GetStoragePath())
 	if err != nil {
 		return models.Resources{}, err
 	}
-	gpus, err := numSystemGPUs()
-	if err != nil {
-		return models.Resources{}, err
-	}
 
-	// the actual resources we have
-	return models.Resources{
+	// the total resources we have
+	resources := models.Resources{
 		CPU:    float64(runtime.NumCPU()),
 		Memory: memory.TotalMemory(),
 		Disk:   diskSpace,
-		GPU:    gpus,
-	}, nil
+	}
+
+	for _, gpuProvider := range p.gpuCapacityProviders {
+		gpuCapacity, err := gpuProvider.GetAvailableCapacity(ctx)
+		if err != nil {
+			return resources, err
+		}
 
+		resources = *resources.Add(gpuCapacity)
+	}
+
+	return resources, nil
 }
 
 // get free disk space for storage path
@@ -84,70 +77,5 @@ func getFreeDiskSpace(path string) (uint64, error) {
 	return usage.Free(), nil
 }
 
-func numSystemGPUs() (uint64, error) {
-	gpus, err := GetSystemGPUs()
-	return uint64(len(gpus)), err
-}
-
-// TODO(forrest) consider switching to: https://github.com/NVIDIA/gpu-monitoring-tools
-const (
-	// nvidiaCLI is the path to the Nvidia helper binary
-	nvidiaCLI = "nvidia-smi"
-	// nvidiaCLIArgs is the args we pass the nvidiaCLI
-	nvidiaCLIQueryArg  = "--query-gpu=index,gpu_name,memory.total"
-	nvidiaCLIFormatArg = "--format=csv,noheader,nounits"
-)
-
-func GetSystemGPUs() ([]GPU, error) {
-	nvidiaPath, err := exec.LookPath(nvidiaCLI)
-	if err != nil {
-		// If the NVIDIA CLI is not installed, we can't know the number of GPUs.
-		// It is not an error to assume zero.
-		log.Info().Msgf("cannot inspect system GPUs: %s not installed. GPUs will not be used.", nvidiaCLI)
-		return []GPU{}, nil
-	}
-	cmd := exec.Command(nvidiaPath, nvidiaCLIQueryArg, nvidiaCLIFormatArg)
-	resp, err := cmd.Output()
-	if err != nil {
-		// we won't error here since some hosts may have nvidia-smi installed but in a misconfigured state
-		// e.g. their drivers are missing, the smi can't communicate with the drivers, etc.
-		// instead we provide a warning show the args to the command we tried and its response
-		// motivation: https://expanso.atlassian.net/browse/GDAY-90
-		log.Warn().Err(err).
-			Str("command", fmt.Sprintf("%s %s %s", nvidiaCLI, nvidiaCLIQueryArg, nvidiaCLIFormatArg)).
-			Str("response", string(resp)).
-			Msgf("inspecting system GPUs failed")
-		return []GPU{}, nil
-	}
-
-	return parseNvidiaCliOutput(bytes.NewReader(resp))
-}
-
-func parseNvidiaCliOutput(resp io.Reader) ([]GPU, error) {
-	reader := csv.NewReader(resp)
-	reader.TrimLeadingSpace = true
-
-	records, err := reader.ReadAll()
-	if err != nil {
-		return []GPU{}, err
-	}
-
-	gpus := make([]GPU, len(records))
-	for index, record := range records {
-		gpus[index].Index, err = strconv.ParseUint(record[0], 10, 64)
-		if err != nil {
-			return gpus, err
-		}
-
-		gpus[index].Name = record[1]
-		gpus[index].Memory, err = strconv.ParseUint(record[2], 10, 64)
-		if err != nil {
-			return gpus, err
-		}
-	}
-
-	return gpus, nil
-}
-
 // compile-time check that the provider implements the interface
 var _ capacity.Provider = (*PhysicalCapacityProvider)(nil)
diff --git a/pkg/compute/capacity/tool.go b/pkg/compute/capacity/tool.go
new file mode 100644
index 0000000000..cc872005be
--- /dev/null
+++ b/pkg/compute/capacity/tool.go
@@ -0,0 +1,54 @@
+package capacity
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/rs/zerolog/log"
+)
+
+// ToolBasedProvider will run an external tool and parse the results in a
+// tool-specific way into a models.Resources instance.
+//
+// The tool is not required to return values for all fields. Any fields that
+// aren't covered by the tool will have a zero value.
+type ToolBasedProvider struct {
+	Command  string
+	Provides string
+	Args     []string
+	Parser   func(io.Reader) (models.Resources, error)
+}
+
+// GetAvailableCapacity implements Provider.
+func (tool *ToolBasedProvider) GetAvailableCapacity(ctx context.Context) (models.Resources, error) {
+	toolPath, err := exec.LookPath(tool.Command)
+	if err != nil {
+		// If the tool is not installed, we can't know the number of GPUs.
+		// It is not an error to assume zero.
+		log.Ctx(ctx).Info().Msgf("cannot inspect system %s: %s not installed. %s will not be used.", tool.Provides, tool.Command, tool.Provides)
+		return models.Resources{}, nil
+	}
+
+	cmd := exec.Command(toolPath, tool.Args...)
+	resp, err := cmd.Output()
+	if err != nil {
+		// we won't error here since some hosts may have the tool installed but
+		// in a misconfigured state e.g. their drivers are missing, the smi
+		// can't communicate with the drivers, etc. instead we provide a warning
+		// show the args to the command we tried and its response.
+		// motivation: https://expanso.atlassian.net/browse/GDAY-90
+		log.Warn().Err(err).
+			Str("command", fmt.Sprintf("%s %v", toolPath, tool.Args)).
+			Str("response", string(resp)).
+			Msgf("inspecting system failed")
+		return models.Resources{}, nil
+	}
+
+	return tool.Parser(bytes.NewReader(resp))
+}
+
+var _ Provider = (*ToolBasedProvider)(nil)
diff --git a/pkg/compute/capacity/trackers.go b/pkg/compute/capacity/trackers.go
index 99eedb36e1..f02453359d 100644
--- a/pkg/compute/capacity/trackers.go
+++ b/pkg/compute/capacity/trackers.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"sync"
 
+	"github.com/bacalhau-project/bacalhau/pkg/lib/math"
 	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
@@ -28,16 +29,25 @@ func (t *LocalTracker) IsWithinLimits(ctx context.Context, usage models.Resource
 	return usage.LessThanEq(t.maxCapacity)
 }
 
-func (t *LocalTracker) AddIfHasCapacity(ctx context.Context, usage models.Resources) bool {
+func (t *LocalTracker) AddIfHasCapacity(ctx context.Context, usage models.Resources) *models.Resources {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	newUsedCapacity := t.usedCapacity.Add(usage)
-	if newUsedCapacity.LessThanEq(t.maxCapacity) {
-		t.usedCapacity = *newUsedCapacity
-		return true
+	if !newUsedCapacity.LessThanEq(t.maxCapacity) {
+		return nil
 	}
-	return false
+
+	// Allocate any GPUs that have been asked for but not chosen
+	unspecifiedGPUs := math.Max(usage.GPU-uint64(len(usage.GPUs)), 0)
+	availableGPUs := t.maxCapacity.Sub(t.usedCapacity).GPUs
+	if unspecifiedGPUs > uint64(len(availableGPUs)) {
+		return nil
+	}
+	usage.GPUs = append(usage.GPUs, availableGPUs[:unspecifiedGPUs]...)
+
+	t.usedCapacity = *t.usedCapacity.Add(usage)
+	return &usage
 }
 
 func (t *LocalTracker) GetAvailableCapacity(ctx context.Context) models.Resources {
diff --git a/pkg/compute/capacity/types.go b/pkg/compute/capacity/types.go
index 174e932d5b..bc4ae840c9 100644
--- a/pkg/compute/capacity/types.go
+++ b/pkg/compute/capacity/types.go
@@ -13,8 +13,10 @@ type Tracker interface {
 	// IsWithinLimits returns true if the given resource usage is within the limits of the compute node.
 	// Limits refer to the total capacity of the compute node, and not to the currently available capacity.
 	IsWithinLimits(ctx context.Context, usage models.Resources) bool
-	// AddIfHasCapacity atomically adds the given resource usage to the tracker if the compute node has capacity for it.
-	AddIfHasCapacity(ctx context.Context, usage models.Resources) bool
+	// AddIfHasCapacity atomically adds the given resource usage to the tracker
+	// if the compute node has capacity for it, returning the resource usage
+	// that was added including any allocations that were made, or nil if the usage could not be added.
+	AddIfHasCapacity(ctx context.Context, usage models.Resources) *models.Resources
 	// GetAvailableCapacity returns the available capacity of the compute node.
 	GetAvailableCapacity(ctx context.Context) models.Resources
 	// GetMaxCapacity returns the total capacity of the compute node.
diff --git a/pkg/compute/executor_buffer.go b/pkg/compute/executor_buffer.go
index 096fe9255e..c44113b4b3 100644
--- a/pkg/compute/executor_buffer.go
+++ b/pkg/compute/executor_buffer.go
@@ -106,9 +106,16 @@ func (s *ExecutorBuffer) Run(ctx context.Context, localExecutionState store.Loca
 		err = fmt.Errorf("execution %s already running", execution.ID)
 		return
 	}
-	if !s.enqueuedCapacity.AddIfHasCapacity(ctx, *execution.TotalAllocatedResources()) {
+	if added := s.enqueuedCapacity.AddIfHasCapacity(ctx, *execution.TotalAllocatedResources()); added == nil {
 		err = fmt.Errorf("not enough capacity to enqueue job")
 		return
+	} else {
+		// Update the execution to include all the resources that have
+		// actually been allocated. Effectively this is picking the GPU(s)
+		// that the job will use. Note that this is not persisted here, as
+		// it was based on current usage information which would change
+		// under a restart, so it will only persist if the job starts
+		execution.AllocateResources(execution.Job.Task().Name, *added)
 	}
 
 	s.queuedTasks.Enqueue(newBufferTask(localExecutionState), execution.Job.Priority)
@@ -173,13 +180,21 @@ func (s *ExecutorBuffer) deque() {
 	for i := 0; i < max; i++ {
 		qitem := s.queuedTasks.DequeueWhere(func(task *bufferTask) bool {
 			// If we don't have enough resources to run this task, then we will skip it
-			add := s.runningCapacity.AddIfHasCapacity(ctx, *task.localExecutionState.Execution.TotalAllocatedResources())
-			if !add {
+			queued := task.localExecutionState.Execution.TotalAllocatedResources()
+			added := s.runningCapacity.AddIfHasCapacity(ctx, *queued)
+			if added == nil {
 				return false
 			}
 
+			// Update the execution to include all the resources that have
+			// actually been allocated
+			task.localExecutionState.Execution.AllocateResources(
+				task.localExecutionState.Execution.Job.Task().Name,
+				*added,
+			)
+
 			// Claim the resources now so that we don't count allocated resources
-			s.enqueuedCapacity.Remove(ctx, *task.localExecutionState.Execution.TotalAllocatedResources())
+			s.enqueuedCapacity.Remove(ctx, *queued)
 			return true
 		})
 
diff --git a/pkg/config/configenv/dev.go b/pkg/config/configenv/dev.go
index 908e8a57fd..afcf942346 100644
--- a/pkg/config/configenv/dev.go
+++ b/pkg/config/configenv/dev.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
 var Development = types.BacalhauConfig{
@@ -69,25 +70,25 @@ var Development = types.BacalhauConfig{
 var DevelopmentComputeConfig = types.ComputeConfig{
 	Capacity: types.CapacityConfig{
 		IgnorePhysicalResourceLimits: false,
-		TotalResourceLimits: model.ResourceUsageConfig{
+		TotalResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		JobResourceLimits: model.ResourceUsageConfig{
+		JobResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		DefaultJobResourceLimits: model.ResourceUsageConfig{
+		DefaultJobResourceLimits: models.ResourcesConfig{
 			CPU:    "500m",
 			Memory: "1Gb",
 			Disk:   "",
 			GPU:    "",
 		},
-		QueueResourceLimits: model.ResourceUsageConfig{
+		QueueResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
diff --git a/pkg/config/configenv/local.go b/pkg/config/configenv/local.go
index a550e8a19c..30f8038c37 100644
--- a/pkg/config/configenv/local.go
+++ b/pkg/config/configenv/local.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
 var Local = types.BacalhauConfig{
@@ -60,25 +61,25 @@ var Local = types.BacalhauConfig{
 var LocalComputeConfig = types.ComputeConfig{
 	Capacity: types.CapacityConfig{
 		IgnorePhysicalResourceLimits: false,
-		TotalResourceLimits: model.ResourceUsageConfig{
+		TotalResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		JobResourceLimits: model.ResourceUsageConfig{
+		JobResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		DefaultJobResourceLimits: model.ResourceUsageConfig{
+		DefaultJobResourceLimits: models.ResourcesConfig{
 			CPU:    "500m",
 			Memory: "1Gb",
 			Disk:   "",
 			GPU:    "",
 		},
-		QueueResourceLimits: model.ResourceUsageConfig{
+		QueueResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
diff --git a/pkg/config/configenv/production.go b/pkg/config/configenv/production.go
index 70fcaa054a..05781c2f86 100644
--- a/pkg/config/configenv/production.go
+++ b/pkg/config/configenv/production.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
 var Production = types.BacalhauConfig{
@@ -77,25 +78,25 @@ var Production = types.BacalhauConfig{
 var ProductionComputeConfig = types.ComputeConfig{
 	Capacity: types.CapacityConfig{
 		IgnorePhysicalResourceLimits: false,
-		TotalResourceLimits: model.ResourceUsageConfig{
+		TotalResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		JobResourceLimits: model.ResourceUsageConfig{
+		JobResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		DefaultJobResourceLimits: model.ResourceUsageConfig{
+		DefaultJobResourceLimits: models.ResourcesConfig{
 			CPU:    "500m",
 			Memory: "1Gb",
 			Disk:   "",
 			GPU:    "",
 		},
-		QueueResourceLimits: model.ResourceUsageConfig{
+		QueueResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
diff --git a/pkg/config/configenv/staging.go b/pkg/config/configenv/staging.go
index cd36d84ee4..7e83ac5e79 100644
--- a/pkg/config/configenv/staging.go
+++ b/pkg/config/configenv/staging.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
 var Staging = types.BacalhauConfig{
@@ -75,25 +76,25 @@ var Staging = types.BacalhauConfig{
 var StagingComputeConfig = types.ComputeConfig{
 	Capacity: types.CapacityConfig{
 		IgnorePhysicalResourceLimits: false,
-		TotalResourceLimits: model.ResourceUsageConfig{
+		TotalResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		JobResourceLimits: model.ResourceUsageConfig{
+		JobResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		DefaultJobResourceLimits: model.ResourceUsageConfig{
+		DefaultJobResourceLimits: models.ResourcesConfig{
 			CPU:    "500m",
 			Memory: "1Gb",
 			Disk:   "",
 			GPU:    "",
 		},
-		QueueResourceLimits: model.ResourceUsageConfig{
+		QueueResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
diff --git a/pkg/config/types/compute.go b/pkg/config/types/compute.go
index e2312da500..b0836694d6 100644
--- a/pkg/config/types/compute.go
+++ b/pkg/config/types/compute.go
@@ -1,6 +1,9 @@
 package types
 
-import "github.com/bacalhau-project/bacalhau/pkg/model"
+import (
+	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+)
 
 type ComputeConfig struct {
 	Capacity       CapacityConfig           `yaml:"Capacity"`
@@ -14,11 +17,11 @@ type ComputeConfig struct {
 type CapacityConfig struct {
 	IgnorePhysicalResourceLimits bool `yaml:"IgnorePhysicalResourceLimits"`
 	// Total amount of resource the system can be using at one time in aggregate for all jobs.
-	TotalResourceLimits model.ResourceUsageConfig `yaml:"TotalResourceLimits"`
+	TotalResourceLimits models.ResourcesConfig `yaml:"TotalResourceLimits"`
 	// Per job amount of resource the system can be using at one time.
-	JobResourceLimits        model.ResourceUsageConfig `yaml:"JobResourceLimits"`
-	DefaultJobResourceLimits model.ResourceUsageConfig `yaml:"DefaultJobResourceLimits"`
-	QueueResourceLimits      model.ResourceUsageConfig `yaml:"QueueResourceLimits"`
+	JobResourceLimits        models.ResourcesConfig `yaml:"JobResourceLimits"`
+	DefaultJobResourceLimits models.ResourcesConfig `yaml:"DefaultJobResourceLimits"`
+	QueueResourceLimits      models.ResourcesConfig `yaml:"QueueResourceLimits"`
 }
 
 type JobTimeoutConfig struct {
diff --git a/pkg/executor/docker/executor.go b/pkg/executor/docker/executor.go
index 282ca98b76..65ce56532f 100644
--- a/pkg/executor/docker/executor.go
+++ b/pkg/executor/docker/executor.go
@@ -13,6 +13,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
 	"go.uber.org/atomic"
 
 	dockermodels "github.com/bacalhau-project/bacalhau/pkg/executor/docker/models"
@@ -303,16 +304,11 @@ func (e *Executor) newDockerJobContainer(ctx context.Context, params *dockerJobC
 
 	// Create GPU request if the job requests it
 	// TODO we need to use the resource units requested by for the GPU.
-	var deviceRequests []container.DeviceRequest
-	if params.Resources.GPU > 0 {
-		deviceRequests = append(deviceRequests,
-			container.DeviceRequest{
-				DeviceIDs:    []string{"0"}, // TODO: how do we know which device ID to use?
-				Capabilities: [][]string{{"gpu"}},
-			},
-		)
-		log.Ctx(ctx).Trace().Msgf("Adding %d GPUs to request", params.Resources.GPU)
+	deviceRequests, deviceMappings, err := configureDevices(ctx, params.Resources)
+	if err != nil {
+		return container.CreateResponse{}, fmt.Errorf("creating container devices: %w", err)
 	}
+	log.Ctx(ctx).Trace().Msgf("Adding %d GPUs to request", params.Resources.GPU)
 
 	hostConfig := &container.HostConfig{
 		Mounts: mounts,
@@ -320,6 +316,7 @@ func (e *Executor) newDockerJobContainer(ctx context.Context, params *dockerJobC
 			Memory:         int64(params.Resources.Memory),
 			NanoCPUs:       int64(params.Resources.CPU * NanoCPUCoefficient),
 			DeviceRequests: deviceRequests,
+			Devices:        deviceMappings,
 		},
 	}
 
@@ -352,6 +349,46 @@ func (e *Executor) newDockerJobContainer(ctx context.Context, params *dockerJobC
 	return jobContainer, nil
 }
 
+func configureDevices(ctx context.Context, resources *models.Resources) ([]container.DeviceRequest, []container.DeviceMapping, error) {
+	requests := []container.DeviceRequest{}
+	mappings := []container.DeviceMapping{}
+	vendorGroups := lo.GroupBy(resources.GPUs, func(gpu models.GPU) models.GPUVendor { return gpu.Vendor })
+
+	for vendor, gpus := range vendorGroups {
+		switch vendor {
+		case models.GPUVendorNvidia:
+			requests = append(requests, container.DeviceRequest{
+				DeviceIDs:    lo.Map(gpus, func(gpu models.GPU, _ int) string { return fmt.Sprint(gpu.Index) }),
+				Capabilities: [][]string{{"gpu"}},
+			})
+		case models.GPUVendorAMDATI:
+			// https://docs.amd.com/en/latest/deploy/docker.html
+			mappings = append(mappings, container.DeviceMapping{
+				PathOnHost:        "/dev/kfd",
+				PathInContainer:   "/dev/kfd",
+				CgroupPermissions: "rwm",
+			})
+			for _, gpu := range gpus {
+				mappings = append(mappings,
+					container.DeviceMapping{
+						PathOnHost:        fmt.Sprintf("/dev/dri/card%d", gpu.Index),
+						PathInContainer:   fmt.Sprintf("/dev/dri/card%d", gpu.Index),
+						CgroupPermissions: "rwm",
+					},
+					container.DeviceMapping{
+						PathOnHost:        fmt.Sprintf("/dev/dri/renderD%d", (128 + gpu.Index)),
+						PathInContainer:   fmt.Sprintf("/dev/dri/renderD%d", (128 + gpu.Index)),
+						CgroupPermissions: "rwm",
+					},
+				)
+			}
+		default:
+			return nil, nil, fmt.Errorf("job requires GPU from unsupported vendor %q", vendor)
+		}
+	}
+	return requests, mappings, nil
+}
+
 func makeContainerMounts(
 	ctx context.Context, inputs []storage.PreparedStorage, outputs []*models.ResultPath, resultsDir string) ([]mount.Mount, error) {
 	// the actual mounts we will give to the container
diff --git a/pkg/model/resource_usage_parser.go b/pkg/model/resource_usage_parser.go
deleted file mode 100644
index b990903506..0000000000
--- a/pkg/model/resource_usage_parser.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package model
-
-import (
-	"strconv"
-	"strings"
-
-	"github.com/BTBurke/k8sresource"
-	"github.com/dustin/go-humanize"
-)
-
-func ParseResourceUsageConfig(usage ResourceUsageConfig) (ResourceUsageData, error) {
-	cpu, err := ConvertCPUString(usage.CPU)
-	if err != nil {
-		return ResourceUsageData{}, err
-	}
-	mem, err := ConvertBytesString(usage.Memory)
-	if err != nil {
-		return ResourceUsageData{}, err
-	}
-	disk, err := ConvertBytesString(usage.Disk)
-	if err != nil {
-		return ResourceUsageData{}, err
-	}
-	gpu, err := ConvertGPUString(usage.GPU)
-	if err != nil {
-		return ResourceUsageData{}, err
-	}
-
-	return ResourceUsageData{
-		CPU:    cpu,
-		Memory: mem,
-		Disk:   disk,
-		GPU:    gpu,
-	}, nil
-}
-func ConvertCPUString(val string) (float64, error) {
-	ret, err := convertCPUStringWithError(val)
-	if err != nil {
-		return 0, err
-	}
-	return ret, nil
-}
-
-func ConvertBytesString(val string) (uint64, error) {
-	ret, err := convertBytesStringWithError(val)
-	if err != nil {
-		return 0, err
-	}
-	return ret, nil
-}
-
-func ConvertGPUString(val string) (uint64, error) {
-	if val == "" {
-		return 0, nil
-	}
-	ret, err := strconv.ParseUint(val, 10, 64) //nolint:gomnd
-	if err != nil {
-		return 0, err
-	}
-	return ret, nil
-}
-
-func convertCPUStringWithError(val string) (float64, error) {
-	if val == "" {
-		return 0, nil
-	}
-	cpu, err := k8sresource.NewCPUFromString(sanitizeBytesString(val))
-	if err != nil {
-		return 0, err
-	}
-	return cpu.ToFloat64(), nil
-}
-
-func convertBytesStringWithError(val string) (uint64, error) {
-	if val == "" {
-		return 0, nil
-	}
-	return humanize.ParseBytes(val)
-}
-
-// allow Mi, Gi to mean Mb, Gb
-// remove spaces
-// lowercase
-func sanitizeBytesString(st string) string {
-	st = strings.ToLower(st)
-	st = strings.ReplaceAll(st, "i", "b")
-	st = strings.ReplaceAll(st, " ", "")
-	return st
-}
diff --git a/pkg/models/resource.go b/pkg/models/resource.go
index 3473e9f2f3..9ffc8d7f3f 100644
--- a/pkg/models/resource.go
+++ b/pkg/models/resource.go
@@ -10,6 +10,7 @@ import (
 	"github.com/c2h5oh/datasize"
 	"github.com/hashicorp/go-multierror"
 	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
 )
 
 type ResourcesConfig struct {
@@ -147,6 +148,24 @@ func (r *ResourcesConfigBuilder) BuildOrDie() *ResourcesConfig {
 	return resources
 }
 
+type GPUVendor string
+
+const (
+	GPUVendorNvidia GPUVendor = "NVIDIA"
+	GPUVendorAMDATI GPUVendor = "AMD/ATI"
+)
+
+type GPU struct {
+	// Self-reported index of the device in the system
+	Index uint64
+	// Model name of the GPU e.g. Tesla T4
+	Name string
+	// Maker of the GPU, e.g. NVidia, AMD, Intel
+	Vendor GPUVendor
+	// Total GPU memory in mebibytes (MiB)
+	Memory uint64
+}
+
 type Resources struct {
 	// CPU units
 	CPU float64 `json:"CPU,omitempty"`
@@ -156,6 +175,8 @@ type Resources struct {
 	Disk uint64 `json:"Disk,omitempty"`
 	// GPU units
 	GPU uint64 `json:"GPU,omitempty"`
+	// GPU details
+	GPUs []GPU `json:"GPUs,omitempty"`
 }
 
 // Copy returns a deep copy of the resources
@@ -177,6 +198,14 @@ func (r *Resources) Validate() error {
 	if r.CPU < 0 {
 		mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid CPU value: %f", r.CPU))
 	}
+	if len(r.GPUs) > int(r.GPU) {
+		// It's not an error for the GPUs specified to be less than the number
+		// given by the GPU field – that just signifies that either:
+		// - the user is requesting "generic GPUs" without specifying more information
+		// - the system knows it has GPUs but no further information about them
+		// But the number should always be at least the length of the GPUs array
+		mErr.Errors = append(mErr.Errors, fmt.Errorf("%d GPUs specified but have details for %d", r.GPU, len(r.GPUs)))
+	}
 	return mErr.ErrorOrNil()
 }
 
@@ -195,6 +224,9 @@ func (r *Resources) Merge(other Resources) *Resources {
 	if newR.GPU <= 0 {
 		newR.GPU = other.GPU
 	}
+	if len(newR.GPUs) <= 0 {
+		newR.GPUs = other.GPUs
+	}
 	return newR
 }
 
@@ -205,6 +237,7 @@ func (r *Resources) Add(other Resources) *Resources {
 		Memory: r.Memory + other.Memory,
 		Disk:   r.Disk + other.Disk,
 		GPU:    r.GPU + other.GPU,
+		GPUs:   append(r.GPUs, other.GPUs...),
 	}
 }
 
@@ -216,6 +249,8 @@ func (r *Resources) Sub(other Resources) *Resources {
 		GPU:    r.GPU - other.GPU,
 	}
 
+	usage.GPUs, _ = lo.Difference(r.GPUs, other.GPUs)
+
 	if r.LessThan(other) {
 		log.Warn().Msgf("Subtracting larger resource usage %s from %s. Replacing negative values with zeros",
 			other.String(), r.String())
diff --git a/pkg/node/requester.go b/pkg/node/requester.go
index e2d16d66d3..5c77572dfb 100644
--- a/pkg/node/requester.go
+++ b/pkg/node/requester.go
@@ -19,6 +19,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/pubsub/libp2p"
 	"github.com/bacalhau-project/bacalhau/pkg/requester/pubsub/jobinfo"
 	s3helper "github.com/bacalhau-project/bacalhau/pkg/s3"
+	"github.com/bacalhau-project/bacalhau/pkg/util"
 	libp2p_pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/host"
@@ -325,22 +326,22 @@ func NewRequesterNode(
 
 		cleanupErr := jobInfoPubSub.Close(ctx)
 		if cleanupErr != nil {
-			log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to shutdown job info pubsub")
+			util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to shutdown job info pubsub")
 		}
 
 		cleanupErr = tracerContextProvider.Shutdown()
 		if cleanupErr != nil {
-			log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to shutdown tracer context provider")
+			util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to shutdown tracer context provider")
 		}
 		cleanupErr = eventTracer.Shutdown()
 		if cleanupErr != nil {
-			log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to shutdown event tracer")
+			util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to shutdown event tracer")
 		}
 
 		// Close the jobstore after the evaluation broker is disabled
 		cleanupErr = jobStore.Close(ctx)
 		if cleanupErr != nil {
-			log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to cleanly shutdown jobstore")
+			util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to cleanly shutdown jobstore")
 		}
 	}
 


Test patch:
diff --git a/pkg/compute/capacity/system/gpu/amd_test.go b/pkg/compute/capacity/system/gpu/amd_test.go
new file mode 100644
index 0000000000..c81182194e
--- /dev/null
+++ b/pkg/compute/capacity/system/gpu/amd_test.go
@@ -0,0 +1,85 @@
+//go:build unit || !integration
+
+package gpu
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParsingAMDGPUsWithOne(t *testing.T) {
+	output := strings.NewReader(
+		`{"card0": {"VRAM Total Memory (B)": "68702699520", ` +
+			`"VRAM Total Used Memory (B)": "10960896", ` +
+			`"Card series": "Instinct MI210", "Card model": "0x0c34", ` +
+			`"Card vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU":` +
+			`"D67301"}}`,
+	)
+
+	resources, err := parseRocmSMIOutput(output)
+	require.NoError(t, err)
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	require.Equal(t, float64(0), resources.CPU)
+	require.Equal(t, uint64(0), resources.Memory)
+	require.Equal(t, uint64(0), resources.Disk)
+
+	gpus := resources.GPUs
+	require.Len(t, gpus, 1)
+	require.Equal(t, models.GPUVendorAMDATI, gpus[0].Vendor)
+	require.Equal(t, uint64(0), gpus[0].Index)
+	require.Equal(t, "Instinct MI210", gpus[0].Name)
+	require.Equal(t, uint64(65520), gpus[0].Memory)
+}
+
+func TestParsingAMDGPUsWithMany(t *testing.T) {
+	output := strings.NewReader(
+		`{"card0": {"VRAM Total Memory (B)": "0", "VRAM Total Used ` +
+			`Memory (B)": "0", "Card series": "Instinct MI210", "Card ` +
+			`model": "0x0c34", "Card vendor": "Advanced Micro Devices, Inc. ` +
+			`[AMD/ATI]", "Card SKU": "D67301"}, "card1": {"VRAM Total Memory (B)": ` +
+			`"8587837440", "VRAM Total Used Memory (B)": "10960896", "Card ` +
+			`series": "Instinct MI210", "Card model": "0x0c34", "Card vendor": ` +
+			`"Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU": "D67301"}, ` +
+			`"card2": {"VRAM Total Memory (B)": "17175674880", "VRAM Total Used ` +
+			`Memory (B)": "10960896", "Card series": "Instinct MI210", "Card ` +
+			`model": "0x0c34", "Card vendor": "Advanced Micro Devices, Inc. ` +
+			`[AMD/ATI]", "Card SKU": "D67301"}, "card3": {"VRAM Total Memory (B)": ` +
+			`"25763512320", "VRAM Total Used Memory (B)": "10960896", "Card ` +
+			`series": "Instinct MI210", "Card model": "0x0c34", "Card vendor": ` +
+			`"Advanced Micro Devices, Inc. [AMD/ATI]", "Card SKU": "D67301"} }`,
+	)
+	resources, err := parseRocmSMIOutput(output)
+	require.NoError(t, err)
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	require.Equal(t, float64(0), resources.CPU)
+	require.Equal(t, uint64(0), resources.Memory)
+	require.Equal(t, uint64(0), resources.Disk)
+
+	gpus := resources.GPUs
+	require.Len(t, gpus, 4)
+
+	for index, gpu := range gpus {
+		require.Equal(t, models.GPUVendorAMDATI, gpu.Vendor)
+		require.Equal(t, uint64(index), gpu.Index)
+		require.Equal(t, "Instinct MI210", gpu.Name)
+		require.Equal(t, uint64(index*8190), gpu.Memory)
+	}
+}
+
+func TestParsingAMDGPUsWithNone(t *testing.T) {
+	output := strings.NewReader(`{}`)
+
+	resources, err := parseRocmSMIOutput(output)
+	require.NoError(t, err)
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	require.Equal(t, float64(0), resources.CPU)
+	require.Equal(t, uint64(0), resources.Memory)
+	require.Equal(t, uint64(0), resources.Disk)
+
+	gpus := resources.GPUs
+	require.Len(t, gpus, 0)
+
+}
diff --git a/pkg/compute/capacity/system/provider_test.go b/pkg/compute/capacity/system/gpu/nvidia_test.go
similarity index 51%
rename from pkg/compute/capacity/system/provider_test.go
rename to pkg/compute/capacity/system/gpu/nvidia_test.go
index 6c3a5bea66..116ce09da2 100644
--- a/pkg/compute/capacity/system/provider_test.go
+++ b/pkg/compute/capacity/system/gpu/nvidia_test.go
@@ -1,48 +1,58 @@
 //go:build unit || !integration
 
-package system
+package gpu
 
 import (
 	"strings"
 	"testing"
 
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 	"github.com/stretchr/testify/require"
 )
 
-func TestParsingGPUsWithMany(t *testing.T) {
+func TestParsingNvidiaGPUsWithMany(t *testing.T) {
 	output := strings.Join([]string{
 		"0, Tesla T4, 15360",
 		"1, Tesla T1, 12345",
 	}, "\n")
 
-	gpus, err := parseNvidiaCliOutput(strings.NewReader(output))
+	resources, err := parseNvidiaCliOutput(strings.NewReader(output))
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	gpus := resources.GPUs
 	require.NoError(t, err)
 	require.Len(t, gpus, 2)
 	require.Equal(t, uint64(0), gpus[0].Index)
 	require.Equal(t, "Tesla T4", gpus[0].Name)
 	require.Equal(t, uint64(15360), gpus[0].Memory)
+	require.Equal(t, models.GPUVendorNvidia, gpus[0].Vendor)
 	require.Equal(t, uint64(1), gpus[1].Index)
 	require.Equal(t, "Tesla T1", gpus[1].Name)
 	require.Equal(t, uint64(12345), gpus[1].Memory)
+	require.Equal(t, models.GPUVendorNvidia, gpus[1].Vendor)
 }
 
-func TestParsingGPUsWithOne(t *testing.T) {
+func TestParsingNvidiaGPUsWithOne(t *testing.T) {
 	output := strings.Join([]string{
 		"0, Tesla T4, 15360",
 	}, "\n")
 
-	gpus, err := parseNvidiaCliOutput(strings.NewReader(output))
+	resources, err := parseNvidiaCliOutput(strings.NewReader(output))
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	gpus := resources.GPUs
 	require.NoError(t, err)
 	require.Len(t, gpus, 1)
 	require.Equal(t, uint64(0), gpus[0].Index)
 	require.Equal(t, "Tesla T4", gpus[0].Name)
 	require.Equal(t, uint64(15360), gpus[0].Memory)
+	require.Equal(t, models.GPUVendorNvidia, gpus[0].Vendor)
 }
 
-func TestParsingGPUsWithNone(t *testing.T) {
+func TestParsingNvidiaGPUsWithNone(t *testing.T) {
 	output := strings.Join([]string{}, "\n")
 
-	gpus, err := parseNvidiaCliOutput(strings.NewReader(output))
+	resources, err := parseNvidiaCliOutput(strings.NewReader(output))
+	require.Equal(t, int(resources.GPU), len(resources.GPUs))
+	gpus := resources.GPUs
 	require.NoError(t, err)
 	require.Len(t, gpus, 0)
 }
diff --git a/pkg/compute/capacity/trackers_test.go b/pkg/compute/capacity/trackers_test.go
new file mode 100644
index 0000000000..6ffdf5350f
--- /dev/null
+++ b/pkg/compute/capacity/trackers_test.go
@@ -0,0 +1,48 @@
+//go:build unit || !integration
+
+package capacity
+
+import (
+	"context"
+	"testing"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAllocatesGPUs(t *testing.T) {
+	tracker := NewLocalTracker(LocalTrackerParams{MaxCapacity: models.Resources{
+		GPU: 2,
+		GPUs: []models.GPU{
+			{Index: 0, Name: "Lancer 2X", Vendor: models.GPUVendorNvidia, Memory: 100},
+			{Index: 1, Name: "Berdly 1.0", Vendor: models.GPUVendorAMDATI, Memory: 100},
+		},
+	}})
+
+	added := tracker.AddIfHasCapacity(context.Background(), models.Resources{GPU: 1})
+	require.NotNil(t, added)
+	require.Equal(t, uint64(1), added.GPU)
+
+	avail := tracker.GetAvailableCapacity(context.Background())
+	require.Equal(t, uint64(1), avail.GPU)
+	require.Len(t, avail.GPUs, 1)
+	require.NotEqual(t, avail.GPUs[0], added.GPUs[0])
+}
+
+func TestDoesntAllocateGPUs(t *testing.T) {
+	tracker := NewLocalTracker(LocalTrackerParams{MaxCapacity: models.Resources{
+		GPU: 2,
+		GPUs: []models.GPU{
+			{Index: 0, Name: "Lancer 2X", Vendor: models.GPUVendorNvidia, Memory: 100},
+			{Index: 1, Name: "Berdly 1.0", Vendor: models.GPUVendorAMDATI, Memory: 100},
+		},
+	}})
+
+	added := tracker.AddIfHasCapacity(context.Background(), models.Resources{GPU: 4})
+	require.Nil(t, added)
+
+	avail := tracker.GetAvailableCapacity(context.Background())
+	require.Equal(t, uint64(2), avail.GPU)
+	require.Len(t, avail.GPUs, 2)
+	require.Equal(t, avail, tracker.maxCapacity)
+}
diff --git a/pkg/config/configenv/test.go b/pkg/config/configenv/test.go
index c52c2c1bb0..4eb141e178 100644
--- a/pkg/config/configenv/test.go
+++ b/pkg/config/configenv/test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/config/types"
 	"github.com/bacalhau-project/bacalhau/pkg/logger"
 	"github.com/bacalhau-project/bacalhau/pkg/model"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
 var Testing = types.BacalhauConfig{
@@ -64,25 +65,25 @@ var Testing = types.BacalhauConfig{
 var TestingComputeConfig = types.ComputeConfig{
 	Capacity: types.CapacityConfig{
 		IgnorePhysicalResourceLimits: false,
-		TotalResourceLimits: model.ResourceUsageConfig{
+		TotalResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		JobResourceLimits: model.ResourceUsageConfig{
+		JobResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
 			GPU:    "",
 		},
-		DefaultJobResourceLimits: model.ResourceUsageConfig{
+		DefaultJobResourceLimits: models.ResourcesConfig{
 			CPU:    "100m",
 			Memory: "100Mi",
 			Disk:   "",
 			GPU:    "",
 		},
-		QueueResourceLimits: model.ResourceUsageConfig{
+		QueueResourceLimits: models.ResourcesConfig{
 			CPU:    "",
 			Memory: "",
 			Disk:   "",
diff --git a/pkg/model/resource_usage_parser_test.go b/pkg/model/resource_usage_parser_test.go
deleted file mode 100644
index 79c7890c57..0000000000
--- a/pkg/model/resource_usage_parser_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-//go:build unit || !integration
-
-package model
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestResourceUsageConfigParser(t *testing.T) {
-	testCases := []struct {
-		usageConfig  ResourceUsageConfig
-		expectedData ResourceUsageData
-	}{
-		{
-			ResourceUsageConfig{
-				CPU:    "100m",
-				Memory: "100Mi",
-				Disk:   "",
-				GPU:    "",
-			},
-			ResourceUsageData{
-				CPU:    0.1,               // 100m
-				Memory: 100 * 1024 * 1024, // 100Mi
-				Disk:   0,
-				GPU:    0,
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		data, err := ParseResourceUsageConfig(tc.usageConfig)
-		require.NoError(t, err)
-		require.Equal(t, tc.expectedData, data)
-	}
-}
diff --git a/pkg/test/compute/resourcelimits_test.go b/pkg/test/compute/resourcelimits_test.go
index 0551590a1a..ac21a30732 100644
--- a/pkg/test/compute/resourcelimits_test.go
+++ b/pkg/test/compute/resourcelimits_test.go
@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/c2h5oh/datasize"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -120,7 +121,8 @@ func (suite *ComputeNodeResourceLimitsSuite) TestTotalResourceLimits() {
 		}
 
 		getVolumeSizeHandler := func(ctx context.Context, volume models.InputSource) (uint64, error) {
-			return model.ConvertBytesString(volume.Target)
+			size, err := datasize.ParseString(volume.Target)
+			return size.Bytes(), err
 		}
 
 		resourcesConfig := legacy.FromLegacyResourceUsageConfig(testCase.totalLimits)
@@ -295,6 +297,7 @@ func (suite *ComputeNodeResourceLimitsSuite) TestParallelGPU() {
 			Memory: 1 * 1024 * 1024 * 1024,
 			Disk:   1 * 1024 * 1024 * 1024,
 			GPU:    1,
+			GPUs:   make([]models.GPU, 1),
 		},
 		IgnorePhysicalResourceLimits: true, // we need to pretend that we have GPUs on each node
 	})