Repository: Granite-Code/continue-for-granite
Pull request #20: Improve wizard hardware detection
Author: login: halfline
Created: 2025-03-18T18:42:09Z | Merged: 2025-03-21T12:23:34Z

This set of commits should make the setup wizard stop promoting the small model so aggressively.  I've only tested on linux, I will need to test on mac before we can merge.


Diff:
diff --git a/core/config/default.ts b/core/config/default.ts
index a84fa25d594..ece91683cbd 100644
--- a/core/config/default.ts
+++ b/core/config/default.ts
@@ -94,7 +94,7 @@ export const defaultConfigJetBrains: SerializedContinueConfig = {
   data: [],
 };
 
-const DEFAULT_CONTEXT_LENGTH = 16384;
+const DEFAULT_CONTEXT_LENGTH = 8192;
 
 const BASE_GRANITE_CONFIG: Partial<ModelDescription> = {
   contextLength: DEFAULT_CONTEXT_LENGTH,
diff --git a/core/granite/commons/modelRequirements.ts b/core/granite/commons/modelRequirements.ts
index 9dcf366637b..94138aceb9a 100644
--- a/core/granite/commons/modelRequirements.ts
+++ b/core/granite/commons/modelRequirements.ts
@@ -1,9 +1,8 @@
 import { GB } from "./sizeUtils";
-import { hasDiscreteGPU, isHighEndApple, SystemInfo } from "./sysInfo";
+import { SystemInfo } from "./sysInfo";
 import { formatSize } from "./textUtils";
 
 export interface ModelRequirements {
-  minMemoryBytes: number;
   recommendedMemoryBytes: number;
   gpuRecommended: boolean;
   sizeBytes: number;
@@ -11,19 +10,16 @@ export interface ModelRequirements {
 
 export const MODEL_REQUIREMENTS: Record<string, ModelRequirements> = {
   "granite3.2:2b": {
-    minMemoryBytes: 4 * GB,
-    recommendedMemoryBytes: 8 * GB,
+    recommendedMemoryBytes: 4 * GB,
     gpuRecommended: false,
     sizeBytes: Math.ceil(1.5 * GB),
   },
   "granite3.2:8b": {
-    minMemoryBytes: 12 * GB,
-    recommendedMemoryBytes: 16 * GB,
+    recommendedMemoryBytes: 10 * GB,
     gpuRecommended: true,
     sizeBytes: Math.ceil(4.9 * GB),
   },
   "nomic-embed-text:latest": {
-    minMemoryBytes: 2 * GB,
     recommendedMemoryBytes: 4 * GB,
     gpuRecommended: false,
     sizeBytes: Math.ceil(0.274 * GB),
@@ -38,58 +34,6 @@ interface ValidationResult {
   errors: string[];
 }
 
-export function checkModelCompatibility(
-  modelId: string | null,
-  systemInfo: SystemInfo | null,
-): ValidationResult {
-  if (!modelId || !systemInfo) {
-    return { isCompatible: true, warnings: [], errors: [] };
-  }
-
-  const requirements = MODEL_REQUIREMENTS[modelId];
-  if (!requirements) {
-    return { isCompatible: true, warnings: [], errors: [] };
-  }
-
-  const warnings: string[] = [];
-  const errors: string[] = [];
-  const totalMemoryBytes = systemInfo.memory.totalMemory;
-  const hasGoodGPU =
-    hasDiscreteGPU(systemInfo.gpus) || isHighEndApple(systemInfo.gpus);
-
-  // const freeDiskBytes = systemInfo.diskSpace.freeDiskSpace;
-  // if (freeDiskBytes < requirements.sizeBytes) {
-  //   errors.push(
-  //     `Insufficient disk space. Model needs ${formatSize(requirements.sizeBytes)}, ` +
-  //     `you only have ${formatSize(freeDiskBytes)} free.`
-  //   );
-  // }
-
-  if (totalMemoryBytes < requirements.minMemoryBytes) {
-    warnings.push(
-      `This model requires at least ${formatSize(requirements.minMemoryBytes)} of RAM. ` +
-        `Your system has ${formatSize(totalMemoryBytes)}.`,
-    );
-  } else if (totalMemoryBytes < requirements.recommendedMemoryBytes) {
-    warnings.push(
-      `This model runs best with ${formatSize(requirements.recommendedMemoryBytes)} of RAM. ` +
-        `Your system has ${formatSize(totalMemoryBytes)}.`,
-    );
-  }
-
-  if (requirements.gpuRecommended && !hasGoodGPU) {
-    warnings.push(
-      "This model performs better with a discrete NVidia or AMD GPU, or an Apple M2+ chip.",
-    );
-  }
-
-  return {
-    isCompatible: errors.length === 0,
-    warnings,
-    errors,
-  };
-}
-
 export function checkCombinedDiskSpace(
   selectedModels: string[],
   systemInfo: SystemInfo | null,
diff --git a/core/granite/commons/sysInfo.ts b/core/granite/commons/sysInfo.ts
index f55d7f8b5e6..cc270714b65 100644
--- a/core/granite/commons/sysInfo.ts
+++ b/core/granite/commons/sysInfo.ts
@@ -1,4 +1,7 @@
+import { DEFAULT_MODEL_GRANITE_LARGE } from "../../config/default";
+import { MODEL_REQUIREMENTS } from "./modelRequirements";
 import { GB } from "./sizeUtils";
+import { SYSTEM_REQUIREMENTS } from "./systemRequirements";
 
 // All the system's information
 export interface SystemInfo {
@@ -8,7 +11,7 @@ export interface SystemInfo {
   };
   cpus: number;
   diskSpace: DiskSpace;
-  gpus: { model: string; cores: number }[];
+  gpus: { model: string; cores: number; vram: number }[];
 }
 
 export interface GpuInfo {
@@ -69,3 +72,23 @@ export function getRecommendedModels(systemInfo: SystemInfo) {
     defaultEmbeddingsModel: "nomic-embed-text:latest",
   };
 }
+
+export function shouldRecommendLargeModel(systemInfo: SystemInfo): boolean {
+  const modelRequirements = MODEL_REQUIREMENTS[DEFAULT_MODEL_GRANITE_LARGE.model];
+  if (!modelRequirements) return false;
+
+  if (isHighEndApple(systemInfo.gpus)) {
+    const availableMemory =
+     systemInfo.memory.totalMemory - SYSTEM_REQUIREMENTS.reservedSystemMemory;
+    return availableMemory >= modelRequirements.recommendedMemoryBytes;
+  }
+
+  if (!hasDiscreteGPU(systemInfo.gpus)) return false;
+
+  const maxVRAM = systemInfo.gpus.reduce((max, gpu) => {
+    return Math.max(max, (gpu.vram || 0) * GB);
+  }, 0);
+
+  const availableVRAM = maxVRAM - SYSTEM_REQUIREMENTS.reservedGraphicsMemory;
+  return availableVRAM >= modelRequirements.recommendedMemoryBytes;
+}
diff --git a/core/granite/commons/systemRequirements.ts b/core/granite/commons/systemRequirements.ts
new file mode 100644
index 00000000000..58c14db69b4
--- /dev/null
+++ b/core/granite/commons/systemRequirements.ts
@@ -0,0 +1,11 @@
+import { GB } from "./sizeUtils";
+
+export interface SystemRequirements {
+  reservedSystemMemory: number;
+  reservedGraphicsMemory: number;
+}
+
+export const SYSTEM_REQUIREMENTS: SystemRequirements = {
+  reservedSystemMemory: 6 * GB,
+  reservedGraphicsMemory: 0 * GB,
+};
diff --git a/extensions/vscode/src/granite/utils/sysUtils.ts b/extensions/vscode/src/granite/utils/sysUtils.ts
index 288bc3fd49e..69b384942f9 100644
--- a/extensions/vscode/src/granite/utils/sysUtils.ts
+++ b/extensions/vscode/src/granite/utils/sysUtils.ts
@@ -31,13 +31,14 @@ export async function getSystemDiskSpace(): Promise<DiskSpace> {
 
 // Returns the system's available GPUs
 export async function getSystemGPUs(): Promise<
-  { model: string; cores: number }[]
+  { model: string; cores: number; vram: number }[]
 > {
   try {
     const gpus = await si.graphics();
     return gpus.controllers.map((controller) => ({
       model: controller.model,
       cores: controller.cores || 0,
+      vram: controller.vram || 0,
     }));
   } catch (error) {
     console.error("Error detecting GPUs:", error);
diff --git a/gui/src/granite/GraniteWizard.tsx b/gui/src/granite/GraniteWizard.tsx
index aa5f0e4c212..c154c730880 100644
--- a/gui/src/granite/GraniteWizard.tsx
+++ b/gui/src/granite/GraniteWizard.tsx
@@ -5,9 +5,11 @@ import {
   DEFAULT_MODEL_GRANITE_SMALL,
 } from "core/config/default";
 import { DEFAULT_MODEL_INFO } from "core/granite/commons/modelInfo";
+import { MODEL_REQUIREMENTS } from "core/granite/commons/modelRequirements";
 import { ProgressData } from "core/granite/commons/progressData";
+import { GB } from "core/granite/commons/sizeUtils";
 import { ModelStatus, ServerStatus } from "core/granite/commons/statuses";
-import { isHighEndMachine, SystemInfo } from "core/granite/commons/sysInfo";
+import { isHighEndApple, shouldRecommendLargeModel, SystemInfo } from "core/granite/commons/sysInfo";
 import { formatSize } from "core/granite/commons/textUtils";
 import {
   FINAL_STEP,
@@ -57,8 +59,8 @@ interface WizardContextProps {
   setInstallationModes: React.Dispatch<
     React.SetStateAction<InstallationMode[]>
   >;
-  recommendedModel: LocalModelSize;
-  setRecommendedModel: React.Dispatch<React.SetStateAction<LocalModelSize>>;
+  preselectedModel: LocalModelSize;
+  setPreselectedModel: React.Dispatch<React.SetStateAction<LocalModelSize>>;
   selectedModel: LocalModelSize;
   setSelectedModel: React.Dispatch<React.SetStateAction<LocalModelSize>>;
   statusByModel: Map<string, ModelStatus>;
@@ -119,10 +121,10 @@ export const WizardProvider: React.FC<WizardProviderProps> = ({ children }) => {
   const [installationModes, setInstallationModes] = useState<
     InstallationMode[]
   >([]);
-  const [recommendedModel, setRecommendedModel] =
+  const [preselectedModel, setPreselectedModel] =
     useState<LocalModelSize>("small");
   const [selectedModel, setSelectedModel] =
-    useState<LocalModelSize>(recommendedModel);
+    useState<LocalModelSize>(preselectedModel);
   const [modelInstallationProgress, setModelInstallationProgress] =
     useState<number>(0);
   const [modelInstallationError, setModelInstallationError] = useState<
@@ -153,8 +155,8 @@ export const WizardProvider: React.FC<WizardProviderProps> = ({ children }) => {
         setSystemInfo,
         installationModes,
         setInstallationModes,
-        recommendedModel,
-        setRecommendedModel,
+        preselectedModel,
+        setPreselectedModel,
         selectedModel,
         setSelectedModel,
         statusByModel,
@@ -372,7 +374,7 @@ const OllamaInstallStep: React.FC<StepProps> = (props) => {
 const ModelSelectionStep: React.FC<StepProps> = (props) => {
   const {
     serverStatus,
-    recommendedModel,
+    preselectedModel,
     selectedModel,
     setSelectedModel,
     modelInstallationProgress,
@@ -440,16 +442,25 @@ const ModelSelectionStep: React.FC<StepProps> = (props) => {
     }
   }, [systemInfo, selectedModel, statusByModel]);
 
+  const recommendedMemoryThreshold = MODEL_REQUIREMENTS[DEFAULT_MODEL_GRANITE_LARGE.model].recommendedMemoryBytes / GB;
   const modelOptions: ModelOption[] = [
     {
       key: "large",
       name: "Large",
-      description: "For machines with 32GB of memory and a fast GPU",
+      description: `For machines with ${recommendedMemoryThreshold}GB of ${
+        systemInfo && isHighEndApple(systemInfo.gpus)
+          ? "system memory"
+          : "video memory and a high-performance GPU"
+      }`,
     },
     {
       key: "small",
       name: "Small",
-      description: "For machines with less than 32GB of memory and a slow GPU",
+      description: `For machines with less than ${recommendedMemoryThreshold}GB of ${
+        systemInfo && isHighEndApple(systemInfo.gpus)
+          ? "system memory"
+          : "video memory and a lower-performance GPU"
+      }`,
     },
   ];
 
@@ -489,11 +500,21 @@ const ModelSelectionStep: React.FC<StepProps> = (props) => {
                         <RadioGroup.Description className="text-sm leading-normal text-[--vscode-editor-foreground] opacity-80">
                           {option.description}
                         </RadioGroup.Description>
-                        {option.key === recommendedModel && (
+                        {option.key === preselectedModel && option.key === "large" && (
                           <p className="text-sm leading-normal text-[--vscode-editorWarning-foreground,#ddb100]">
                             Recommended for your machine
                           </p>
                         )}
+                        {option.key === "large" && option.key !== preselectedModel && selectedModel === "large" && (
+                          <p className="text-sm leading-normal text-[--vscode-errorForeground]">
+                            Not recommended for your machine
+                          </p>
+                        )}
+                        {option.key === "small" && selectedModel === "small" && (
+                          <p className="text-sm leading-normal text-[--vscode-errorForeground]">
+                            Limited capabilities, for experimentation only
+                          </p>
+                        )}
                       </div>
                     </div>
                   )}
@@ -618,7 +639,7 @@ const WizardContent: React.FC = () => {
     setServerStatus,
     setSystemInfo,
     setInstallationModes,
-    setRecommendedModel,
+    setPreselectedModel,
     setSelectedModel,
     setModelInstallationProgress,
     setModelInstallationError,
@@ -651,22 +672,22 @@ const WizardContent: React.FC = () => {
           setInstallationModes(data.installModes);
           const sysinfo = data.systemInfo as SystemInfo;
           setSystemInfo(sysinfo);
-          const recommendedModel = isHighEndMachine(sysinfo)
+          const preselectedModel = shouldRecommendLargeModel(sysinfo)
             ? "large"
             : "small";
-          setRecommendedModel(recommendedModel);
+          setPreselectedModel(preselectedModel);
           const wizardState = data.wizardState as WizardState | undefined;
           if (wizardState) {
             if (wizardState?.selectedModelSize) {
               setSelectedModel(wizardState.selectedModelSize);
             } else {
-              setSelectedModel(recommendedModel);
+              setSelectedModel(preselectedModel);
             }
             if (wizardState?.stepStatuses) {
               setStepStatuses(wizardState.stepStatuses);
             }
           } else {
-            setSelectedModel(recommendedModel);
+            setSelectedModel(preselectedModel);
           }
 
           break;


Reviews:
user: login: owtaylor
body: Other than the threshold issue, looks good to me.
state: CHANGES_REQUESTED
author_association: NONE
submitted_at: 2025-03-18T19:22:34Z
user: login: halfline
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-19T11:25:03Z
user: login: halfline
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-19T11:38:07Z
user: login: halfline
state: COMMENTED
author_association: NONE
submitted_at: 2025-03-20T13:50:50Z

Review comments:
diff_hunk: @@ -69,3 +71,19 @@ export function getRecommendedModels(systemInfo: SystemInfo) {
     defaultEmbeddingsModel: "nomic-embed-text:latest",
   };
 }
+
+export function shouldRecommendLargeModel(systemInfo: SystemInfo): boolean {
+  const requirements = MODEL_REQUIREMENTS[DEFAULT_MODEL_GRANITE_LARGE.model];
+  if (!requirements) return false;
+
+  if (isHighEndApple(systemInfo.gpus))
+    return systemInfo.memory.totalMemory >= requirements.recommendedMemoryBytes;
+
+  if (!hasDiscreteGPU(systemInfo.gpus)) return false;
+
+  const maxVRAM = systemInfo.gpus.reduce((max, gpu) => {
+    return Math.max(max, (gpu.vram || 0) * GB);
+  }, 0);
+
+  return maxVRAM >= requirements.recommendedMemoryBytes;
path: core/granite/commons/sysInfo.ts
user: login: owtaylor
body: I don't think we should be using the same number for the VRAM threshold for a discrete GPU and the memory requirement for a Mac (or eventually other iGPU systems). 12GB of VRAM for a discrete GPU is quite reasonale for the :8b model. 12GB of RAM is not.
created_at: 2025-03-18T19:20:28Z
author_association: NONE
diff_hunk: @@ -69,3 +71,19 @@ export function getRecommendedModels(systemInfo: SystemInfo) {
     defaultEmbeddingsModel: "nomic-embed-text:latest",
   };
 }
+
+export function shouldRecommendLargeModel(systemInfo: SystemInfo): boolean {
+  const requirements = MODEL_REQUIREMENTS[DEFAULT_MODEL_GRANITE_LARGE.model];
+  if (!requirements) return false;
+
+  if (isHighEndApple(systemInfo.gpus))
+    return systemInfo.memory.totalMemory >= requirements.recommendedMemoryBytes;
+
+  if (!hasDiscreteGPU(systemInfo.gpus)) return false;
+
+  const maxVRAM = systemInfo.gpus.reduce((max, gpu) => {
+    return Math.max(max, (gpu.vram || 0) * GB);
+  }, 0);
+
+  return maxVRAM >= requirements.recommendedMemoryBytes;
path: core/granite/commons/sysInfo.ts
user: login: halfline
body: okay will add a 6GB offset for other tasks on system when using system memory
created_at: 2025-03-19T11:25:03Z
author_association: NONE
diff_hunk: @@ -69,3 +71,19 @@ export function getRecommendedModels(systemInfo: SystemInfo) {
     defaultEmbeddingsModel: "nomic-embed-text:latest",
   };
 }
+
+export function shouldRecommendLargeModel(systemInfo: SystemInfo): boolean {
+  const requirements = MODEL_REQUIREMENTS[DEFAULT_MODEL_GRANITE_LARGE.model];
+  if (!requirements) return false;
+
+  if (isHighEndApple(systemInfo.gpus))
+    return systemInfo.memory.totalMemory >= requirements.recommendedMemoryBytes;
+
+  if (!hasDiscreteGPU(systemInfo.gpus)) return false;
+
+  const maxVRAM = systemInfo.gpus.reduce((max, gpu) => {
+    return Math.max(max, (gpu.vram || 0) * GB);
+  }, 0);
+
+  return maxVRAM >= requirements.recommendedMemoryBytes;
path: core/granite/commons/sysInfo.ts
user: login: halfline
body: so truthfully the 8b model takes like 8gb of memory and we have it set like:

```
    minMemoryBytes: 12 * GB,
    recommendedMemoryBytes: 16 * GB,
```
So there's already a buffer built in that both discrete and mac hardware cases use, but:

- we only use recommendedMemoryBytes for both, and the buffer it has there isn't really right for either. I think we should drop minMemoryBytes and make recommendedMemoryBytes represent the actual usage of the model in memory, then have two model agnostic "reserve" numbers for amounts of the allocation of system and graphics memory that we won't encroach on when we do model selection.
created_at: 2025-03-19T11:38:07Z
author_association: NONE
diff_hunk: @@ -0,0 +1,11 @@
+import { GB } from "./sizeUtils";
+
+export interface SystemRequirements {
+  reservedSystemMemory: number;
+  reservedGraphicsMemory: number;
+}
+
+export const SYSTEM_REQUIREMENTS: SystemRequirements = {
+  reservedSystemMemory: 6 * GB,
+  reservedGraphicsMemory: 0 * GB,
path: core/granite/commons/systemRequirements.ts
user: login: halfline
body: Note i set this to 0 for now so medium end discrete cards still recommend the large model. we may need to tweak this value.
created_at: 2025-03-20T13:50:50Z
author_association: NONE

PR comments:
user: login: halfline
created_at: 2025-03-18T19:06:01Z
body: so trying it on mac it seems to work as designed, but looking closer my design tweak is a little wacky:

![image](https://github.com/user-attachments/assets/e956c7ea-3e80-442a-b0b9-fdbdf08f9ffd)

It recommends the small model for "less than 8 gb" and the large model for "more than 16 gb". So all those users with 12gb cards won't know what to do. probably just need to pick the bigger threshold.
author_association: NONE
user: login: halfline
created_at: 2025-03-18T19:08:03Z
body: the third commit message was improperly squashed as well. i need to fix that.
author_association: NONE
user: login: halfline
created_at: 2025-03-18T19:16:00Z
body: turns out IsHighEndMachine is used elsewhere in the code, so i dropped the commit that dropped that definition.
author_association: NONE
user: login: halfline
created_at: 2025-03-20T13:20:23Z
body: I think this is probably good to go now, but I want to check it on apple hardware before merging. Owen do you want to take a look at the new commits?
author_association: NONE
user: login: halfline
created_at: 2025-03-20T13:42:44Z
body: just did some quick testing on the mac mini with the updates and it seems to still properly recommend the large model.
author_association: NONE