Repository: skypilot-org/skypilot
Pull request #6218 (2025-07-10 10:45:44)

[Test] Read constant from package rather than source code file
<!-- Describe the changes in this PR -->

Fix the failure happened [here](https://buildkite.com/skypilot-1/quicktest-core/builds/628#0197f3e2-c0a7-436d-86cd-3f2318bde7d9)

The test reads the constant from the file, but if we're comparing with the previous PyPI package rather than source code, we don't have the constant.

<!-- Describe the tests ran -->
<!-- Unit tests (tests/test_*.py) are part of GitHub CI; below are tests that launch on the cloud. -->

Tested (run the relevant ones):

- [x] Code formatting: install pre-commit (auto-check on commit) or `bash format.sh`
- [ ] Any manual or new tests for this PR (please specify below)
- [ ] All smoke tests: `/smoke-test` (CI) or `pytest tests/test_smoke.py` (local)
- [x] Relevant individual tests: `/smoke-test -k test_name` (CI) or `pytest tests/test_smoke.py::test_name` (local)
- [ ] Backward compatibility: `/quicktest-core` (CI) or `pytest tests/smoke_tests/test_backward_compat.py` (local)

<!-- CI commands (/-prefixed) can only be triggered by repo members -->


zpoint: /quicktest-core
/quicktest-core --base-branch releases/0.9.3
/quicktest-core --base-branch pypi/skypilot-nightly
zpoint: /quicktest-core
/quicktest-core --base-branch releases/0.9.3
/quicktest-core --base-branch pypi/skypilot-nightly

Patch:
diff --git a/sky/core.py b/sky/core.py
index d34939bea..af59ac2f0 100644
--- a/sky/core.py
+++ b/sky/core.py
@@ -878,8 +878,10 @@ def cancel(
         f'handle for cluster {cluster_name!r} should not be None')
 
     backend = backend_utils.get_backend_from_handle(handle)
+    user_hash: Optional[str] = common_utils.get_current_user().id
 
     if all_users:
+        user_hash = None
         sky_logging.print(
             f'{colorama.Fore.YELLOW}'
             f'Cancelling all users\' jobs on cluster {cluster_name!r}...'
@@ -904,7 +906,7 @@ def cancel(
     backend.cancel_jobs(handle,
                         job_ids,
                         cancel_all=all or all_users,
-                        user_hash=common_utils.get_current_user().id)
+                        user_hash=user_hash)
 
 
 @usage_lib.entrypoint
diff --git a/sky/dashboard/src/components/clusters.jsx b/sky/dashboard/src/components/clusters.jsx
index be276c4bb..6559b4cbe 100755
--- a/sky/dashboard/src/components/clusters.jsx
+++ b/sky/dashboard/src/components/clusters.jsx
@@ -35,7 +35,7 @@ import { getClusters, getClusterHistory } from '@/data/connectors/clusters';
 import { getWorkspaces } from '@/data/connectors/workspaces';
 import { getUsers } from '@/data/connectors/users';
 import { sortData } from '@/data/utils';
-import { SquareCode, Terminal, RotateCwIcon } from 'lucide-react';
+import { SquareCode, Terminal, RotateCwIcon, Brackets } from 'lucide-react';
 import { relativeTime } from '@/components/utils';
 import { Layout } from '@/components/elements/layout';
 import {
@@ -66,25 +66,29 @@ import { UserDisplay } from '@/components/elements/UserDisplay';
 //   return cost.toFixed(2);
 // };
 
-const ALL_WORKSPACES_VALUE = '__ALL_WORKSPACES__'; // Define constant for "All Workspaces"
-
-// Define constant for "All Users" similar to workspaces
-const ALL_USERS_VALUE = '__ALL_USERS__';
-
-// Helper function to filter clusters by name
-export function filterClustersByName(clusters, nameFilter) {
-  // If no name filter, return all clusters
-  if (!nameFilter || nameFilter.trim() === '') {
-    return clusters;
-  }
-
-  // Filter clusters by the name filter (case-insensitive partial match)
-  const filterLower = nameFilter.toLowerCase().trim();
-  return clusters.filter((cluster) => {
-    const clusterName = cluster.cluster || '';
-    return clusterName.toLowerCase().includes(filterLower);
-  });
-}
+// Define filter options for the filter dropdown
+const PROPERTY_OPTIONS = [
+  {
+    label: 'Status',
+    value: 'status',
+  },
+  {
+    label: 'Cluster',
+    value: 'cluster',
+  },
+  {
+    label: 'User',
+    value: 'user',
+  },
+  {
+    label: 'Infra',
+    value: 'infra',
+  },
+  {
+    label: 'Workspace',
+    value: 'workspace',
+  },
+];
 
 // Helper function to format autostop information, similar to _get_autostop in CLI utils
 const formatAutostop = (autostop, toDown) => {
@@ -169,96 +173,24 @@ export function Clusters() {
   const [isSSHModalOpen, setIsSSHModalOpen] = useState(false);
   const [isVSCodeModalOpen, setIsVSCodeModalOpen] = useState(false);
   const [selectedCluster, setSelectedCluster] = useState(null);
-  const [workspaceFilter, setWorkspaceFilter] = useState(ALL_WORKSPACES_VALUE);
-  const [userFilter, setUserFilter] = useState(ALL_USERS_VALUE);
-  const [nameFilter, setNameFilter] = useState('');
-  const [workspaces, setWorkspaces] = useState([]);
-  const [users, setUsers] = useState([]);
   const [showHistory, setShowHistory] = useState(false); // 'active' or 'history'
   const isMobile = useMobile();
 
+  const [filters, setFilters] = useState([]);
+  const [optionValues, setOptionValues] = useState({
+    status: [],
+    cluster: [],
+    user: [],
+    workspace: [],
+    infra: [],
+  }); /// Option values for properties
+
   // Handle URL query parameters for workspace and user filtering
   useEffect(() => {
     if (router.isReady) {
-      if (router.query.workspace) {
-        const workspaceParam = Array.isArray(router.query.workspace)
-          ? router.query.workspace[0]
-          : router.query.workspace;
-        setWorkspaceFilter(workspaceParam);
-      }
-      if (router.query.user) {
-        const userParam = Array.isArray(router.query.user)
-          ? router.query.user[0]
-          : router.query.user;
-        setUserFilter(userParam);
-      }
-      if (router.query.name) {
-        const nameParam = Array.isArray(router.query.name)
-          ? router.query.name[0]
-          : router.query.name;
-        setNameFilter(nameParam);
-      }
+      updateFiltersByURLParams();
     }
-  }, [
-    router.isReady,
-    router.query.workspace,
-    router.query.user,
-    router.query.name,
-  ]);
-
-  // Helper function to update URL query parameters
-  const updateURLParams = (newWorkspace, newUser, newName) => {
-    const query = { ...router.query };
-
-    // Update workspace parameter
-    if (newWorkspace && newWorkspace !== ALL_WORKSPACES_VALUE) {
-      query.workspace = newWorkspace;
-    } else {
-      delete query.workspace;
-    }
-
-    // Update user parameter
-    if (newUser && newUser !== ALL_USERS_VALUE) {
-      query.user = newUser;
-    } else {
-      delete query.user;
-    }
-
-    // Update name parameter
-    if (newName && newName.trim() !== '') {
-      query.name = newName.trim();
-    } else {
-      delete query.name;
-    }
-
-    // Use replace to avoid adding to browser history for filter changes
-    router.replace(
-      {
-        pathname: router.pathname,
-        query,
-      },
-      undefined,
-      { shallow: true }
-    );
-  };
-
-  // Handle workspace filter change
-  const handleWorkspaceFilterChange = (newWorkspace) => {
-    setWorkspaceFilter(newWorkspace);
-    updateURLParams(newWorkspace, userFilter, nameFilter);
-  };
-
-  // Handle user filter change
-  const handleUserFilterChange = (newUser) => {
-    setUserFilter(newUser);
-    updateURLParams(workspaceFilter, newUser, nameFilter);
-  };
-
-  // Handle name filter change
-  const handleNameFilterChange = (newName) => {
-    setNameFilter(newName);
-    updateURLParams(workspaceFilter, userFilter, newName);
-  };
+  }, [router.isReady]);
 
   useEffect(() => {
     const fetchFilterData = async () => {
@@ -294,8 +226,6 @@ export function Clusters() {
           finalWorkspaces.add(wsName)
         );
 
-        setWorkspaces(Array.from(finalWorkspaces).sort());
-
         // Fetch users for the filter dropdown
         const fetchedUsers = await dashboardCache.get(getUsers);
         const uniqueClusterUsers = [
@@ -331,21 +261,85 @@ export function Clusters() {
             });
           }
         });
-
-        setUsers(
-          Array.from(finalUsers.values()).sort((a, b) =>
-            a.display.localeCompare(b.display)
-          )
-        );
       } catch (error) {
         console.error('Error fetching data for filters:', error);
-        setWorkspaces(['default']); // Fallback or error state
-        setUsers([]); // Fallback or error state
       }
     };
+
     fetchFilterData();
   }, []);
 
+  // Helper function to update URL query parameters
+  const updateURLParams = (filters) => {
+    const query = { ...router.query };
+
+    let properties = [];
+    let operators = [];
+    let values = [];
+
+    filters.map((filter, _index) => {
+      properties.push(filter.property.toLowerCase() ?? '');
+      operators.push(filter.operator);
+      values.push(filter.value);
+    });
+
+    query.property = properties;
+    query.operator = operators;
+    query.value = values;
+
+    // Use replace to avoid adding to browser history for filter changes
+    router.replace(
+      {
+        pathname: router.pathname,
+        query,
+      },
+      undefined,
+      { shallow: true }
+    );
+  };
+
+  const updateFiltersByURLParams = () => {
+    const query = { ...router.query };
+
+    const properties = query.property;
+    const operators = query.operator;
+    const values = query.value;
+
+    if (properties === undefined) {
+      return;
+    }
+
+    let filters = [];
+
+    const length = Array.isArray(properties) ? properties.length : 1;
+
+    const propertyMap = new Map();
+    propertyMap.set('', '');
+    propertyMap.set('status', 'Status');
+    propertyMap.set('cluster', 'Cluster');
+    propertyMap.set('user', 'User');
+    propertyMap.set('workspace', 'Workspace');
+    propertyMap.set('infra', 'Infra');
+
+    if (length === 1) {
+      filters.push({
+        property: propertyMap.get(properties),
+        operator: operators,
+        value: values,
+      });
+    } else {
+      for (let i = 0; i < length; i++) {
+        filters.push({
+          property: propertyMap.get(properties[i]),
+          operator: operators[i],
+          value: values[i],
+        });
+      }
+    }
+
+    setFilters(filters);
+  };
+
   const handleRefresh = () => {
     // Invalidate cache to ensure fresh data is fetched
     dashboardCache.invalidate(getClusters);
@@ -368,6 +362,7 @@ export function Clusters() {
           >
             Sky Clusters
           </Link>
+
           <div className="flex items-center ml-6 space-x-3">
             <label className="flex items-center cursor-pointer">
               <input
@@ -392,76 +387,14 @@ export function Clusters() {
               </span>
             </label>
           </div>
-          <div className="relative ml-4 mr-2">
-            <input
-              type="text"
-              placeholder="Filter by cluster name"
-              value={nameFilter}
-              onChange={(e) => handleNameFilterChange(e.target.value)}
-              className="h-8 w-32 sm:w-48 px-3 pr-8 text-sm border border-gray-300 rounded-md focus:ring-1 focus:ring-sky-500 focus:border-sky-500 outline-none"
-            />
-            {nameFilter && (
-              <button
-                onClick={() => handleNameFilterChange('')}
-                className="absolute right-2 top-1/2 transform -translate-y-1/2 text-gray-400 hover:text-gray-600"
-                title="Clear filter"
-              >
-                <svg
-                  className="h-4 w-4"
-                  fill="none"
-                  stroke="currentColor"
-                  viewBox="0 0 24 24"
-                >
-                  <path
-                    strokeLinecap="round"
-                    strokeLinejoin="round"
-                    strokeWidth={2}
-                    d="M6 18L18 6M6 6l12 12"
-                  />
-                </svg>
-              </button>
-            )}
-          </div>
-          <Select
-            value={workspaceFilter}
-            onValueChange={handleWorkspaceFilterChange}
-          >
-            <SelectTrigger className="h-8 w-48 ml-2 mr-2 text-sm border-none focus:ring-0 focus:outline-none">
-              <SelectValue placeholder="Filter by workspace...">
-                {workspaceFilter === ALL_WORKSPACES_VALUE
-                  ? 'All Workspaces'
-                  : workspaceFilter}
-              </SelectValue>
-            </SelectTrigger>
-            <SelectContent>
-              <SelectItem value={ALL_WORKSPACES_VALUE}>
-                All Workspaces
-              </SelectItem>
-              {workspaces.map((ws) => (
-                <SelectItem key={ws} value={ws}>
-                  {ws}
-                </SelectItem>
-              ))}
-            </SelectContent>
-          </Select>
-          <Select value={userFilter} onValueChange={handleUserFilterChange}>
-            <SelectTrigger className="h-8 w-48 ml-2 mr-2 text-sm border-none focus:ring-0 focus:outline-none">
-              <SelectValue placeholder="Filter by user...">
-                {userFilter === ALL_USERS_VALUE
-                  ? 'All Users'
-                  : users.find((u) => u.userId === userFilter)?.display ||
-                    userFilter}
-              </SelectValue>
-            </SelectTrigger>
-            <SelectContent>
-              <SelectItem value={ALL_USERS_VALUE}>All Users</SelectItem>
-              {users.map((user) => (
-                <SelectItem key={user.userId} value={user.userId}>
-                  {user.display}
-                </SelectItem>
-              ))}
-            </SelectContent>
-          </Select>
+
+          <FilterDropdown
+            propertyList={PROPERTY_OPTIONS}
+            valueList={optionValues}
+            setFilters={setFilters}
+            updateURLParams={updateURLParams}
+            placeholder="Filter clusters"
+          />
         </div>
         <div className="flex items-center">
           {loading && (
@@ -480,13 +413,18 @@ export function Clusters() {
           </button>
         </div>
       </div>
+
+      <Filters
+        filters={filters}
+        setFilters={setFilters}
+        updateURLParams={updateURLParams}
+      />
+
       <ClusterTable
         refreshInterval={REFRESH_INTERVAL}
         setLoading={setLoading}
         refreshDataRef={refreshDataRef}
-        workspaceFilter={workspaceFilter}
-        userFilter={userFilter}
-        nameFilter={nameFilter}
+        filters={filters}
         showHistory={showHistory}
         onOpenSSHModal={(cluster) => {
           setSelectedCluster(cluster);
@@ -496,6 +434,7 @@ export function Clusters() {
           setSelectedCluster(cluster);
           setIsVSCodeModalOpen(true);
         }}
+        setOptionValues={setOptionValues}
       />
 
       {/* SSH Instructions Modal */}
@@ -518,12 +457,11 @@ export function ClusterTable({
   refreshInterval,
   setLoading,
   refreshDataRef,
-  workspaceFilter,
-  userFilter,
-  nameFilter,
+  filters,
   showHistory,
   onOpenSSHModal,
   onOpenVSCodeModal,
+  setOptionValues,
 }) {
   const [data, setData] = useState([]);
   const [sortConfig, setSortConfig] = useState({
@@ -535,6 +473,32 @@ export function ClusterTable({
   const [currentPage, setCurrentPage] = useState(1);
   const [pageSize, setPageSize] = useState(10);
 
+  const fetchOptionValuesFromClusters = (clusters) => {
+    let optionValues = {
+      status: [],
+      cluster: [],
+      user: [],
+      workspace: [],
+      infra: [],
+    };
+
+    const pushWithoutDuplication = (array, item) => {
+      if (array.includes(item)) return;
+
+      array.push(item);
+    };
+
+    clusters.map((cluster) => {
+      pushWithoutDuplication(optionValues.status, cluster.status);
+      pushWithoutDuplication(optionValues.cluster, cluster.cluster);
+      pushWithoutDuplication(optionValues.user, cluster.user);
+      pushWithoutDuplication(optionValues.workspace, cluster.workspace);
+      pushWithoutDuplication(optionValues.infra, cluster.infra);
+    });
+
+    return optionValues;
+  };
+
   const fetchData = React.useCallback(async () => {
     setLoading(true);
     setLocalLoading(true);
@@ -565,6 +529,9 @@ export function ClusterTable({
             combinedData.push(histCluster);
           }
         });
+
+        setOptionValues(fetchOptionValuesFromClusters(combinedData));
+
         setData(combinedData);
       } else {
         // Mark active clusters for consistency
@@ -572,10 +539,14 @@ export function ClusterTable({
           ...cluster,
           isHistorical: false,
         }));
+
+        setOptionValues(fetchOptionValuesFromClusters(markedActiveClusters));
+
         setData(markedActiveClusters);
       }
     } catch (error) {
       console.error('Error fetching cluster data:', error);
+      setOptionValues(fetchOptionValuesFromClusters([]));
       setData([]);
     }
 
@@ -584,29 +555,63 @@ export function ClusterTable({
     setIsInitialLoad(false);
   }, [setLoading, showHistory]);
 
+  // Utility: checks a condition based on operator
+  const evaluateCondition = (item, filter) => {
+    const { property, operator, value } = filter;
+
+    if (!value) return true; // skip empty filters
+
+    // Global search: check all values
+    if (!property) {
+      const strValue = value.toLowerCase();
+      return Object.values(item).some((val) =>
+        val?.toString().toLowerCase().includes(strValue)
+      );
+    }
+
+    const itemValue = item[property.toLowerCase()]?.toString().toLowerCase();
+    const filterValue = value.toString().toLowerCase();
+
+    switch (operator) {
+      case '=':
+        return itemValue === filterValue;
+      case ':':
+        return itemValue?.includes(filterValue);
+      default:
+        return true;
+    }
+  };
+
   // Use useMemo to compute sorted data
   const sortedData = React.useMemo(() => {
-    let filteredData = data;
-    // Filter by workspace if workspaceFilter is set and not 'ALL_WORKSPACES_VALUE'
-    if (workspaceFilter && workspaceFilter !== ALL_WORKSPACES_VALUE) {
-      filteredData = filteredData.filter((item) => {
-        const itemWorkspace = item.workspace || 'default'; // Treat missing/empty workspace as 'default'
-        return itemWorkspace.toLowerCase() === workspaceFilter.toLowerCase();
-      });
-    }
-    // Filter by user if userFilter is set and not 'ALL_USERS_VALUE'
-    if (userFilter && userFilter !== ALL_USERS_VALUE) {
-      filteredData = filteredData.filter((item) => {
-        const itemUserId = item.user_hash || item.user;
-        return itemUserId === userFilter;
+    // Main filter function
+    const filterData = (data, filters) => {
+      if (filters.length === 0) {
+        return data;
+      }
+
+      return data.filter((item) => {
+        let result = null;
+
+        for (let i = 0; i < filters.length; i++) {
+          const filter = filters[i];
+          const current = evaluateCondition(item, filter);
+
+          if (result === null) {
+            result = current;
+          } else {
+            result = result && current;
+          }
+        }
+
+        return result;
       });
-    }
-    // Filter by name if nameFilter is set
-    if (nameFilter) {
-      filteredData = filterClustersByName(filteredData, nameFilter);
-    }
+    };
+
+    const filteredData = filterData(data, filters);
+
     return sortData(filteredData, sortConfig.key, sortConfig.direction);
-  }, [data, sortConfig, workspaceFilter, userFilter, nameFilter]);
+  }, [data, sortConfig, filters]);
 
   // Expose fetchData to parent component
   React.useEffect(() => {
@@ -893,8 +898,7 @@ export function ClusterTable({
               </div>
             </div>
             <div>
-              {startIndex + 1} – {Math.min(endIndex, data.length)} of{' '}
-              {data.length}
+              {`${startIndex + 1} - ${Math.min(endIndex, sortedData.length)} of ${sortedData.length}`}
             </div>
             <div className="flex items-center space-x-2">
               <Button
@@ -1059,3 +1063,256 @@ export function Status2Actions({
     </>
   );
 }
+
+const FilterDropdown = ({
+  propertyList = [],
+  valueList,
+  setFilters,
+  updateURLParams,
+  placeholder = 'Filter clusters',
+}) => {
+  const inputRef = useRef(null);
+
+  const [isOpen, setIsOpen] = useState(false);
+  const [value, setValue] = useState('');
+  const [propertyValue, setPropertValue] = useState('cluster');
+  const [valueOptions, setValueOptions] = useState([]);
+
+  let filteredOptions = [];
+
+  useEffect(() => {
+    let updatedValueOptions = [];
+
+    switch (propertyValue) {
+      case 'status':
+        updatedValueOptions = valueList.status;
+        break;
+
+      case 'user':
+        updatedValueOptions = valueList.user;
+        break;
+
+      case 'cluster':
+        updatedValueOptions = valueList.cluster;
+        break;
+
+      case 'workspace':
+        updatedValueOptions = valueList.workspace;
+        break;
+
+      case 'infra':
+        updatedValueOptions = valueList.infra;
+        break;
+
+      default:
+        break;
+    }
+
+    updatedValueOptions = updatedValueOptions.filter((item) =>
+      item.includes(value.toLowerCase())
+    );
+
+    setValueOptions(updatedValueOptions);
+  }, [propertyValue, valueList, value]);
+
+  const handleValueChange = (e) => {
+    setValue(e.target.value);
+  };
+
+  return (
+    <>
+      <div className="flex flex-row ml-4 mr-2 border border-gray-300 rounded-md">
+        <div className="border-r">
+          <Select
+            onValueChange={(value) => setPropertValue(value)}
+            value={propertyValue}
+          >
+            <SelectTrigger
+              aria-label="Node"
+              className="focus:ring-0 focus:ring-offset-0 border-none w-36"
+            >
+              <SelectValue placeholder="Select Property" />
+            </SelectTrigger>
+            <SelectContent>
+              {propertyList.map((item, index) => (
+                <SelectItem key={`property-item-${index}`} value={item.value}>
+                  {item.label}
+                </SelectItem>
+              ))}
+            </SelectContent>
+          </Select>
+        </div>
+        <div className="relative">
+          <input
+            type="text"
+            ref={inputRef}
+            placeholder={placeholder}
+            value={value}
+            onChange={(e) => handleValueChange(e)}
+            onFocus={() => setIsOpen(true)}
+            onBlur={() => setIsOpen(false)}
+            onKeyDown={(e) => {
+              if (e.key === 'Enter' && value !== '') {
+                setFilters((prevFilters) => {
+                  const updatedFilters = [
+                    ...prevFilters,
+                    {
+                      property: propertyValue,
+                      operator: ':',
+                      value: value,
+                    },
+                  ];
+
+                  updateURLParams(updatedFilters);
+
+                  return updatedFilters;
+                });
+                setValue('');
+                setIsOpen(false);
+                inputRef.current.blur();
+              }
+            }}
+            className="h-10 w-32 sm:w-96 px-3 pr-8 text-sm rounded-md outline-none my-auto"
+            autoComplete="off"
+          />
+          {value && (
+            <button
+              onClick={() => {
+                setValue('');
+              }}
+              className="absolute right-2 top-1/2 transform -translate-y-1/2 text-gray-400 hover:text-gray-600"
+              title="Clear filter"
+              tabIndex={-1}
+            >
+              <svg
+                className="h-4 w-4"
+                fill="none"
+                stroke="currentColor"
+                viewBox="0 0 24 24"
+              >
+                <path
+                  strokeLinecap="round"
+                  strokeLinejoin="round"
+                  strokeWidth={2}
+                  d="M6 18L18 6M6 6l12 12"
+                />
+              </svg>
+            </button>
+          )}
+          {isOpen && valueOptions.length > 0 && (
+            <div className="flex flex-col absolute z-10 mt-1 w-full bg-white border border-gray-200 rounded-md shadow-lg">
+              {valueOptions.map((option, index) => (
+                <div
+                  key={option}
+                  className={`flex flex-col pl-7 py-2 cursor-pointer hover:bg-sky-50 text-sm ${index != filteredOptions.length - 1} && border-b`}
+                  onMouseDown={(e) => {
+                    e.preventDefault();
+                    setFilters((prevFilters) => {
+                      const updatedFilters = [
+                        ...prevFilters,
+                        {
+                          property: propertyValue,
+                          operator: '=',
+                          value: option,
+                        },
+                      ];
+
+                      updateURLParams(updatedFilters);
+
+                      return updatedFilters;
+                    });
+                    setIsOpen(false);
+                    setValue('');
+                    inputRef.current.blur();
+                  }}
+                >
+                  <span className="text-sm text-gray-600">{option}</span>
+                </div>
+              ))}
+            </div>
+          )}
+        </div>
+      </div>
+    </>
+  );
+};
+
+const Filters = ({ filters = [], setFilters, updateURLParams }) => {
+  const onRemove = (index) => {
+    setFilters((prevFilters) => {
+      const updatedFilters = prevFilters.filter(
+        (_, _index) => _index !== index
+      );
+
+      updateURLParams(updatedFilters);
+
+      return updatedFilters;
+    });
+  };
+
+  const clearFilters = () => {
+    updateURLParams([]);
+    setFilters([]);
+  };
+
+  return (
+    <>
+      <div className="flex items-center gap-4 p-2">
+        <div className="flex flex-wrap items-content gap-2">
+          {filters.map((filter, _index) => (
+            <FilterItem
+              key={`filteritem-${_index}`}
+              filter={filter}
+              onRemove={() => onRemove(_index)}
+            />
+          ))}
+
+          {filters.length > 0 && (
+            <>
+              <div className="border border-gray-400 h-8"></div>
+              <button
+                onClick={clearFilters}
+                className="rounded-full px-4 py-1 text-blue-600 bg-blue-100 hover:bg-blue-200"
+              >
+                Clear filters
+              </button>
+            </>
+          )}
+        </div>
+      </div>
+    </>
+  );
+};
+
+const FilterItem = ({ filter, onRemove }) => {
+  return (
+    <>
+      <div className="flex items-center gap-2 text-blue-600 bg-blue-100 px-2 rounded-full">
+        <div className="flex items-center gap-1 px-2 py-1 rounded-l-lg">
+          <span>{`${filter.property} `}</span>
+          <span className="font-bold">{`${filter.operator} ${filter.value}`}</span>
+        </div>
+
+        <button
+          onClick={() => onRemove()}
+          className="p-1 transform text-gray-400 hover:text-gray-600 bg-blue-500 hover:bg-blue-600 rounded-full flex flex-col items-center"
+          title="Clear filter"
+        >
+          <svg
+            className="h-3 w-3"
+            fill="none"
+            stroke="white"
+            viewBox="0 0 24 24"
+          >
+            <path
+              strokeLinecap="round"
+              strokeLinejoin="round"
+              strokeWidth={5}
+              d="M6 18L18 6M6 6l12 12"
+            />
+          </svg>
+        </button>
+      </div>
+    </>
+  );
+};
diff --git a/sky/global_user_state.py b/sky/global_user_state.py
index 6559b6a0c..14543ef75 100644
--- a/sky/global_user_state.py
+++ b/sky/global_user_state.py
@@ -220,17 +220,16 @@ def _glob_to_similar(glob_pattern):
     return like_pattern
 
 
-def create_table():
+def create_table(engine: sqlalchemy.engine.Engine):
     # Enable WAL mode to avoid locking issues.
     # See: issue #1441 and PR #1509
     # https://github.com/microsoft/WSL/issues/2395
     # TODO(romilb): We do not enable WAL for WSL because of known issue in WSL.
     #  This may cause the database locked problem from WSL issue #1441.
-    if (_SQLALCHEMY_ENGINE.dialect.name
-            == db_utils.SQLAlchemyDialect.SQLITE.value and
+    if (engine.dialect.name == db_utils.SQLAlchemyDialect.SQLITE.value and
             not common_utils.is_wsl()):
         try:
-            with orm.Session(_SQLALCHEMY_ENGINE) as session:
+            with orm.Session(engine) as session:
                 session.execute(sqlalchemy.text('PRAGMA journal_mode=WAL'))
                 session.commit()
         except sqlalchemy_exc.OperationalError as e:
@@ -240,12 +239,12 @@ def create_table():
             # is not critical and is likely to be enabled by other processes.
 
     # Create tables if they don't exist
-    db_utils.add_tables_to_db_sqlalchemy(Base.metadata, _SQLALCHEMY_ENGINE)
+    db_utils.add_tables_to_db_sqlalchemy(Base.metadata, engine)
 
     # For backward compatibility.
     # TODO(zhwu): Remove this function after all users have migrated to
     # the latest version of SkyPilot.
-    with orm.Session(_SQLALCHEMY_ENGINE) as session:
+    with orm.Session(engine) as session:
         # Add autostop column to clusters table
         db_utils.add_column_to_table_sqlalchemy(session,
                                                 'clusters',
@@ -391,15 +390,15 @@ def initialize_and_get_db() -> sqlalchemy.engine.Engine:
                 conn_string = skypilot_config.get_nested(('db',), None)
             if conn_string:
                 logger.debug(f'using db URI from {conn_string}')
-                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine(
-                    conn_string, poolclass=sqlalchemy.NullPool)
+                engine = sqlalchemy.create_engine(conn_string,
+                                                  poolclass=sqlalchemy.NullPool)
             else:
                 db_path = os.path.expanduser('~/.sky/state.db')
                 pathlib.Path(db_path).parents[0].mkdir(parents=True,
                                                        exist_ok=True)
-                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine('sqlite:///' +
-                                                              db_path)
-            create_table()
+                engine = sqlalchemy.create_engine('sqlite:///' + db_path)
+            create_table(engine)
+            _SQLALCHEMY_ENGINE = engine
     return _SQLALCHEMY_ENGINE
 
 
diff --git a/sky/jobs/state.py b/sky/jobs/state.py
index aac6ca82f..5e437686f 100644
--- a/sky/jobs/state.py
+++ b/sky/jobs/state.py
@@ -112,17 +112,16 @@ ha_recovery_script_table = sqlalchemy.Table(
 )
 
 
-def create_table():
+def create_table(engine: sqlalchemy.engine.Engine):
     # Enable WAL mode to avoid locking issues.
     # See: issue #3863, #1441 and PR #1509
     # https://github.com/microsoft/WSL/issues/2395
     # TODO(romilb): We do not enable WAL for WSL because of known issue in WSL.
     #  This may cause the database locked problem from WSL issue #1441.
-    if (_SQLALCHEMY_ENGINE.dialect.name
-            == db_utils.SQLAlchemyDialect.SQLITE.value and
+    if (engine.dialect.name == db_utils.SQLAlchemyDialect.SQLITE.value and
             not common_utils.is_wsl()):
         try:
-            with orm.Session(_SQLALCHEMY_ENGINE) as session:
+            with orm.Session(engine) as session:
                 session.execute(sqlalchemy.text('PRAGMA journal_mode=WAL'))
                 session.commit()
         except sqlalchemy_exc.OperationalError as e:
@@ -132,10 +131,10 @@ def create_table():
             # is not critical and is likely to be enabled by other processes.
 
     # Create tables if they don't exist
-    db_utils.add_tables_to_db_sqlalchemy(Base.metadata, _SQLALCHEMY_ENGINE)
+    db_utils.add_tables_to_db_sqlalchemy(Base.metadata, engine)
 
     # Backward compatibility: add columns that not exist in older databases
-    with orm.Session(_SQLALCHEMY_ENGINE) as session:
+    with orm.Session(engine) as session:
         db_utils.add_column_to_table_sqlalchemy(session, 'spot',
                                                 'failure_reason',
                                                 sqlalchemy.Text())
@@ -228,15 +227,15 @@ def initialize_and_get_db() -> sqlalchemy.engine.Engine:
                 conn_string = skypilot_config.get_nested(('db',), None)
             if conn_string:
                 logger.debug(f'using db URI from {conn_string}')
-                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine(
-                    conn_string, poolclass=sqlalchemy.NullPool)
+                engine = sqlalchemy.create_engine(conn_string,
+                                                  poolclass=sqlalchemy.NullPool)
             else:
                 db_path = os.path.expanduser('~/.sky/spot_jobs.db')
                 pathlib.Path(db_path).parents[0].mkdir(parents=True,
                                                        exist_ok=True)
-                _SQLALCHEMY_ENGINE = sqlalchemy.create_engine('sqlite:///' +
-                                                              db_path)
-            create_table()
+                engine = sqlalchemy.create_engine('sqlite:///' + db_path)
+            create_table(engine)
+            _SQLALCHEMY_ENGINE = engine
     return _SQLALCHEMY_ENGINE
 
 
diff --git a/sky/jobs/utils.py b/sky/jobs/utils.py
index 76c8adca3..b0c9044d2 100644
--- a/sky/jobs/utils.py
+++ b/sky/jobs/utils.py
@@ -30,6 +30,7 @@ from sky.backends import backend_utils
 from sky.jobs import constants as managed_job_constants
 from sky.jobs import scheduler
 from sky.jobs import state as managed_job_state
+from sky.server import common as server_common
 from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.skylet import log_lib
@@ -38,6 +39,7 @@ from sky.utils import annotations
 from sky.utils import command_runner
 from sky.utils import common_utils
 from sky.utils import controller_utils
+from sky.utils import env_options
 from sky.utils import infra_utils
 from sky.utils import log_utils
 from sky.utils import message_utils
@@ -128,9 +130,15 @@ def terminate_cluster(cluster_name: str, max_retry: int = 6) -> None:
             time.sleep(backoff.current_backoff())
 
 
-def _check_consolidation_mode_consistency(
+def _validate_consolidation_mode_config(
         current_is_consolidation_mode: bool) -> None:
-    """Check the consistency of the consolidation mode."""
+    """Validate the consolidation mode config."""
+    if (current_is_consolidation_mode and
+            not env_options.Options.IS_DEVELOPER.get() and
+            server_common.is_api_server_local()):
+        with ux_utils.print_exception_no_traceback():
+            raise exceptions.NotSupportedError(
+                'Consolidation mode is not supported when running locally.')
     # Check whether the consolidation mode config is changed.
     if current_is_consolidation_mode:
         controller_cn = (
@@ -176,7 +184,7 @@ def _check_consolidation_mode_consistency(
 def is_consolidation_mode() -> bool:
     consolidation_mode = skypilot_config.get_nested(
         ('jobs', 'controller', 'consolidation_mode'), default_value=False)
-    _check_consolidation_mode_consistency(consolidation_mode)
+    _validate_consolidation_mode_config(consolidation_mode)
     return consolidation_mode
 
 
diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py
index bb9907864..04c7f739b 100644
--- a/sky/skypilot_config.py
+++ b/sky/skypilot_config.py
@@ -52,6 +52,7 @@ import contextlib
 import copy
 import json
 import os
+import pathlib
 import tempfile
 import threading
 import typing
@@ -848,7 +849,9 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
 
     global_config_path = _resolve_server_config_path()
     if global_config_path is None:
-        global_config_path = get_user_config_path()
+        # Fallback to ~/.sky/config.yaml, and make sure it exists.
+        global_config_path = os.path.expanduser(get_user_config_path())
+        pathlib.Path(global_config_path).touch(exist_ok=True)
 
     db_updated = False
     if os.environ.get(constants.ENV_VAR_IS_SKYPILOT_SERVER) is not None:
diff --git a/sky/users/permission.py b/sky/users/permission.py
index 9c7b2fdc6..87949f234 100644
--- a/sky/users/permission.py
+++ b/sky/users/permission.py
@@ -34,6 +34,11 @@ class PermissionService:
     """Permission service for SkyPilot API Server."""
 
     def __init__(self):
+        self.enforcer = None
+
+    def _lazy_initialize(self):
+        if self.enforcer is not None:
+            return
         with _policy_lock():
             global _enforcer_instance
             if _enforcer_instance is None:
@@ -73,7 +78,6 @@ class PermissionService:
 
     def _maybe_initialize_policies(self) -> None:
         """Initialize policies if they don't already exist."""
-        # TODO(zhwu): we should avoid running this on client side.
         logger.debug(f'Initializing policies in process: {os.getpid()}')
         self._load_policy_no_lock()
 
@@ -152,6 +156,7 @@ class PermissionService:
 
     def add_user_if_not_exists(self, user_id: str) -> None:
         """Add user role relationship."""
+        self._lazy_initialize()
         with _policy_lock():
             self._add_user_if_not_exists_no_lock(user_id)
 
@@ -171,6 +176,7 @@ class PermissionService:
 
     def delete_user(self, user_id: str) -> None:
         """Delete user role relationship."""
+        self._lazy_initialize()
         with _policy_lock():
             # Get current roles
             self._load_policy_no_lock()
@@ -184,6 +190,7 @@ class PermissionService:
 
     def update_role(self, user_id: str, new_role: str) -> None:
         """Update user role relationship."""
+        self._lazy_initialize()
         with _policy_lock():
             # Get current roles
             self._load_policy_no_lock()
@@ -216,6 +223,7 @@ class PermissionService:
         Returns:
             A list of role names that the user has.
         """
+        self._lazy_initialize()
         self._load_policy_no_lock()
         return self.enforcer.get_roles_for_user(user_id)
 
@@ -228,6 +236,7 @@ class PermissionService:
         # it is a hot path in every request. It is ok to have a stale policy,
         # as long as it is eventually consistent.
         # self._load_policy_no_lock()
+        self._lazy_initialize()
         return self.enforcer.enforce(user_id, path, method)
 
     def _load_policy_no_lock(self):
@@ -236,6 +245,7 @@ class PermissionService:
 
     def load_policy(self):
         """Load policy from storage with lock."""
+        self._lazy_initialize()
         with _policy_lock():
             self._load_policy_no_lock()
 
@@ -251,6 +261,7 @@ class PermissionService:
         For public workspaces, the permission is granted via a wildcard policy
         ('*').
         """
+        self._lazy_initialize()
         if os.getenv(constants.ENV_VAR_IS_SKYPILOT_SERVER) is None:
             # When it is not on API server, we allow all users to access all
             # workspaces, as the workspace check has been done on API server.
@@ -307,6 +318,7 @@ class PermissionService:
                    For public workspaces, this should be ['*'].
                    For private workspaces, this should be specific user IDs.
         """
+        self._lazy_initialize()
         with _policy_lock():
             for user in users:
                 logger.debug(f'Adding workspace policy: user={user}, '
@@ -324,6 +336,7 @@ class PermissionService:
                    For public workspaces, this should be ['*'].
                    For private workspaces, this should be specific user IDs.
         """
+        self._lazy_initialize()
         with _policy_lock():
             self._load_policy_no_lock()
             # Remove all existing policies for this workspace
@@ -337,6 +350,7 @@ class PermissionService:
 
     def remove_workspace_policy(self, workspace_name: str) -> None:
         """Remove workspace policy."""
+        self._lazy_initialize()
         with _policy_lock():
             self.enforcer.remove_filtered_policy(1, workspace_name)
             self.enforcer.save_policy()
diff --git a/sky/users/token_service.py b/sky/users/token_service.py
index 5935c94f1..e97670f9e 100644
--- a/sky/users/token_service.py
+++ b/sky/users/token_service.py
@@ -48,8 +48,8 @@ class TokenService:
 
     def _get_or_generate_secret(self) -> str:
         """Get JWT secret from database or generate a new one."""
-        with _jwt_secret_lock():
-            # Try to get from database (persistent across deployments)
+
+        def _get_secret_from_db():
             try:
                 db_secret = global_user_state.get_system_config(
                     JWT_SECRET_DB_KEY)
@@ -58,7 +58,17 @@ class TokenService:
                     return db_secret
             except Exception as e:  # pylint: disable=broad-except
                 logger.debug(f'Failed to get JWT secret from database: {e}')
+            return None
 
+        # Try to get from database (persistent across deployments)
+        token_from_db = _get_secret_from_db()
+        if token_from_db:
+            return token_from_db
+
+        with _jwt_secret_lock():
+            token_from_db = _get_secret_from_db()
+            if token_from_db:
+                return token_from_db
             # Generate a new secret and store in database
             new_secret = secrets.token_urlsafe(64)
             try:


Test patch:
diff --git a/tests/smoke_tests/backward_compat/test_backward_compat.py b/tests/smoke_tests/backward_compat/test_backward_compat.py
index 1ec2fd3df..8d4316af6 100644
--- a/tests/smoke_tests/backward_compat/test_backward_compat.py
+++ b/tests/smoke_tests/backward_compat/test_backward_compat.py
@@ -66,20 +66,29 @@ class TestBackwardCompatibility:
             job_name=job_name, job_status=status, timeout=300)
 
     def _get_base_skylet_version(self) -> str:
-        """Get SKYLET_VERSION from the base environment directory."""
-        base_constants_file = self.BASE_SKY_DIR / 'sky' / 'skylet' / 'constants.py'
-        try:
-            with open(base_constants_file, 'r') as f:
-                content = f.read()
-                # Find the SKYLET_VERSION line
-                for line in content.split('\n'):
-                    if line.strip().startswith('SKYLET_VERSION = '):
-                        # Extract version string between quotes
-                        version = line.split('=')[1].strip().strip('\'"')
-                        return version
-            raise ValueError('SKYLET_VERSION not found in base constants.py')
-        except Exception as e:
-            raise ValueError(f'Failed to read base SKYLET_VERSION: {e}') from e
+        """Get SKYLET_VERSION from the base environment by running Python code."""
+        base_skylet_version = subprocess.run(
+            f'{self.ACTIVATE_BASE} && python -c "'
+            'from sky.skylet import constants; '
+            'print(constants.SKYLET_VERSION);'
+            '"',
+            shell=True,
+            check=False,
+            executable='/bin/bash',
+            text=True,
+            capture_output=True)
+
+        if base_skylet_version.returncode != 0:
+            raise ValueError(f'Failed to get base SKYLET_VERSION. '
+                             f'Return code: {base_skylet_version.returncode}, '
+                             f'stderr: {base_skylet_version.stderr}, '
+                             f'stdout: {base_skylet_version.stdout}')
+
+        version = base_skylet_version.stdout.strip().split('\n')[-1]
+        if not version:
+            raise ValueError('SKYLET_VERSION is empty')
+
+        return version
 
     @pytest.fixture(scope="session", autouse=True)
     def session_setup(self, request):
diff --git a/tests/test_failover.py b/tests/test_failover.py
index 404088873..1f8a8e09c 100644
--- a/tests/test_failover.py
+++ b/tests/test_failover.py
@@ -28,7 +28,7 @@ def _mock_db_conn(tmp_path, monkeypatch):
     monkeypatch.setattr(global_user_state, '_SQLALCHEMY_ENGINE',
                         sqlalchemy_engine)
 
-    global_user_state.create_table()
+    global_user_state.create_table(sqlalchemy_engine)
 
 
 @pytest.mark.parametrize('enable_all_clouds', [[sky.AWS()]], indirect=True)
diff --git a/tests/test_jobs.py b/tests/test_jobs.py
index f2e337297..1136d9878 100644
--- a/tests/test_jobs.py
+++ b/tests/test_jobs.py
@@ -19,7 +19,7 @@ def _mock_db_conn(tmp_path, monkeypatch):
     monkeypatch.setattr(global_user_state, '_SQLALCHEMY_ENGINE',
                         sqlalchemy_engine)
 
-    global_user_state.create_table()
+    global_user_state.create_table(sqlalchemy_engine)
 
 
 @pytest.fixture
diff --git a/tests/test_jobs_and_serve.py b/tests/test_jobs_and_serve.py
index d13431ef2..e9e8f2dfa 100644
--- a/tests/test_jobs_and_serve.py
+++ b/tests/test_jobs_and_serve.py
@@ -47,7 +47,7 @@ def _mock_db_conn(tmp_path, monkeypatch):
     monkeypatch.setattr(global_user_state, '_SQLALCHEMY_ENGINE',
                         sqlalchemy_engine)
 
-    global_user_state.create_table()
+    global_user_state.create_table(sqlalchemy_engine)
 
 
 def _generate_tmp_yaml(tmp_path, filename: str) -> str:
diff --git a/tests/unit_tests/test_core.py b/tests/unit_tests/test_core.py
new file mode 100644
index 000000000..65eb075e2
--- /dev/null
+++ b/tests/unit_tests/test_core.py
@@ -0,0 +1,35 @@
+from unittest import mock
+
+from sky.backends.cloud_vm_ray_backend import CloudVmRayResourceHandle
+from sky.core import cancel
+from sky.utils import common_utils
+
+
+@mock.patch('sky.backends.backend_utils.check_cluster_available')
+@mock.patch('sky.backends.cloud_vm_ray_backend.CloudVmRayBackend.cancel_jobs')
+def test_cancel_jobs_for_current_user(mock_cancel_jobs,
+                                      mock_check_cluster_available) -> None:
+    mock_handle = mock.create_autospec(CloudVmRayResourceHandle, instance=True)
+    mock_check_cluster_available.return_value = mock_handle
+    cancel('test-cluster', all=True)
+    mock_cancel_jobs.assert_called_once_with(
+        mock_handle,
+        None,
+        cancel_all=True,
+        user_hash=common_utils.get_current_user().id,
+    )
+
+
+@mock.patch('sky.backends.backend_utils.check_cluster_available')
+@mock.patch('sky.backends.cloud_vm_ray_backend.CloudVmRayBackend.cancel_jobs')
+def test_cancel_jobs_for_all_users(mock_cancel_jobs,
+                                   mock_check_cluster_available) -> None:
+    mock_handle = mock.create_autospec(CloudVmRayResourceHandle, instance=True)
+    mock_check_cluster_available.return_value = mock_handle
+    cancel('test-cluster', all_users=True)
+    mock_cancel_jobs.assert_called_once_with(
+        mock_handle,
+        None,
+        cancel_all=True,
+        user_hash=None,
+    )
diff --git a/tests/unit_tests/test_sky/users/test_permission.py b/tests/unit_tests/test_sky/users/test_permission.py
index 953abaad4..4ef0273b2 100644
--- a/tests/unit_tests/test_sky/users/test_permission.py
+++ b/tests/unit_tests/test_sky/users/test_permission.py
@@ -65,6 +65,7 @@ class TestPermissionService:
         with mock.patch.object(permission.PermissionService,
                                '_maybe_initialize_policies'):
             service = permission.PermissionService()
+            service._lazy_initialize()
 
             # Verify SQLAlchemy adapter was created with the correct engine
             mock_adapter_class.assert_called_once_with(mock_engine)
@@ -122,30 +123,26 @@ class TestPermissionService:
         # Mock users
         mock_get_users.return_value = mock_users
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service._maybe_initialize_policies()
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service._maybe_initialize_policies()
 
-            # Verify policies were added for user role blocklist
-            mock_enforcer.add_policy.assert_any_call('user',
-                                                     '/workspaces/config',
-                                                     'POST')
+        # Verify policies were added for user role blocklist
+        mock_enforcer.add_policy.assert_any_call('user', '/workspaces/config',
+                                                 'POST')
 
-            # Verify workspace policies were added
-            mock_enforcer.add_policy.assert_any_call('*', 'default', '*')
-            mock_enforcer.add_policy.assert_any_call('user1', 'private-ws', '*')
-            mock_enforcer.add_policy.assert_any_call('user2', 'private-ws', '*')
+        # Verify workspace policies were added
+        mock_enforcer.add_policy.assert_any_call('*', 'default', '*')
+        mock_enforcer.add_policy.assert_any_call('user1', 'private-ws', '*')
+        mock_enforcer.add_policy.assert_any_call('user2', 'private-ws', '*')
 
-            # Verify users were assigned default roles
-            for user in mock_users:
-                mock_enforcer.add_grouping_policy.assert_any_call(
-                    user.id, rbac.get_default_role())
+        # Verify users were assigned default roles
+        for user in mock_users:
+            mock_enforcer.add_grouping_policy.assert_any_call(
+                user.id, rbac.get_default_role())
 
-            # Verify policy was saved
-            mock_enforcer.save_policy.assert_called()
+        # Verify policy was saved
+        mock_enforcer.save_policy.assert_called()
 
     @mock.patch('sky.users.permission._policy_lock')
     @mock.patch('sky.users.rbac.get_workspace_policy_permissions')
@@ -198,17 +195,14 @@ class TestPermissionService:
         # Mock users
         mock_get_users.return_value = mock_users
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service._maybe_initialize_policies()
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service._maybe_initialize_policies()
 
-            # Verify no new policies were added (since they already exist)
-            mock_enforcer.remove_filtered_policy.assert_not_called()
-            # save_policy should not be called if no updates were made
-            # (users already have roles, policies already exist)
+        # Verify no new policies were added (since they already exist)
+        mock_enforcer.remove_filtered_policy.assert_not_called()
+        # save_policy should not be called if no updates were made
+        # (users already have roles, policies already exist)
 
     def test_add_user_if_not_exists_new_user(self):
         """Test adding a new user that doesn't exist."""
@@ -216,17 +210,14 @@ class TestPermissionService:
         mock_enforcer.get_roles_for_user.return_value = []  # No existing roles
         mock_enforcer.add_grouping_policy.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            result = service._add_user_if_not_exists_no_lock('new_user')
+        result = service._add_user_if_not_exists_no_lock('new_user')
 
-            assert result is True
-            mock_enforcer.add_grouping_policy.assert_called_once_with(
-                'new_user', rbac.get_default_role())
+        assert result is True
+        mock_enforcer.add_grouping_policy.assert_called_once_with(
+            'new_user', rbac.get_default_role())
 
     def test_add_user_if_not_exists_existing_user(self):
         """Test adding a user that already exists."""
@@ -235,16 +226,13 @@ class TestPermissionService:
             'user'
         ]  # User already has roles
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            result = service._add_user_if_not_exists_no_lock('existing_user')
+        result = service._add_user_if_not_exists_no_lock('existing_user')
 
-            assert result is False
-            mock_enforcer.add_grouping_policy.assert_not_called()
+        assert result is False
+        mock_enforcer.add_grouping_policy.assert_not_called()
 
     @mock.patch('sky.users.permission._policy_lock')
     def test_update_role_new_role(self, mock_policy_lock):
@@ -257,21 +245,18 @@ class TestPermissionService:
         mock_enforcer.remove_grouping_policy.return_value = True
         mock_enforcer.add_grouping_policy.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service._load_policy_no_lock = mock.Mock()
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service._load_policy_no_lock = mock.Mock()
 
-            service.update_role('user1', 'admin')
+        service.update_role('user1', 'admin')
 
-            # Verify old role was removed and new role was added
-            mock_enforcer.remove_grouping_policy.assert_called_once_with(
-                'user1', 'user')
-            mock_enforcer.add_grouping_policy.assert_called_once_with(
-                'user1', 'admin')
-            mock_enforcer.save_policy.assert_called_once()
+        # Verify old role was removed and new role was added
+        mock_enforcer.remove_grouping_policy.assert_called_once_with(
+            'user1', 'user')
+        mock_enforcer.add_grouping_policy.assert_called_once_with(
+            'user1', 'admin')
+        mock_enforcer.save_policy.assert_called_once()
 
     @mock.patch('sky.users.permission._policy_lock')
     def test_update_role_same_role(self, mock_policy_lock):
@@ -283,54 +268,44 @@ class TestPermissionService:
         mock_enforcer.get_roles_for_user.return_value = ['admin'
                                                         ]  # Current role
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service._load_policy_no_lock = mock.Mock()
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service._load_policy_no_lock = mock.Mock()
 
-            service.update_role('user1', 'admin')
+        service.update_role('user1', 'admin')
 
-            # Verify no changes were made
-            mock_enforcer.remove_grouping_policy.assert_not_called()
-            mock_enforcer.add_grouping_policy.assert_not_called()
-            mock_enforcer.save_policy.assert_not_called()
+        # Verify no changes were made
+        mock_enforcer.remove_grouping_policy.assert_not_called()
+        mock_enforcer.add_grouping_policy.assert_not_called()
+        mock_enforcer.save_policy.assert_not_called()
 
     def test_get_user_roles(self):
         """Test getting user roles."""
         mock_enforcer = mock.Mock()
         mock_enforcer.get_roles_for_user.return_value = ['admin', 'user']
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service._load_policy_no_lock = mock.Mock()
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service._load_policy_no_lock = mock.Mock()
 
-            roles = service.get_user_roles('user1')
+        roles = service.get_user_roles('user1')
 
-            assert roles == ['admin', 'user']
-            mock_enforcer.get_roles_for_user.assert_called_once_with('user1')
+        assert roles == ['admin', 'user']
+        mock_enforcer.get_roles_for_user.assert_called_once_with('user1')
 
     def test_check_endpoint_permission(self):
         """Test checking endpoint permissions."""
         mock_enforcer = mock.Mock()
         mock_enforcer.enforce.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            result = service.check_endpoint_permission('user1', '/api/test',
-                                                       'GET')
+        result = service.check_endpoint_permission('user1', '/api/test', 'GET')
 
-            assert result is True
-            mock_enforcer.enforce.assert_called_once_with(
-                'user1', '/api/test', 'GET')
+        assert result is True
+        mock_enforcer.enforce.assert_called_once_with('user1', '/api/test',
+                                                      'GET')
 
     def test_check_workspace_permission_non_server(self):
         """Test workspace permission check when not on API server."""
@@ -338,16 +313,13 @@ class TestPermissionService:
         if constants.ENV_VAR_IS_SKYPILOT_SERVER in os.environ:
             del os.environ[constants.ENV_VAR_IS_SKYPILOT_SERVER]
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
+        service = permission.PermissionService()
+        service._lazy_initialize()
 
-            result = service.check_workspace_permission('user1',
-                                                        'test-workspace')
+        result = service.check_workspace_permission('user1', 'test-workspace')
 
-            # Should always return True when not on API server
-            assert result is True
+        # Should always return True when not on API server
+        assert result is True
 
     def test_check_workspace_permission_admin_user(self):
         """Test workspace permission check for admin user."""
@@ -355,18 +327,15 @@ class TestPermissionService:
 
         mock_enforcer = mock.Mock()
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service.get_user_roles = mock.Mock(return_value=['admin'])
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service.get_user_roles = mock.Mock(return_value=['admin'])
 
-            result = service.check_workspace_permission('admin_user',
-                                                        'test-workspace')
+        result = service.check_workspace_permission('admin_user',
+                                                    'test-workspace')
 
-            # Admin should always have access
-            assert result is True
+        # Admin should always have access
+        assert result is True
 
     def test_check_workspace_permission_regular_user(self):
         """Test workspace permission check for regular user."""
@@ -375,19 +344,15 @@ class TestPermissionService:
         mock_enforcer = mock.Mock()
         mock_enforcer.enforce.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
-            service.get_user_roles = mock.Mock(return_value=['user'])
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
+        service.get_user_roles = mock.Mock(return_value=['user'])
 
-            result = service.check_workspace_permission('user1',
-                                                        'test-workspace')
+        result = service.check_workspace_permission('user1', 'test-workspace')
 
-            assert result is True
-            mock_enforcer.enforce.assert_called_once_with(
-                'user1', 'test-workspace', '*')
+        assert result is True
+        mock_enforcer.enforce.assert_called_once_with('user1', 'test-workspace',
+                                                      '*')
 
     @mock.patch('sky.users.permission._policy_lock')
     def test_add_workspace_policy(self, mock_policy_lock):
@@ -398,20 +363,15 @@ class TestPermissionService:
         mock_enforcer = mock.Mock()
         mock_enforcer.add_policy.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            service.add_workspace_policy('test-workspace', ['user1', 'user2'])
+        service.add_workspace_policy('test-workspace', ['user1', 'user2'])
 
-            # Verify policies were added for each user
-            mock_enforcer.add_policy.assert_any_call('user1', 'test-workspace',
-                                                     '*')
-            mock_enforcer.add_policy.assert_any_call('user2', 'test-workspace',
-                                                     '*')
-            mock_enforcer.save_policy.assert_called_once()
+        # Verify policies were added for each user
+        mock_enforcer.add_policy.assert_any_call('user1', 'test-workspace', '*')
+        mock_enforcer.add_policy.assert_any_call('user2', 'test-workspace', '*')
+        mock_enforcer.save_policy.assert_called_once()
 
     @mock.patch('sky.users.permission.filelock.FileLock')
     def test_policy_lock_context_manager(self, mock_filelock):
@@ -510,9 +470,11 @@ class TestPermissionServiceMultiProcess:
                                '_maybe_initialize_policies'):
             # Create first instance
             service1 = permission.PermissionService()
+            service1._lazy_initialize()
 
             # Create second instance
             service2 = permission.PermissionService()
+            service2._lazy_initialize()
 
             # Both should share the same enforcer
             assert service1.enforcer is service2.enforcer
@@ -579,13 +541,10 @@ class TestPermissionServiceMultiProcess:
 
         def create_service():
             try:
-                with mock.patch.object(permission.PermissionService,
-                                       '__init__',
-                                       return_value=None):
-                    service = permission.PermissionService()
-                    service.enforcer = mock_enforcer
-                    service._maybe_initialize_policies()
-                    services.append(service)
+                service = permission.PermissionService()
+                service.enforcer = mock_enforcer
+                service._maybe_initialize_policies()
+                services.append(service)
             except Exception as e:
                 errors.append(e)
 
@@ -674,26 +633,23 @@ class TestPermissionServiceMultiProcess:
         mock_get_workspace_perms.return_value = {}
         mock_get_users.return_value = mock_users
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            # Initialize policies multiple times
-            service._maybe_initialize_policies()
-            service._maybe_initialize_policies()
-            service._maybe_initialize_policies()
+        # Initialize policies multiple times
+        service._maybe_initialize_policies()
+        service._maybe_initialize_policies()
+        service._maybe_initialize_policies()
 
-            # Each user should only be added once (3 users total)
-            assert len(grouping_policy_calls) == len(mock_users)
+        # Each user should only be added once (3 users total)
+        assert len(grouping_policy_calls) == len(mock_users)
 
-            # Verify each user was added exactly once
-            expected_calls = {
-                (user.id, rbac.get_default_role()) for user in mock_users
-            }
-            actual_calls = set(grouping_policy_calls)
-            assert actual_calls == expected_calls
+        # Verify each user was added exactly once
+        expected_calls = {
+            (user.id, rbac.get_default_role()) for user in mock_users
+        }
+        actual_calls = set(grouping_policy_calls)
+        assert actual_calls == expected_calls
 
     @mock.patch('sky.users.permission._policy_lock')
     def test_concurrent_policy_updates_use_lock(self, mock_policy_lock):
@@ -718,20 +674,17 @@ class TestPermissionServiceMultiProcess:
         mock_enforcer.add_policy.return_value = True
         mock_enforcer.save_policy.return_value = True
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            # Test a single workspace policy update to verify lock is used
-            service.add_workspace_policy('test-workspace', ['user1'])
+        # Test a single workspace policy update to verify lock is used
+        service.add_workspace_policy('test-workspace', ['user1'])
 
-            # Verify that lock was acquired and released
-            assert 'enter' in lock_calls
-            assert 'exit' in lock_calls
-            assert len([call for call in lock_calls if call == 'enter']) == 1
-            assert len([call for call in lock_calls if call == 'exit']) == 1
+        # Verify that lock was acquired and released
+        assert 'enter' in lock_calls
+        assert 'exit' in lock_calls
+        assert len([call for call in lock_calls if call == 'enter']) == 1
+        assert len([call for call in lock_calls if call == 'exit']) == 1
 
     @mock.patch('sky.users.permission._policy_lock')
     @mock.patch('sky.users.rbac.get_workspace_policy_permissions')
@@ -775,18 +728,15 @@ class TestPermissionServiceMultiProcess:
 
         mock_get_users.return_value = mock_users
 
-        with mock.patch.object(permission.PermissionService,
-                               '__init__',
-                               return_value=None):
-            service = permission.PermissionService()
-            service.enforcer = mock_enforcer
+        service = permission.PermissionService()
+        service.enforcer = mock_enforcer
 
-            # Initialize policies when they already exist
-            service._maybe_initialize_policies()
+        # Initialize policies when they already exist
+        service._maybe_initialize_policies()
 
-            # Since policies already exist and users already have roles,
-            # no new policies or grouping policies should be added
-            mock_enforcer.remove_filtered_policy.assert_not_called()
+        # Since policies already exist and users already have roles,
+        # no new policies or grouping policies should be added
+        mock_enforcer.remove_filtered_policy.assert_not_called()
 
-            # The save_policy might not be called if no changes were made
-            # We mainly want to ensure no duplicate additions occurred
+        # The save_policy might not be called if no changes were made
+        # We mainly want to ensure no duplicate additions occurred