{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA A100-PCIE-40GB", "totalGlobalMem": 42411098112,
      "computeMajor": 8, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "regsPerMultiprocessor": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "sharedMemPerMultiprocessor": 167936,
      "numSms": 108, "sharedMemPerBlockOptin": 166912
    }
  ],
  "with_modules": 1,
  "record_shapes": 1,
  "with_stack": 1,
  "profile_memory": 1,
  "traceEvents": [
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814408, "dur": 3779,
    "args": {
      "Python parent id": null, "Python id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814409, "dur": 3778,
    "args": {
      "Python parent id": 1, "Python id": 2, "Ev Idx": 1
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733491535814411, "dur": 3776,
    "args": {
      "Python parent id": 2, "Python id": 3, "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(323): check_internal_messages", "pid": 0, "tid": 0,
    "ts": 1733491535814412, "dur": 3775,
    "args": {
      "Python parent id": 3, "Python id": 4, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733491535814415, "dur": 3772,
    "args": {
      "Python parent id": 4, "Python id": 5, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814416, "dur": 3771,
    "args": {
      "Python parent id": 5, "Python id": 6, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814417, "dur": 3770,
    "args": {
      "Python parent id": 6, "Python id": 7, "Ev Idx": 6
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814421, "dur": 3766,
    "args": {
      "Python parent id": 7, "Python id": 8, "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814421, "dur": 3766,
    "args": {
      "Python parent id": 8, "Python id": 9, "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733491535814421, "dur": 3766,
    "args": {
      "Python parent id": 9, "Python id": 10, "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(279): check_network_status", "pid": 0, "tid": 0,
    "ts": 1733491535814422, "dur": 3765,
    "args": {
      "Python parent id": 10, "Python id": 11, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733491535814423, "dur": 3764,
    "args": {
      "Python parent id": 11, "Python id": 12, "Ev Idx": 11
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814423, "dur": 3764,
    "args": {
      "Python parent id": 12, "Python id": 13, "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814424, "dur": 3763,
    "args": {
      "Python parent id": 13, "Python id": 14, "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814429, "dur": 3758,
    "args": {
      "Python parent id": 14, "Python id": 15, "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814430, "dur": 3757,
    "args": {
      "Python parent id": 15, "Python id": 16, "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733491535814430, "dur": 3757,
    "args": {
      "Python parent id": 16, "Python id": 17, "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(303): check_stop_status", "pid": 0, "tid": 0,
    "ts": 1733491535814431, "dur": 3756,
    "args": {
      "Python parent id": 17, "Python id": 18, "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733491535814432, "dur": 3755,
    "args": {
      "Python parent id": 18, "Python id": 19, "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814433, "dur": 3754,
    "args": {
      "Python parent id": 19, "Python id": 20, "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814433, "dur": 3754,
    "args": {
      "Python parent id": 20, "Python id": 21, "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814439, "dur": 3748,
    "args": {
      "Python parent id": 21, "Python id": 22, "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814439, "dur": 3748,
    "args": {
      "Python parent id": 22, "Python id": 23, "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733491535814439, "dur": 3748,
    "args": {
      "Python parent id": 23, "Python id": 24, "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router.py(70): message_loop", "pid": 0, "tid": 0,
    "ts": 1733491535814440, "dur": 3747,
    "args": {
      "Python parent id": 24, "Python id": 25, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router_sock.py(27): _read_message", "pid": 0, "tid": 0,
    "ts": 1733491535814441, "dur": 3746,
    "args": {
      "Python parent id": 25, "Python id": 26, "Ev Idx": 25
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(285): read_server_response", "pid": 0, "tid": 0,
    "ts": 1733491535814442, "dur": 3745,
    "args": {
      "Python parent id": 26, "Python id": 27, "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(255): _read_packet_bytes", "pid": 0, "tid": 0,
    "ts": 1733491535814442, "dur": 3745,
    "args": {
      "Python parent id": 27, "Python id": 28, "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814444, "dur": 3743,
    "args": {
      "Python parent id": 28, "Python id": 29, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814444, "dur": 3743,
    "args": {
      "Python parent id": 29, "Python id": 30, "Ev Idx": 29
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0,
    "ts": 1733491535814445, "dur": 3742,
    "args": {
      "Python parent id": 30, "Python id": 31, "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814445, "dur": 3742,
    "args": {
      "Python parent id": 31, "Python id": 32, "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814446, "dur": 3741,
    "args": {
      "Python parent id": 32, "Python id": 33, "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733491535814448, "dur": 3739,
    "args": {
      "Python parent id": 33, "Python id": 34, "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733491535814448, "dur": 3739,
    "args": {
      "Python parent id": 34, "Python id": 35, "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0,
    "ts": 1733491535814449, "dur": 3738,
    "args": {
      "Python parent id": 35, "Python id": 36, "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814449, "dur": 3738,
    "args": {
      "Python parent id": 36, "Python id": 37, "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733491535814450, "dur": 3737,
    "args": {
      "Python parent id": 37, "Python id": 38, "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "ft_llm.py(299): <module>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814454, "dur": 3733,
    "args": {
      "Python parent id": null, "Python id": 39, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "ft_llm.py(261): train", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814455, "dur": 3732,
    "args": {
      "Python parent id": 39, "Python id": 40, "Ev Idx": 39
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(699): __exit__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814457, "dur": 3730,
    "args": {
      "Python parent id": 40, "Python id": 41, "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(715): stop", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814457, "dur": 3730,
    "args": {
      "Python parent id": 41, "Python id": 42, "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(744): _transit_action", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814459, "dur": 3728,
    "args": {
      "Python parent id": 42, "Python id": 43, "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(161): start_trace", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814459, "dur": 3577,
    "args": {
      "Python parent id": 43, "Python id": 44, "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(289): _start_trace", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535814460, "dur": 2084,
    "args": {
      "Python parent id": 44, "Python id": 45, "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535816556, "dur": 1400,
    "args": {
      "Python parent id": 44, "Python id": 46, "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817946, "dur": 7,
    "args": {
      "Python parent id": 46, "Python id": 47, "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817958, "dur": 3,
    "args": {
      "Python parent id": 44, "Python id": 48, "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817959, "dur": 2,
    "args": {
      "Python parent id": 48, "Python id": 49, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817961, "dur": 1,
    "args": {
      "Python parent id": 44, "Python id": 50, "Ev Idx": 49
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817961, "dur": 1,
    "args": {
      "Python parent id": 50, "Python id": 51, "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817963, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 52, "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817963, "dur": 0,
    "args": {
      "Python parent id": 52, "Python id": 53, "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method kineto_available of PyCapsule object at 0x7f5530dd24b0>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817966, "dur": 1,
    "args": {
      "Python parent id": 44, "Python id": 54, "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(279): _get_distributed_info", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817969, "dur": 31,
    "args": {
      "Python parent id": 44, "Python id": 55, "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/__init__.py(9): is_available", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817975, "dur": 13,
    "args": {
      "Python parent id": 55, "Python id": 56, "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817985, "dur": 3,
    "args": {
      "Python parent id": 56, "Python id": 57, "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(973): is_initialized", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817990, "dur": 10,
    "args": {
      "Python parent id": 55, "Python id": 58, "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(588): WORLD", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817997, "dur": 3,
    "args": {
      "Python parent id": 58, "Python id": 59, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(460): default_pg", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535817999, "dur": 1,
    "args": {
      "Python parent id": 59, "Python id": 60, "Ev Idx": 59
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818001, "dur": 5,
    "args": {
      "Python parent id": 44, "Python id": 61, "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(356): __getattr__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818022, "dur": 9,
    "args": {
      "Python parent id": 44, "Python id": 62, "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(140): __getattr__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818028, "dur": 3,
    "args": {
      "Python parent id": 62, "Python id": 63, "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method items of dict object at 0x7f5525fe95c0>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818034, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 64, "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(195): stop_trace", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818038, "dur": 149,
    "args": {
      "Python parent id": 43, "Python id": 65, "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(291): __exit__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818041, "dur": 146,
    "args": {
      "Python parent id": 65, "Python id": 66, "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(782): synchronize", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818046, "dur": 141,
    "args": {
      "Python parent id": 66, "Python id": 67, "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818047, "dur": 12,
    "args": {
      "Python parent id": 67, "Python id": 68, "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818049, "dur": 9,
    "args": {
      "Python parent id": 68, "Python id": 69, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818056, "dur": 2,
    "args": {
      "Python parent id": 69, "Python id": 70, "Ev Idx": 69
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(360): __init__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818062, "dur": 77,
    "args": {
      "Python parent id": 67, "Python id": 71, "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/_utils.py(9): _get_device_index", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818064, "dur": 73,
    "args": {
      "Python parent id": 71, "Python id": 72, "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818066, "dur": 1,
    "args": {
      "Python parent id": 72, "Python id": 73, "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818067, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 74, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818069, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 75, "Ev Idx": 74
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818072, "dur": 1,
    "args": {
      "Python parent id": 72, "Python id": 76, "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818073, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 77, "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(759): _get_device_index", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818080, "dur": 57,
    "args": {
      "Python parent id": 72, "Python id": 78, "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818081, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 79, "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818081, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 80, "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818081, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 81, "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818082, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 82, "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(733): _get_current_device_index", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818083, "dur": 54,
    "args": {
      "Python parent id": 78, "Python id": 83, "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(721): _get_device_attr", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818084, "dur": 53,
    "args": {
      "Python parent id": 83, "Python id": 84, "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(708): _get_available_device_type", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818086, "dur": 26,
    "args": {
      "Python parent id": 84, "Python id": 85, "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(105): is_available", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818089, "dur": 22,
    "args": {
      "Python parent id": 85, "Python id": 86, "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(96): _is_compiled", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818091, "dur": 2,
    "args": {
      "Python parent id": 86, "Python id": 87, "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818092, "dur": 0,
    "args": {
      "Python parent id": 87, "Python id": 88, "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(101): _nvml_based_avail", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818093, "dur": 12,
    "args": {
      "Python parent id": 86, "Python id": 89, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(771): getenv", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818095, "dur": 10,
    "args": {
      "Python parent id": 89, "Python id": 90, "Ev Idx": 89
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "_collections_abc.py(760): get", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818097, "dur": 8,
    "args": {
      "Python parent id": 90, "Python id": 91, "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(674): __getitem__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818098, "dur": 6,
    "args": {
      "Python parent id": 91, "Python id": 92, "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(754): encode", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818099, "dur": 3,
    "args": {
      "Python parent id": 92, "Python id": 93, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818099, "dur": 0,
    "args": {
      "Python parent id": 93, "Python id": 94, "Ev Idx": 93
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method encode of str object at 0x7f558c5afa80>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818101, "dur": 1,
    "args": {
      "Python parent id": 93, "Python id": 95, "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDeviceCount>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818110, "dur": 1,
    "args": {
      "Python parent id": 86, "Python id": 96, "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method lower of str object at 0x7f56183438b0>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818115, "dur": 0,
    "args": {
      "Python parent id": 84, "Python id": 97, "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(735): <lambda>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818116, "dur": 20,
    "args": {
      "Python parent id": 84, "Python id": 98, "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(776): current_device", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818118, "dur": 18,
    "args": {
      "Python parent id": 98, "Python id": 99, "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818118, "dur": 1,
    "args": {
      "Python parent id": 99, "Python id": 100, "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818119, "dur": 0,
    "args": {
      "Python parent id": 100, "Python id": 101, "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818119, "dur": 0,
    "args": {
      "Python parent id": 101, "Python id": 102, "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDevice>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818120, "dur": 16,
    "args": {
      "Python parent id": 99, "Python id": 103, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(364): __enter__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818141, "dur": 5,
    "args": {
      "Python parent id": 67, "Python id": 104, "Ev Idx": 103
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_exchangeDevice>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818143, "dur": 2,
    "args": {
      "Python parent id": 104, "Python id": 105, "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_synchronize>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818147, "dur": 23,
    "args": {
      "Python parent id": 67, "Python id": 106, "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(367): __exit__", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818171, "dur": 16,
    "args": {
      "Python parent id": 67, "Python id": 107, "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_maybeExchangeDevice>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818172, "dur": 2,
    "args": {
      "Python parent id": 107, "Python id": 108, "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _disable_profiler of PyCapsule object at 0x7f5530dd23f0>", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818177, "dur": 10,
    "args": {
      "Python parent id": 107, "Python id": 109, "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cuda_runtime", "name": "cudaDeviceSynchronize", "pid": 3866212, "tid": 3866212,
    "ts": 1733491535818149, "dur": 20,
    "args": {
      "External id": 155102795,
      "cbid": 165, "correlation": 155102795
    }
  },
  {
    "ph": "s", "id": 155102795, "pid": 3866212, "tid": 3866212, "ts": 1733491535818149,
    "cat": "ac2g", "name": "ac2g"
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0,
    "args": {
      "sort_index": 3866212
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0,
    "args": {
      "labels": "GPU 0"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0,
    "args": {
      "sort_index": 16777216
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0,
    "args": {
      "labels": "GPU 1"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0,
    "args": {
      "sort_index": 16777217
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0,
    "args": {
      "labels": "GPU 2"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0,
    "args": {
      "sort_index": 16777218
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0,
    "args": {
      "labels": "GPU 3"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0,
    "args": {
      "sort_index": 16777219
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0,
    "args": {
      "labels": "GPU 4"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0,
    "args": {
      "sort_index": 16777220
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0,
    "args": {
      "labels": "GPU 5"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0,
    "args": {
      "sort_index": 16777221
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0,
    "args": {
      "labels": "GPU 6"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0,
    "args": {
      "sort_index": 16777222
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0,
    "args": {
      "labels": "GPU 7"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0,
    "args": {
      "sort_index": 16777223
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0,
    "args": {
      "name": "thread 0 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0,
    "args": {
      "sort_index": 0
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212,
    "args": {
      "name": "thread 3866212 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212,
    "args": {
      "sort_index": 3866212
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212,
    "args": {
      "name": "thread 3866212 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212,
    "args": {
      "sort_index": 3866212
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1733491535812185, "dur": 6002,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733491535812185,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1733491535812185
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1733491535821938
  }
  ],
  "traceName": "/home/datta0/models/lora_final/profile/profile_trace.json"
}