| { | |
| "schemaVersion": 1, | |
| "deviceProperties": [ | |
| { | |
| "id": 0, "name": "NVIDIA A100-PCIE-40GB", "totalGlobalMem": 42411098112, | |
| "computeMajor": 8, "computeMinor": 0, | |
| "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048, | |
| "regsPerBlock": 65536, "regsPerMultiprocessor": 65536, "warpSize": 32, | |
| "sharedMemPerBlock": 49152, "sharedMemPerMultiprocessor": 167936, | |
| "numSms": 108, "sharedMemPerBlockOptin": 166912 | |
| } | |
| ], | |
| "with_modules": 1, | |
| "record_shapes": 1, | |
| "with_stack": 1, | |
| "profile_memory": 1, | |
| "traceEvents": [ | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814408, "dur": 3779, | |
| "args": { | |
| "Python parent id": null, "Python id": 1, "Ev Idx": 0 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814409, "dur": 3778, | |
| "args": { | |
| "Python parent id": 1, "Python id": 2, "Ev Idx": 1 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814411, "dur": 3776, | |
| "args": { | |
| "Python parent id": 2, "Python id": 3, "Ev Idx": 2 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(323): check_internal_messages", "pid": 0, "tid": 0, | |
| "ts": 1733491535814412, "dur": 3775, | |
| "args": { | |
| "Python parent id": 3, "Python id": 4, "Ev Idx": 3 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733491535814415, "dur": 3772, | |
| "args": { | |
| "Python parent id": 4, "Python id": 5, "Ev Idx": 4 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814416, "dur": 3771, | |
| "args": { | |
| "Python parent id": 5, "Python id": 6, "Ev Idx": 5 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814417, "dur": 3770, | |
| "args": { | |
| "Python parent id": 6, "Python id": 7, "Ev Idx": 6 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814421, "dur": 3766, | |
| "args": { | |
| "Python parent id": 7, "Python id": 8, "Ev Idx": 7 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814421, "dur": 3766, | |
| "args": { | |
| "Python parent id": 8, "Python id": 9, "Ev Idx": 8 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814421, "dur": 3766, | |
| "args": { | |
| "Python parent id": 9, "Python id": 10, "Ev Idx": 9 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(279): check_network_status", "pid": 0, "tid": 0, | |
| "ts": 1733491535814422, "dur": 3765, | |
| "args": { | |
| "Python parent id": 10, "Python id": 11, "Ev Idx": 10 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733491535814423, "dur": 3764, | |
| "args": { | |
| "Python parent id": 11, "Python id": 12, "Ev Idx": 11 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814423, "dur": 3764, | |
| "args": { | |
| "Python parent id": 12, "Python id": 13, "Ev Idx": 12 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814424, "dur": 3763, | |
| "args": { | |
| "Python parent id": 13, "Python id": 14, "Ev Idx": 13 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814429, "dur": 3758, | |
| "args": { | |
| "Python parent id": 14, "Python id": 15, "Ev Idx": 14 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814430, "dur": 3757, | |
| "args": { | |
| "Python parent id": 15, "Python id": 16, "Ev Idx": 15 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814430, "dur": 3757, | |
| "args": { | |
| "Python parent id": 16, "Python id": 17, "Ev Idx": 16 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(303): check_stop_status", "pid": 0, "tid": 0, | |
| "ts": 1733491535814431, "dur": 3756, | |
| "args": { | |
| "Python parent id": 17, "Python id": 18, "Ev Idx": 17 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733491535814432, "dur": 3755, | |
| "args": { | |
| "Python parent id": 18, "Python id": 19, "Ev Idx": 18 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814433, "dur": 3754, | |
| "args": { | |
| "Python parent id": 19, "Python id": 20, "Ev Idx": 19 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814433, "dur": 3754, | |
| "args": { | |
| "Python parent id": 20, "Python id": 21, "Ev Idx": 20 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814439, "dur": 3748, | |
| "args": { | |
| "Python parent id": 21, "Python id": 22, "Ev Idx": 21 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814439, "dur": 3748, | |
| "args": { | |
| "Python parent id": 22, "Python id": 23, "Ev Idx": 22 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814439, "dur": 3748, | |
| "args": { | |
| "Python parent id": 23, "Python id": 24, "Ev Idx": 23 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router.py(70): message_loop", "pid": 0, "tid": 0, | |
| "ts": 1733491535814440, "dur": 3747, | |
| "args": { | |
| "Python parent id": 24, "Python id": 25, "Ev Idx": 24 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router_sock.py(27): _read_message", "pid": 0, "tid": 0, | |
| "ts": 1733491535814441, "dur": 3746, | |
| "args": { | |
| "Python parent id": 25, "Python id": 26, "Ev Idx": 25 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(285): read_server_response", "pid": 0, "tid": 0, | |
| "ts": 1733491535814442, "dur": 3745, | |
| "args": { | |
| "Python parent id": 26, "Python id": 27, "Ev Idx": 26 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(255): _read_packet_bytes", "pid": 0, "tid": 0, | |
| "ts": 1733491535814442, "dur": 3745, | |
| "args": { | |
| "Python parent id": 27, "Python id": 28, "Ev Idx": 27 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814444, "dur": 3743, | |
| "args": { | |
| "Python parent id": 28, "Python id": 29, "Ev Idx": 28 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814444, "dur": 3743, | |
| "args": { | |
| "Python parent id": 29, "Python id": 30, "Ev Idx": 29 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814445, "dur": 3742, | |
| "args": { | |
| "Python parent id": 30, "Python id": 31, "Ev Idx": 30 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814445, "dur": 3742, | |
| "args": { | |
| "Python parent id": 31, "Python id": 32, "Ev Idx": 31 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814446, "dur": 3741, | |
| "args": { | |
| "Python parent id": 32, "Python id": 33, "Ev Idx": 32 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733491535814448, "dur": 3739, | |
| "args": { | |
| "Python parent id": 33, "Python id": 34, "Ev Idx": 33 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733491535814448, "dur": 3739, | |
| "args": { | |
| "Python parent id": 34, "Python id": 35, "Ev Idx": 34 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0, | |
| "ts": 1733491535814449, "dur": 3738, | |
| "args": { | |
| "Python parent id": 35, "Python id": 36, "Ev Idx": 35 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814449, "dur": 3738, | |
| "args": { | |
| "Python parent id": 36, "Python id": 37, "Ev Idx": 36 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733491535814450, "dur": 3737, | |
| "args": { | |
| "Python parent id": 37, "Python id": 38, "Ev Idx": 37 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "ft_llm.py(299): <module>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814454, "dur": 3733, | |
| "args": { | |
| "Python parent id": null, "Python id": 39, "Ev Idx": 38 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "ft_llm.py(261): train", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814455, "dur": 3732, | |
| "args": { | |
| "Python parent id": 39, "Python id": 40, "Ev Idx": 39 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(699): __exit__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814457, "dur": 3730, | |
| "args": { | |
| "Python parent id": 40, "Python id": 41, "Ev Idx": 40 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(715): stop", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814457, "dur": 3730, | |
| "args": { | |
| "Python parent id": 41, "Python id": 42, "Ev Idx": 41 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(744): _transit_action", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814459, "dur": 3728, | |
| "args": { | |
| "Python parent id": 42, "Python id": 43, "Ev Idx": 42 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(161): start_trace", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814459, "dur": 3577, | |
| "args": { | |
| "Python parent id": 43, "Python id": 44, "Ev Idx": 43 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(289): _start_trace", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535814460, "dur": 2084, | |
| "args": { | |
| "Python parent id": 44, "Python id": 45, "Ev Idx": 44 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535816556, "dur": 1400, | |
| "args": { | |
| "Python parent id": 44, "Python id": 46, "Ev Idx": 45 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817946, "dur": 7, | |
| "args": { | |
| "Python parent id": 46, "Python id": 47, "Ev Idx": 46 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817958, "dur": 3, | |
| "args": { | |
| "Python parent id": 44, "Python id": 48, "Ev Idx": 47 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817959, "dur": 2, | |
| "args": { | |
| "Python parent id": 48, "Python id": 49, "Ev Idx": 48 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817961, "dur": 1, | |
| "args": { | |
| "Python parent id": 44, "Python id": 50, "Ev Idx": 49 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817961, "dur": 1, | |
| "args": { | |
| "Python parent id": 50, "Python id": 51, "Ev Idx": 50 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817963, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 52, "Ev Idx": 51 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f5530dd2450>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817963, "dur": 0, | |
| "args": { | |
| "Python parent id": 52, "Python id": 53, "Ev Idx": 52 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method kineto_available of PyCapsule object at 0x7f5530dd24b0>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817966, "dur": 1, | |
| "args": { | |
| "Python parent id": 44, "Python id": 54, "Ev Idx": 53 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(279): _get_distributed_info", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817969, "dur": 31, | |
| "args": { | |
| "Python parent id": 44, "Python id": 55, "Ev Idx": 54 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/__init__.py(9): is_available", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817975, "dur": 13, | |
| "args": { | |
| "Python parent id": 55, "Python id": 56, "Ev Idx": 55 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817985, "dur": 3, | |
| "args": { | |
| "Python parent id": 56, "Python id": 57, "Ev Idx": 56 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(973): is_initialized", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817990, "dur": 10, | |
| "args": { | |
| "Python parent id": 55, "Python id": 58, "Ev Idx": 57 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(588): WORLD", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817997, "dur": 3, | |
| "args": { | |
| "Python parent id": 58, "Python id": 59, "Ev Idx": 58 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(460): default_pg", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535817999, "dur": 1, | |
| "args": { | |
| "Python parent id": 59, "Python id": 60, "Ev Idx": 59 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818001, "dur": 5, | |
| "args": { | |
| "Python parent id": 44, "Python id": 61, "Ev Idx": 60 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(356): __getattr__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818022, "dur": 9, | |
| "args": { | |
| "Python parent id": 44, "Python id": 62, "Ev Idx": 61 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(140): __getattr__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818028, "dur": 3, | |
| "args": { | |
| "Python parent id": 62, "Python id": 63, "Ev Idx": 62 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method items of dict object at 0x7f5525fe95c0>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818034, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 64, "Ev Idx": 63 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(195): stop_trace", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818038, "dur": 149, | |
| "args": { | |
| "Python parent id": 43, "Python id": 65, "Ev Idx": 64 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(291): __exit__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818041, "dur": 146, | |
| "args": { | |
| "Python parent id": 65, "Python id": 66, "Ev Idx": 65 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(782): synchronize", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818046, "dur": 141, | |
| "args": { | |
| "Python parent id": 66, "Python id": 67, "Ev Idx": 66 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818047, "dur": 12, | |
| "args": { | |
| "Python parent id": 67, "Python id": 68, "Ev Idx": 67 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818049, "dur": 9, | |
| "args": { | |
| "Python parent id": 68, "Python id": 69, "Ev Idx": 68 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818056, "dur": 2, | |
| "args": { | |
| "Python parent id": 69, "Python id": 70, "Ev Idx": 69 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(360): __init__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818062, "dur": 77, | |
| "args": { | |
| "Python parent id": 67, "Python id": 71, "Ev Idx": 70 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/_utils.py(9): _get_device_index", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818064, "dur": 73, | |
| "args": { | |
| "Python parent id": 71, "Python id": 72, "Ev Idx": 71 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818066, "dur": 1, | |
| "args": { | |
| "Python parent id": 72, "Python id": 73, "Ev Idx": 72 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818067, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 74, "Ev Idx": 73 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818069, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 75, "Ev Idx": 74 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818072, "dur": 1, | |
| "args": { | |
| "Python parent id": 72, "Python id": 76, "Ev Idx": 75 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818073, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 77, "Ev Idx": 76 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(759): _get_device_index", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818080, "dur": 57, | |
| "args": { | |
| "Python parent id": 72, "Python id": 78, "Ev Idx": 77 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818081, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 79, "Ev Idx": 78 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818081, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 80, "Ev Idx": 79 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818081, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 81, "Ev Idx": 80 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818082, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 82, "Ev Idx": 81 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(733): _get_current_device_index", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818083, "dur": 54, | |
| "args": { | |
| "Python parent id": 78, "Python id": 83, "Ev Idx": 82 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(721): _get_device_attr", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818084, "dur": 53, | |
| "args": { | |
| "Python parent id": 83, "Python id": 84, "Ev Idx": 83 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(708): _get_available_device_type", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818086, "dur": 26, | |
| "args": { | |
| "Python parent id": 84, "Python id": 85, "Ev Idx": 84 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(105): is_available", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818089, "dur": 22, | |
| "args": { | |
| "Python parent id": 85, "Python id": 86, "Ev Idx": 85 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(96): _is_compiled", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818091, "dur": 2, | |
| "args": { | |
| "Python parent id": 86, "Python id": 87, "Ev Idx": 86 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818092, "dur": 0, | |
| "args": { | |
| "Python parent id": 87, "Python id": 88, "Ev Idx": 87 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(101): _nvml_based_avail", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818093, "dur": 12, | |
| "args": { | |
| "Python parent id": 86, "Python id": 89, "Ev Idx": 88 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(771): getenv", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818095, "dur": 10, | |
| "args": { | |
| "Python parent id": 89, "Python id": 90, "Ev Idx": 89 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "_collections_abc.py(760): get", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818097, "dur": 8, | |
| "args": { | |
| "Python parent id": 90, "Python id": 91, "Ev Idx": 90 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(674): __getitem__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818098, "dur": 6, | |
| "args": { | |
| "Python parent id": 91, "Python id": 92, "Ev Idx": 91 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(754): encode", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818099, "dur": 3, | |
| "args": { | |
| "Python parent id": 92, "Python id": 93, "Ev Idx": 92 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818099, "dur": 0, | |
| "args": { | |
| "Python parent id": 93, "Python id": 94, "Ev Idx": 93 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method encode of str object at 0x7f558c5afa80>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818101, "dur": 1, | |
| "args": { | |
| "Python parent id": 93, "Python id": 95, "Ev Idx": 94 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDeviceCount>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818110, "dur": 1, | |
| "args": { | |
| "Python parent id": 86, "Python id": 96, "Ev Idx": 95 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method lower of str object at 0x7f56183438b0>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818115, "dur": 0, | |
| "args": { | |
| "Python parent id": 84, "Python id": 97, "Ev Idx": 96 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(735): <lambda>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818116, "dur": 20, | |
| "args": { | |
| "Python parent id": 84, "Python id": 98, "Ev Idx": 97 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(776): current_device", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818118, "dur": 18, | |
| "args": { | |
| "Python parent id": 98, "Python id": 99, "Ev Idx": 98 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818118, "dur": 1, | |
| "args": { | |
| "Python parent id": 99, "Python id": 100, "Ev Idx": 99 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818119, "dur": 0, | |
| "args": { | |
| "Python parent id": 100, "Python id": 101, "Ev Idx": 100 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818119, "dur": 0, | |
| "args": { | |
| "Python parent id": 101, "Python id": 102, "Ev Idx": 101 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDevice>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818120, "dur": 16, | |
| "args": { | |
| "Python parent id": 99, "Python id": 103, "Ev Idx": 102 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(364): __enter__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818141, "dur": 5, | |
| "args": { | |
| "Python parent id": 67, "Python id": 104, "Ev Idx": 103 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_exchangeDevice>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818143, "dur": 2, | |
| "args": { | |
| "Python parent id": 104, "Python id": 105, "Ev Idx": 104 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_synchronize>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818147, "dur": 23, | |
| "args": { | |
| "Python parent id": 67, "Python id": 106, "Ev Idx": 105 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(367): __exit__", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818171, "dur": 16, | |
| "args": { | |
| "Python parent id": 67, "Python id": 107, "Ev Idx": 106 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_maybeExchangeDevice>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818172, "dur": 2, | |
| "args": { | |
| "Python parent id": 107, "Python id": 108, "Ev Idx": 107 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _disable_profiler of PyCapsule object at 0x7f5530dd23f0>", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818177, "dur": 10, | |
| "args": { | |
| "Python parent id": 107, "Python id": 109, "Ev Idx": 108 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "cuda_runtime", "name": "cudaDeviceSynchronize", "pid": 3866212, "tid": 3866212, | |
| "ts": 1733491535818149, "dur": 20, | |
| "args": { | |
| "External id": 155102795, | |
| "cbid": 165, "correlation": 155102795 | |
| } | |
| }, | |
| { | |
| "ph": "s", "id": 155102795, "pid": 3866212, "tid": 3866212, "ts": 1733491535818149, | |
| "cat": "ac2g", "name": "ac2g" | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0, | |
| "args": { | |
| "labels": "CPU" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 0, | |
| "args": { | |
| "sort_index": 3866212 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0, | |
| "args": { | |
| "labels": "GPU 0" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0, | |
| "args": { | |
| "sort_index": 16777216 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0, | |
| "args": { | |
| "labels": "GPU 1" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 1, "tid": 0, | |
| "args": { | |
| "sort_index": 16777217 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0, | |
| "args": { | |
| "labels": "GPU 2" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 2, "tid": 0, | |
| "args": { | |
| "sort_index": 16777218 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0, | |
| "args": { | |
| "labels": "GPU 3" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3, "tid": 0, | |
| "args": { | |
| "sort_index": 16777219 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0, | |
| "args": { | |
| "labels": "GPU 4" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 4, "tid": 0, | |
| "args": { | |
| "sort_index": 16777220 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0, | |
| "args": { | |
| "labels": "GPU 5" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 5, "tid": 0, | |
| "args": { | |
| "sort_index": 16777221 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0, | |
| "args": { | |
| "labels": "GPU 6" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 6, "tid": 0, | |
| "args": { | |
| "sort_index": 16777222 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0, | |
| "args": { | |
| "labels": "GPU 7" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 7, "tid": 0, | |
| "args": { | |
| "sort_index": 16777223 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0, | |
| "args": { | |
| "name": "thread 0 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 0, "tid": 0, | |
| "args": { | |
| "sort_index": 0 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212, | |
| "args": { | |
| "name": "thread 3866212 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212, | |
| "args": { | |
| "sort_index": 3866212 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212, | |
| "args": { | |
| "name": "thread 3866212 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733491535816534, "pid": 3866212, "tid": 3866212, | |
| "args": { | |
| "sort_index": 3866212 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "Trace", "ts": 1733491535812185, "dur": 6002, | |
| "pid": "Spans", "tid": "PyTorch Profiler", | |
| "name": "PyTorch Profiler (0)", | |
| "args": { | |
| "Op count": 0 | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733491535812185, | |
| "pid": "Spans", "tid": 0, | |
| "args": { | |
| "sort_index": 536870912 | |
| } | |
| }, | |
| { | |
| "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g", | |
| "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1733491535812185 | |
| }, | |
| { | |
| "name": "Record Window End", "ph": "i", "s": "g", | |
| "pid": "", "tid": "", "ts": 1733491535821938 | |
| } | |
| ], | |
| "traceName": "/home/datta0/models/lora_final/profile/profile_trace.json" | |
| } |