Repository: mosaicml/composer (branch: error_logging_callback)
Commit date: 2023-11-27T15:09:00Z

Workflow file: .github/workflows/smoketest.yaml (Smoketest)
name: Smoketest
on:
  push:
    branches:
      - dev
      - main
      - release/**
  pull_request:
  workflow_call:
  workflow_dispatch:
# Cancel old runs when a new commit is pushed to the same branch if not on main or dev
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
defaults:
  run:
    working-directory: .
jobs:
  smoketest:
    runs-on: ubuntu-20.04
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    timeout-minutes: 10
    strategy:
      matrix:
        python_version:
          - "3.8"
          - "3.9"
          - "3.10"
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}
      - name: Setup
        run: |
          set -ex
          python -m pip install --upgrade 'pip<23' wheel
          python -m pip install --upgrade .
          python -m pip install pytest==7.2.1 pytest_codeblocks==0.16.1
      - name: Run checks
        run: |
          pytest tests/test_smoketest.py

env:
  PIP_INDEX_URL: http://localhost:8629/2023-11-27
  UV_INDEX_URL: http://localhost:8629/2023-11-27


Failed build logs:
step_name: smoketest (3.8)/5_Run checks.txt
log: ##[group]Run pytest tests/test_smoketest.py
[36;1mpytest tests/test_smoketest.py[0m
shell: /usr/bin/bash -e {0}
env:
  pythonLocation: /opt/hostedtoolcache/Python/3.8.18/x64
  PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.8.18/x64/lib/pkgconfig
  Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.8.18/x64
  LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.18/x64/lib
##[endgroup]
ImportError while loading conftest '/home/runner/work/composer/composer/tests/conftest.py'.
tests/conftest.py:9: in <module>
    from composer.utils import reproducibility
composer/__init__.py:10: in <module>
    from composer.trainer import Trainer
composer/trainer/__init__.py:6: in <module>
    from composer.trainer.trainer import Trainer
composer/trainer/trainer.py:37: in <module>
    from composer.callbacks import CheckpointSaver, OptimizerMonitor
composer/callbacks/__init__.py:12: in <module>
    from composer.callbacks.eval_output_logging_callback import EvalOutputLogging
composer/callbacks/eval_output_logging_callback.py:16: in <module>
    from composer.datasets.in_context_learning_evaluation import (InContextLearningCodeEvalDataset,
composer/datasets/in_context_learning_evaluation.py:13: in <module>
    import transformers
E   ModuleNotFoundError: No module named 'transformers'
##[error]Process completed with exit code 4.

step_name: smoketest (3.9)/5_Run checks.txt
log: ##[group]Run pytest tests/test_smoketest.py
[36;1mpytest tests/test_smoketest.py[0m
shell: /usr/bin/bash -e {0}
env:
  pythonLocation: /opt/hostedtoolcache/Python/3.9.18/x64
  PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.9.18/x64/lib/pkgconfig
  Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.9.18/x64
  Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.9.18/x64
  Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.9.18/x64
  LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.9.18/x64/lib
##[endgroup]
ImportError while loading conftest '/home/runner/work/composer/composer/tests/conftest.py'.
tests/conftest.py:9: in <module>
    from composer.utils import reproducibility
composer/__init__.py:10: in <module>
    from composer.trainer import Trainer
composer/trainer/__init__.py:6: in <module>
    from composer.trainer.trainer import Trainer
composer/trainer/trainer.py:37: in <module>
    from composer.callbacks import CheckpointSaver, OptimizerMonitor
composer/callbacks/__init__.py:12: in <module>
    from composer.callbacks.eval_output_logging_callback import EvalOutputLogging
composer/callbacks/eval_output_logging_callback.py:16: in <module>
    from composer.datasets.in_context_learning_evaluation import (InContextLearningCodeEvalDataset,
composer/datasets/in_context_learning_evaluation.py:13: in <module>
    import transformers
E   ModuleNotFoundError: No module named 'transformers'
##[error]Process completed with exit code 4.

step_name: smoketest (3.10)/5_Run checks.txt
log: ##[group]Run pytest tests/test_smoketest.py
[36;1mpytest tests/test_smoketest.py[0m
shell: /usr/bin/bash -e {0}
env:
  pythonLocation: /opt/hostedtoolcache/Python/3.10.13/x64
  PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.10.13/x64/lib/pkgconfig
  Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.13/x64
  Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.13/x64
  Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.13/x64
  LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.10.13/x64/lib
##[endgroup]
ImportError while loading conftest '/home/runner/work/composer/composer/tests/conftest.py'.
tests/conftest.py:9: in <module>
    from composer.utils import reproducibility
composer/__init__.py:10: in <module>
    from composer.trainer import Trainer
composer/trainer/__init__.py:6: in <module>
    from composer.trainer.trainer import Trainer
composer/trainer/trainer.py:37: in <module>
    from composer.callbacks import CheckpointSaver, OptimizerMonitor
composer/callbacks/__init__.py:12: in <module>
    from composer.callbacks.eval_output_logging_callback import EvalOutputLogging
composer/callbacks/eval_output_logging_callback.py:16: in <module>
    from composer.datasets.in_context_learning_evaluation import (InContextLearningCodeEvalDataset,
composer/datasets/in_context_learning_evaluation.py:13: in <module>
    import transformers
E   ModuleNotFoundError: No module named 'transformers'
##[error]Process completed with exit code 4.


Changed files:
composer/callbacks/eval_output_logging_callback.py
composer/datasets/in_context_learning_evaluation.py

Fix diff:
diff --git a/composer/callbacks/eval_output_logging_callback.py b/composer/callbacks/eval_output_logging_callback.py
index 48e4f76d..65cbdebb 100644
--- a/composer/callbacks/eval_output_logging_callback.py
+++ b/composer/callbacks/eval_output_logging_callback.py
@@ -7,7 +7,7 @@ import hashlib
 import os
 import random
 import shutil
-import time
+import tempfile
 from typing import Callable, Optional
 
 from torch.utils.data import DataLoader
@@ -58,6 +58,8 @@ class EvalOutputLogging(Callback):
         self.hash = hashlib.sha256()
         self.destination_file = None
 
+    # with tempfile.NamedTemporaryFile
+    #  tmp_dir =
     def _write_tables_to_output_dir(self, state: State):
         try:
             import pandas as pd
@@ -66,16 +68,9 @@ class EvalOutputLogging(Callback):
                                                 conda_package='pandas',
                                                 conda_channel='conda-forge') from e
         # write tmp files
-        self.hash.update((str(time.time()) + str(random.randint(0, 1_000_000))).encode('utf-8'))
-        tmp_dir = os.getcwd() + '/' + self.hash.hexdigest()
-
-        if not os.path.exists(tmp_dir):
-            with dist.local_rank_zero_download_and_wait(tmp_dir):
-                if dist.get_local_rank() == 0:
-                    os.mkdir(tmp_dir)
 
         full_df = pd.DataFrame()
-        file_name = f'eval-outputs-ba{state.timestamp.batch.value}.tsv'
+        upload_file_name = f'eval-outputs-ba{state.timestamp.batch.value}.tsv'
 
         for benchmark in self.table:
             cols, rows = self.table[benchmark]
@@ -84,18 +79,14 @@ class EvalOutputLogging(Callback):
             df['benchmark'] = benchmark
             full_df = pd.concat([full_df, df], ignore_index=True)
 
-        with dist.local_rank_zero_download_and_wait(f'{tmp_dir}/{file_name}'):
-            if dist.get_local_rank() == 0:
-                with open(f'{tmp_dir}/{file_name}', 'wb') as f:
-                    full_df.to_csv(f, sep='\t', index=False)
+        tmp_file = ''
+        with tempfile.NamedTemporaryFile('wb') as f:
+            full_df.to_csv(f, sep='\t', index=False)
+            tmp_file = f.name
 
         # copy/upload tmp files
-        _write(destination_path=f'{self.output_directory}/{file_name}', src_file=f'{tmp_dir}/{file_name}')
-        os.remove(f'{tmp_dir}/{file_name}')
-        self.destination_file = f'{self.output_directory}/{file_name}'
-
-        # delete tmp files
-        os.rmdir(tmp_dir)
+        _write(destination_path=f'{self.output_directory}/{upload_file_name}', src_file=tmp_file)
+        self.destination_file = f'{self.output_directory}/{upload_file_name}'
 
     def _prep_response_cache(self, state, cache):
         benchmark = state.dataloader_label
diff --git a/composer/datasets/in_context_learning_evaluation.py b/composer/datasets/in_context_learning_evaluation.py
index e1566509..1a53f75c 100644
--- a/composer/datasets/in_context_learning_evaluation.py
+++ b/composer/datasets/in_context_learning_evaluation.py
@@ -10,7 +10,6 @@ import random
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import torch
-import transformers
 from torch.utils.data import DataLoader, Dataset
 from tqdm import tqdm