internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: pypa/cibuildwheel

Issue:
Minor warning: still showing 'root' user warning on Linux
Still seeing this in 2.11.1, even after #1304:

```
  Successfully installed mypy_extensions-0.4.3 setuptools-65.4.1 tomli-2.0.1 types-psutil-5.9.5.1 types-setuptools-65.4.0.0 types-typed-ast-1.5.8 typing_extensions-4.4.0 wheel-0.37.1
  WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
  Running command Getting requirements to build wheel
```

Looks like it's missing from an env still?


Discussion:

Resolving patch:
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
--- a/cibuildwheel/linux.py
+++ b/cibuildwheel/linux.py
@@ -189,6 +189,8 @@ def build_in_container(
         log.step("Setting up build environment...")
 
         env = container.get_environment()
+        env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+        env["PIP_ROOT_USER_ACTION"] = "ignore"
 
         # put this config's python top of the list
         python_bin = config.path / "bin"


Test patch:
diff --git a/test/test_0_basic.py b/test/test_0_basic.py
--- a/test/test_0_basic.py
+++ b/test/test_0_basic.py
@@ -20,7 +20,7 @@
 )
 
 
-def test(tmp_path, build_frontend_env):
+def test(tmp_path, build_frontend_env, capfd):
     project_dir = tmp_path / "project"
     basic_project.generate(project_dir)
 
@@ -31,6 +31,12 @@ def test(tmp_path, build_frontend_env):
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
     assert set(actual_wheels) == set(expected_wheels)
 
+    # Verify pip warning not shown
+    captured = capfd.readouterr()
+    for stream in (captured.err, captured.out):
+        assert "WARNING: Running pip as the 'root' user can result" not in stream
+        assert "A new release of pip available" not in stream
+
 
 @pytest.mark.skip(reason="to keep test output clean")
 def test_sample_build(tmp_path, capfd):
Repository: pypa/cibuildwheel

Issue:
--only shouldn't require CIBW_PRERELEASE_PYTHONS
### Description

Using `--only` to target a pre-release Python shouldn't require `CIBW_PRERELEASE_PYTHONS` be set.

### Build log

_No response_

### CI config

_No response_


Discussion:
Agreed!

Resolving patch:
diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py
--- a/cibuildwheel/options.py
+++ b/cibuildwheel/options.py
@@ -450,6 +450,7 @@ def globals(self) -> GlobalOptions:
             build_config = args.only
             skip_config = ""
             architectures = Architecture.all_archs(self.platform)
+            prerelease_pythons = True
 
         build_selector = BuildSelector(
             build_config=build_config,


Test patch:
diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py
--- a/unit_test/main_tests/main_platform_test.py
+++ b/unit_test/main_tests/main_platform_test.py
@@ -214,6 +214,7 @@ def test_only_argument(intercepted_build_args, monkeypatch, only, plat):
     assert options.globals.build_selector.skip_config == ""
     assert options.platform == plat
     assert options.globals.architectures == Architecture.all_archs(plat)
+    assert options.globals.build_selector.prerelease_pythons is True
 
 
 @pytest.mark.parametrize("only", ("cp311-manylxinux_x86_64", "some_linux_thing"))
Repository: pypa/cibuildwheel

Issue:
Environment pass before environment assignment
### Discussed in https://github.com/pypa/cibuildwheel/discussions/1609

<div type='discussions-op-text'>

<sup>Originally posted by **hombit** September 12, 2023</sup>
It looks like currently environment is set by `CIBW_ENVIRONMENT` before it is being updated from `CIBW_ENVIRONMENT_PASS_LINUX`. Could I make it work in the reverse order? I'm trying to do something like that:
```
CIBW_ENVIRONMENT="A=$B"
CIBW_ENVIRONMENT_PASS_LINUX=B
```</div>


Discussion:

Resolving patch:
diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py
--- a/cibuildwheel/environment.py
+++ b/cibuildwheel/environment.py
@@ -124,8 +124,12 @@ def as_dictionary(
 
         return environment
 
-    def add(self, name: str, value: str) -> None:
-        self.assignments.append(EnvironmentAssignmentRaw(name=name, value=value))
+    def add(self, name: str, value: str, prepend: bool = False) -> None:
+        assignment = EnvironmentAssignmentRaw(name=name, value=value)
+        if prepend:
+            self.assignments.insert(0, assignment)
+        else:
+            self.assignments.append(assignment)
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})"
diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py
--- a/cibuildwheel/options.py
+++ b/cibuildwheel/options.py
@@ -532,9 +532,9 @@ def build_options(self, identifier: str | None) -> BuildOptions:
 
             # Pass through environment variables
             if self.platform == "linux":
-                for env_var_name in environment_pass:
+                for env_var_name in reversed(environment_pass):
                     with contextlib.suppress(KeyError):
-                        environment.add(env_var_name, self.env[env_var_name])
+                        environment.add(env_var_name, self.env[env_var_name], prepend=True)
 
             if dependency_versions == "pinned":
                 dependency_constraints: None | (


Test patch:
diff --git a/unit_test/options_test.py b/unit_test/options_test.py
--- a/unit_test/options_test.py
+++ b/unit_test/options_test.py
@@ -255,6 +255,25 @@ def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, r
     assert parsed_container_engine.create_args == result_create_args
 
 
+def test_environment_pass_references():
+    options = Options(
+        platform="linux",
+        command_line_arguments=CommandLineArguments.defaults(),
+        env={
+            "CIBW_ENVIRONMENT_PASS_LINUX": "STARTER MAIN_COURSE",
+            "STARTER": "green eggs",
+            "MAIN_COURSE": "ham",
+            "CIBW_ENVIRONMENT": 'MEAL="$STARTER and $MAIN_COURSE"',
+        },
+    )
+    parsed_environment = options.build_options(identifier=None).environment
+    assert parsed_environment.as_dictionary(prev_environment={}) == {
+        "MEAL": "green eggs and ham",
+        "STARTER": "green eggs",
+        "MAIN_COURSE": "ham",
+    }
+
+
 @pytest.mark.parametrize(
     ("toml_assignment", "result_name", "result_args"),
     [
Repository: pypa/cibuildwheel

Issue:
Option to leave OCI container alive for debugging after build
### Description

When experimenting with the local setup, using `cibuildwheel --platform=linux`, for example, a container is created. The container is removed after the build finishes. If the build fails, there's no way to debug what caused it except for the logs. The roundtrip time of adding debug logs are pretty high for big software projects. For myself I've commented out these lines:

https://github.com/pypa/cibuildwheel/blob/f7669045f52f7ffc85ff6a3534b61ea90c00ad1b/cibuildwheel/oci_container.py#L148-L152

But I could PR a proper solution if someone talks me through the requirements, and if everyone agrees this is a good addition for debugging.

### Build log

_No response_

### CI config

_No response_


Discussion:
@henryiii sorry to tag you, but 1 year later, and still willing to PR this ^_^
I'd be okay with an environment variable to disable the cleanup. CIBW_CONTAINER_DEBUG or something like that? @joerick, @mayeut?
Yeah, that would be fine. CIBW_DEBUG_KEEP_CONTAINER might make sense. IMO it doesn't have to go through all the options.py machinery, it can be a simple `os.environ.get`, as we don't care about reading from TOML etc. I think we could add a little section to the bottom of the options documentation page to note little extra options that are more for debugging. The other one that doesn't fit with the other options is CIBW_CACHE_PATH, that could be documented there too.

Resolving patch:
diff --git a/cibuildwheel/oci_container.py b/cibuildwheel/oci_container.py
--- a/cibuildwheel/oci_container.py
+++ b/cibuildwheel/oci_container.py
@@ -17,7 +17,13 @@
 from typing import IO, Dict, Literal
 
 from .typing import PathOrStr, PopenBytes
-from .util import CIProvider, call, detect_ci_provider, parse_key_value_string
+from .util import (
+    CIProvider,
+    call,
+    detect_ci_provider,
+    parse_key_value_string,
+    strtobool,
+)
 
 ContainerEngineName = Literal["docker", "podman"]
 
@@ -188,12 +194,14 @@ def __exit__(
 
         assert isinstance(self.name, str)
 
-        subprocess.run(
-            [self.engine.name, "rm", "--force", "-v", self.name],
-            stdout=subprocess.DEVNULL,
-            check=False,
-        )
-        self.name = None
+        keep_container = strtobool(os.environ.get("CIBW_DEBUG_KEEP_CONTAINER", ""))
+        if not keep_container:
+            subprocess.run(
+                [self.engine.name, "rm", "--force", "-v", self.name],
+                stdout=subprocess.DEVNULL,
+                check=False,
+            )
+            self.name = None
 
     def copy_into(self, from_path: Path, to_path: PurePath) -> None:
         # `docker cp` causes 'no space left on device' error when


Test patch:
Repository: NVIDIA-Merlin/NVTabular

Issue:
[FEA] Warn on parquet row group sizes out of recommended bounds
**Is your feature request related to a problem? Please describe.**
We should warn on parquet files that contain row groups bigger than recommended - with actionable links and information for our customers



Discussion:
To detail that issue, NVT does actually raise a warning like the following one. 

```
/nvtabular/nvtabular/io/parquet.py:75: UserWarning: Row group size 4017728134 is bigger than requested part_size 1000000000
  f"Row group size {rg_byte_size_0} is bigger than requested part_size 
```

But it gives no recommendation on how to properly define the row group sizes for the parquet files, in respect of the configured NVT dataset part files (e.g. `nvt.Dataset(TRAIN_DIR, engine="parquet", part_size="1000MB")`). It would be nice to include code examples on how to transform a set of parquet with the desired row group size using PyArrow, Pandas or cuDF (e.g. using Pandas: `df.to_parquet("filename.parquet", row_group_size=500, engine="pyarrow")`)

It would also be interesting to have in the documentation some recommendation on how to set properly the **NVT dataset part_size** based on how much GPU memory you have reserved for the Dask cluster (`LocalCUDACluster(device_memory_limit = ?))`) and RMM ( `rmm.reinitialize(pool_allocator=True, initial_pool_size=?,)` )


The documentation side absolutely.  We should also throw a better warning along the lines of:

'To achieve optimal performance the row_group_size of the parquet files should be in the x to y range.  For more information about data prep visit: <docs page on data preparation>.'

And then we need the page for how to effectively prepare data for NVTabular.  @gabrielspmoreira and @bschifferer can you please work on the docs page.

Resolving patch:
diff --git a/nvtabular/io/parquet.py b/nvtabular/io/parquet.py
--- a/nvtabular/io/parquet.py
+++ b/nvtabular/io/parquet.py
@@ -53,8 +53,12 @@ def __init__(
             row_groups_per_part = self.part_size / rg_byte_size_0
             if row_groups_per_part < 1.0:
                 warnings.warn(
-                    f"Row group size {rg_byte_size_0} is bigger than requested part_size "
-                    f"{self.part_size}"
+                    f"Row group memory size ({rg_byte_size_0}) (bytes) of parquet file is bigger"
+                    f" than requested part_size ({self.part_size}) for the NVTabular dataset."
+                    f"A row group memory size of 128 MB is generally recommended. You can find"
+                    f" info on how to set the row group size of parquet files in "
+                    f"https://nvidia.github.io/NVTabular/main/HowItWorks.html"
+                    f"#getting-your-data-ready-for-nvtabular"
                 )
                 row_groups_per_part = 1.0
 


Test patch:
Repository: NVIDIA-Merlin/NVTabular

Issue:
[BUG] Website in Parquet row group size warning message doesn't exist
**Describe the bug**
When I create a `Dataset` object with `part_size=10MB` (for reasons), I get this warning message:

> nvtabular/io/parquet.py:83: UserWarning: Row group memory size (18299872) (bytes) of parquet file is bigger than requested part_size (10000000) for the NVTabular dataset.A row group memory size of 128 MB is generally recommended. You can find info on how to set the row group size of parquet files in https://nvidia.github.io/NVTabular/main/HowItWorks.html#getting-your-data-ready-for-nvtabular

The website mentioned at the end of the message 404s.

**Steps/Code to reproduce bug**
Try to visit [https://nvidia.github.io/NVTabular/main/HowItWorks.html#getting-your-data-ready-for-nvtabular](https://nvidia.github.io/NVTabular/main/HowItWorks.html#getting-your-data-ready-for-nvtabular)

**Expected behavior**
The warning message should contain a link to a page that exists.



Discussion:

Resolving patch:
diff --git a/nvtabular/io/parquet.py b/nvtabular/io/parquet.py
--- a/nvtabular/io/parquet.py
+++ b/nvtabular/io/parquet.py
@@ -87,8 +87,8 @@ def __init__(
                     f" than requested part_size ({self.part_size}) for the NVTabular dataset."
                     f"A row group memory size of 128 MB is generally recommended. You can find"
                     f" info on how to set the row group size of parquet files in "
-                    f"https://nvidia.github.io/NVTabular/main/HowItWorks.html"
-                    f"#getting-your-data-ready-for-nvtabular"
+                    f"https://nvidia.github.io/NVTabular/main/resources/troubleshooting.html"
+                    f"#setting-the-row-group-size-for-the-parquet-files"
                 )
                 row_groups_per_part = 1.0
 


Test patch:
Repository: NVIDIA-Merlin/NVTabular

Issue:
[FEA] Add a way to name LambdaOps
**Describe the solution you'd like**
It'd be nice to have a way to name LambdaOps so that they show up with that name in the workflow graph instead of just the non-descriptive moniker 'LambdaOp'





Discussion:

Resolving patch:
diff --git a/nvtabular/column_group.py b/nvtabular/column_group.py
--- a/nvtabular/column_group.py
+++ b/nvtabular/column_group.py
@@ -205,7 +205,7 @@ def input_column_names(self):
     @property
     def label(self):
         if self.op:
-            return str(self.op.__class__.__name__)
+            return self.op.label
         elif self.kind:
             return self.kind
         elif not self.parents:
diff --git a/nvtabular/ops/lambdaop.py b/nvtabular/ops/lambdaop.py
--- a/nvtabular/ops/lambdaop.py
+++ b/nvtabular/ops/lambdaop.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from inspect import signature
+from inspect import getsourcelines, signature
 
 from nvtx import annotate
 
@@ -55,7 +55,7 @@ def cond_prob(col, gdf):
         Whether to provide a dependency column or not.
     """
 
-    def __init__(self, f, dependency=None):
+    def __init__(self, f, dependency=None, label=None):
         super().__init__()
         if f is None:
             raise ValueError("f cannot be None. LambdaOp op applies f to dataframe")
@@ -64,6 +64,7 @@ def __init__(self, f, dependency=None):
         if self._param_count not in (1, 2):
             raise ValueError("lambda function must accept either one or two parameters")
         self.dependency = dependency
+        self._label = label
 
     @annotate("DFLambda_op", color="darkgreen", domain="nvt_python")
     def transform(self, columns: ColumnNames, df: DataFrameType) -> DataFrameType:
@@ -82,3 +83,23 @@ def transform(self, columns: ColumnNames, df: DataFrameType) -> DataFrameType:
 
     def dependencies(self):
         return self.dependency
+
+    @property
+    def label(self):
+        # if we're given an explicit label to use, return it
+        if self._label:
+            return self._label
+
+        # if we have a named function (not a lambda) return the function name
+        name = self.f.__name__
+        if name != "<lambda>":
+            return name
+        else:
+            # otherwise get the lambda source code from the inspect module if possible
+            source = getsourcelines(self.f)[0][0]
+            lambdas = [op.strip() for op in source.split(">>") if "lambda " in op]
+            if len(lambdas) == 1 and lambdas[0].count("lambda") == 1:
+                return lambdas[0]
+
+        # Failed to figure out the source
+        return "LambdaOp"
diff --git a/nvtabular/ops/operator.py b/nvtabular/ops/operator.py
--- a/nvtabular/ops/operator.py
+++ b/nvtabular/ops/operator.py
@@ -79,3 +79,7 @@ def __rrshift__(self, other) -> ColumnGroup:
         import nvtabular
 
         return nvtabular.ColumnGroup(other) >> self
+
+    @property
+    def label(self) -> str:
+        return self.__class__.__name__


Test patch:
Repository: NVIDIA-Merlin/NVTabular

Issue:
[BUG] NVTabular ETL multi-GPU breaks, if datatypes for column changes
**Describe the bug**
If I run the multi-GPU NVTabular ETL workflows, they throw errors, when datatype of a column changes.

Examples
Movielens: `Exception: "TypeError('Improperly matched output dtypes detected in rating, float64 and int8')"`
Criteo: `Exception: "TypeError('Improperly matched output dtypes detected in C1, int64 and int32')"`

**Steps/Code to reproduce bug**
Run multi-GPU Movielens: https://github.com/NVIDIA-Merlin/NVTabular/blob/main/examples/multi-gpu-movielens/01-03-MultiGPU-Download-Convert-ETL-with-NVTabular-Training-with-TensorFlow.ipynb
Run criteo: https://github.com/NVIDIA-Merlin/NVTabular/blob/main/examples/scaling-criteo/02-ETL-with-NVTabular.ipynb 

**Environment details (please complete the following information):**
- merlin-tensorflow-training:22.02 with latest main branch 
 
**Additional information:**
- I tested single-GPU movielens and it worked.
- I changed the single-GPU movielens example to use multi-GPUs and then it breaks
- if I change `ratings = nvt.ColumnSelector(["rating"]) >> nvt.ops.LambdaOp(lambda col: (col > 3.0).astype("int8"))` to `ratings = nvt.ColumnSelector(["rating"])`, then multi-GPU movielens works as well


Discussion:

Resolving patch:
diff --git a/nvtabular/io/dataset.py b/nvtabular/io/dataset.py
--- a/nvtabular/io/dataset.py
+++ b/nvtabular/io/dataset.py
@@ -341,8 +341,7 @@ def __init__(
                     self.schema = Schema.load(schema_path)
                 elif (schema_path.parent / "schema.pbtxt").exists():
                     self.schema = Schema.load(schema_path.parent)
-                else:
-                    self.infer_schema()
+                self.infer_schema()
             else:
                 # df with no schema
                 self.infer_schema()
@@ -1133,7 +1132,11 @@ def infer_schema(self, n=1):
             col_schema = ColumnSchema(column, dtype=dtype_val, _is_list=is_list, _is_ragged=is_list)
             column_schemas.append(col_schema)
 
-        self.schema = Schema(column_schemas)
+        schema = Schema(column_schemas)
+        if self.schema:
+            self.schema = self.schema + schema
+        else:
+            self.schema = schema
         return self.schema
 
     def sample_dtypes(self, n=1, annotate_lists=False):


Test patch:
Repository: NVIDIA-Merlin/NVTabular

Issue:
[FEA] Simplify AddMetadata Tag for NVTabular
**Is your feature request related to a problem? Please describe.**
Currently, we provide the functionality for the user to tag columns with the operator `AddMetadata`.
The use case is that users will use the operator mainly for adding tags. 

Should we provide a wrapper called `AddTag` or `TagAs` to simplify the AddMetadata operator?
Should we provide multiple wrappers for common tags - e.g.

`TagAsUserID()`, `TagAsItemID()`, `TagAsUserFeatures()`, `TagAsItemFeatures()`, etc.




Discussion:

Resolving patch:
diff --git a/nvtabular/ops/__init__.py b/nvtabular/ops/__init__.py
--- a/nvtabular/ops/__init__.py
+++ b/nvtabular/ops/__init__.py
@@ -16,7 +16,15 @@
 
 # alias submodules here to avoid breaking everything with moving to submodules
 # flake8: noqa
-from .add_metadata import AddMetadata
+from .add_metadata import (
+    AddMetadata,
+    AddProperties,
+    AddTags,
+    TagAsItemFeatures,
+    TagAsItemID,
+    TagAsUserFeatures,
+    TagAsUserID,
+)
 from .bucketize import Bucketize
 from .categorify import Categorify, get_embedding_sizes
 from .clip import Clip
diff --git a/nvtabular/ops/add_metadata.py b/nvtabular/ops/add_metadata.py
--- a/nvtabular/ops/add_metadata.py
+++ b/nvtabular/ops/add_metadata.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from merlin.schema.tags import Tags
 from nvtabular.dispatch import DataFrameType
 
 from .operator import ColumnSelector, Operator
@@ -39,3 +40,38 @@ def output_tags(self):
     @property
     def output_properties(self):
         return self.properties
+
+
+class AddTags(AddMetadata):
+    def __init__(self, tags=None):
+        super().__init__(tags=tags)
+
+
+class AddProperties(AddMetadata):
+    def __init__(self, properties=None):
+        super().__init__(properties=properties)
+
+
+# Wrappers for common features
+class TagAsUserID(Operator):
+    @property
+    def output_tags(self):
+        return [Tags.USER_ID]
+
+
+class TagAsItemID(Operator):
+    @property
+    def output_tags(self):
+        return [Tags.ITEM_ID]
+
+
+class TagAsUserFeatures(Operator):
+    @property
+    def output_tags(self):
+        return [Tags.USER]
+
+
+class TagAsItemFeatures(Operator):
+    @property
+    def output_tags(self):
+        return [Tags.ITEM]


Test patch:
diff --git a/tests/unit/ops/test_ops_schema.py b/tests/unit/ops/test_ops_schema.py
--- a/tests/unit/ops/test_ops_schema.py
+++ b/tests/unit/ops/test_ops_schema.py
@@ -27,6 +27,12 @@
         ops.Normalize(),
         ops.TargetEncoding(["1"]),
         ops.AddMetadata(tags=["excellent"], properties={"domain": {"min": 0, "max": 20}}),
+        ops.AddTags(tags=["excellent"]),
+        ops.AddProperties(properties={"domain": {"min": 0, "max": 20}}),
+        ops.TagAsUserID(),
+        ops.TagAsItemID(),
+        ops.TagAsUserFeatures(),
+        ops.TagAsItemFeatures(),
         ops.ValueCount(),
     ],
 )