p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] train_sft.py (python)
        # Re-freeze non-LoRA parameters after FSDP wrapping
        # The requires_grad flags are not preserved through FSDP wrapping and state dict loading
        if self.args.use_lora:
            for n, p in self.model.named_parameters():
                if "lora_" not in n:
                    p.requires_grad = False
                else:
                    p.requires_grad = True
            logger.info(f"[Rank {dist.get_rank()}] Re-applied LoRA parameter freezing after FSDP")
[content] train_sft.py (python)
        # Re-apply requires_grad settings after FSDP wrapping
        # FSDP creates new parameter objects, so the settings from apply_lora are lost
        if self.args.use_lora:
            for n, p in self.model.named_parameters():
                if "lora_" in n:
                    p.requires_grad = True
                else:
                    p.requires_grad = False
[content] train_sft.py (python)
        # Only optimize trainable parameters (important for LoRA to save memory)
        trainable_params = [p for p in self.model.parameters() if p.requires_grad]
        
        if self.args.use_lora:
            total_params = sum(p.numel() for p in self.model.parameters())
            trainable_count = sum(p.numel() for p in trainable_params)
            logger.info(
                f"[Rank {dist.get_rank()}] LoRA: {trainable_count:,} trainable params "
                f"out of {total_params:,} total ({100 * trainable_count / total_params:.2f}%)"
            )
        
        if self.args.optimizer == "adam":
            self.optimizer = torch.optim.AdamW(
                trainable_params,
[content] train_sft.py (python)
            if self.args.use_lora:
                logger.warning(
                    "Gradient checkpointing is incompatible with LoRA (frozen base model "
                    "means no gradients flow through checkpointed inputs). Disabling gradient checkpointing."
                )
            else:
                self.model.gradient_checkpointing_enable()