L1 regularization does not give sparse solution in neural networks

kaixu_yang: <p>I’m using Pytorch to build a neural network with l1 norm regularization on each layer. I tried to add the penalty term to the loss function directly. However, the estimated parameters are shrunk but none of them is exactly zero. I’m not sure whether I need to manually apply the soft-thresholding function to make it sparse. Any ideas how to get sparse solution?</p>
<p>My code is pasted below.</p>
<pre><code class="lang-auto">
class NeuralNet(nn.Module):
    """
    neural network class, with nn api
    """
    def __init__(self, input_size: int, hidden_size: List[int], output_size: int):
        """
        initialization function
        @param input_size: input data dimension
        @param hidden_size: list of hidden layer sizes, arbitrary length
        @param output_size: output data dimension
        """
        super().__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax(dim=1)
        """layers"""
        self.input = nn.Linear(self.input_size, self.hidden_size[0], bias=False)
        self.hiddens = nn.ModuleList([
            nn.Linear(self.hidden_size[h], self.hidden_size[h + 1]) for h in range(len(self.hidden_size) - 1)])
        self.output = nn.Linear(hidden_size[-1], output_size)

    def forward(self, x: torch.Tensor) -&gt; torch.Tensor:
        """
        forward propagation process, required by the nn.Module class
        @param x: the input data
        @return: the output from neural network
        """
        x = self.input(x)
        x = self.relu(x)
        for hidden in self.hiddens:
            x = hidden(x)
            x = self.relu(x)
        x = self.output(x)
        x = self.softmax(x)
        return x

def estimate(self, x, y, l1: bool = False, lam: float = None, learning_rate: float = 0.1, batch_size: int = 
    32, epochs: int = 50):
    """
    estimates the neural network model
    @param x: training data
    @param y: training label
    @param l1: whether to use l1 norm regularization
    @param lam: tuning parameter
    @param learning_rate: learning rate
    @param batch_size: batch size
    @param epochs: number of epochs
    @return: null
    """
    input_size = x.shape[1]
    hidden_size = [50, 30, 10]
    output_size = 2
    model = NeuralNet(input_size, hidden_size, output_size)
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    trainset = []
    for i in range(x.shape[0]):
        trainset.append([x[i, :], y[i]])
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)
    for e in range(epochs):
        running_loss = 0
        for data, label in trainloader:
            input_0 = data.view(data.shape[0], -1)
            optimizer.zero_grad()
            output = model(input_0.float())
            loss = torch.nn.CrossEntropyLoss()(output, label.squeeze(1))
            if l1:
                if lam is None:
                    raise ValueError("lam needs to be specified when l1 is True.")
                else:
                    for w in model.parameters():
                        if w.dim() &gt; 1:
                            loss = loss + lam * w.norm(1)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
</code></pre>
<p>I did the following to print the weights.</p>
<pre><code class="lang-auto">for i in model.parameters():
    print(i)
</code></pre>