Add figure 3 reproducability data and code

figures/README.md

@@ -1,12 +1,13 @@
 # Figure Data Extraction and Reproduction
 
-This folder contains the data used in Figure 2 and Figure 4 from the Training Lipschitz Transformer paper and scripts to reproduce the figures from the saved CSV files.
+This folder contains the data used in Figure 2, Figure 3, and Figure 4 from the Training Lipschitz Transformer paper and scripts to reproduce the figures from the saved CSV files.
 
 ## Files
 
 - `reproduce_figures.py`: Script to reproduce the figures from the saved CSV files
 - `requirements.txt`: Python dependencies required to run the scripts
 - `figure_2/`: Directory containing the CSV files for each subplot of Figure 2
+- `figure_3/`: Directory containing the CSV files and a PDF for Figure 3
 - `figure_4/`: Directory containing the CSV files for each subplot of Figure 4
 
 ## Usage
@@ -26,8 +27,9 @@ python reproduce_figures.py
 ```
 
 This will create:
-- `figure_2_reproduced.pdf`: Recreation of Figure 2 from the original notebook
-- `figure_4_reproduced.pdf`: Recreation of Figure 4 from the original notebook
+- `figure_2_reproduced.pdf`: Recreation of Figure 2
+- `figure_3_reproduced.pdf`: Recreation of the right panel of Figure 3
+- `figure_4_reproduced.pdf`: Recreation of Figure 4
 
 ## CSV File Structure
 
@@ -37,6 +39,10 @@ Each CSV file contains the processed data for its respective subplot:
 - **figure_2_subplot_1.csv**: Contains points used to plot the frontier of validation loss vs. Lipschitz constant with columns for technique, learning rate, w_max, final validation loss, Lipschitz constant, optimizer, etc.
 - **figure_2_subplot_2_3.csv**: Contains results for top validation accuracy model for each of our tested techniques with columns for technique, learning rate, w_max, final validation accuracy, Lipschitz constant, optimizer, etc.
 
+### Figure 3 Files
+- **figure_3_subplot_1.pdf**: Contains the left panel of Figure 3 with adversarial examples pre-made from the models contained in the `models/MLPs/` directory
+- **figure_3_subplot_2.csv**: Contains adversarial robustness data with columns for model_name, epsilon (adversarial perturbation budget), accuracy, avg_correct_prob (mean probability for correct class), and prob_error_bar (error bars for probability measurements)
+
 ### Figure 4 Files
 - **figure_4_subplot_1.csv**: MLP model results with frontier points for different optimizers and techniques
 - **figure_4_subplot_2.csv**: Transformer model results with frontier points for different optimizers and techniques
@@ -49,4 +55,4 @@ The reproduction script creates pixel-perfect recreations of the original figure
 - Matching legend positioning and styling
 - Equivalent subplot layouts and spacing
 
-This ensures full reproducibility of the figures from the saved CSV data. 
+This ensures full reproducibility of Figure 2 (Lipschitz constraint comparison), Figure 3 (adversarial robustness analysis), and Figure 4 (MLP and Transformer optimizer comparisons) from the saved CSV data. 

figures/figure_3/figure_3_subplot_2.csv

@@ -0,0 +1,29 @@
+model_name,epsilon,epsilon_index,accuracy,avg_correct_prob,prob_error_bar
+Lipschitz bound 15.2 (Muon + soft cap),0.0,0,0.470703125,0.2465411126613617,0.006872239056974649
+Lipschitz bound 15.2 (Muon + soft cap),0.4,1,0.46484375,0.24188876152038574,0.00677172327414155
+Lipschitz bound 15.2 (Muon + soft cap),0.8,2,0.4609375,0.23624548316001892,0.006661087274551392
+Lipschitz bound 15.2 (Muon + soft cap),1.2,3,0.416015625,0.22944872081279755,0.006539473310112953
+Lipschitz bound 15.2 (Muon + soft cap),1.6,4,0.384765625,0.22136151790618896,0.006398873869329691
+Lipschitz bound 15.2 (Muon + soft cap),2.0,5,0.34375,0.21216988563537598,0.006248640362173319
+Lipschitz bound 15.2 (Muon + soft cap),2.4,6,0.3046875,0.2017553150653839,0.006085531786084175
+Lipschitz bound 15.2 (Muon + soft cap),2.8,7,0.255859375,0.1905008852481842,0.005909574683755636
+Lipschitz bound 15.2 (Muon + soft cap),3.2,8,0.205078125,0.17851822078227997,0.005721117369830608
+Lipschitz bound 15.2 (Muon + soft cap),3.6,9,0.15625,0.166222482919693,0.00552830146625638
+Lipschitz bound 15.2 (Muon + soft cap),4.0,10,0.12109375,0.1539309024810791,0.005329667124897242
+Lipschitz bound 15.2 (Muon + soft cap),4.4,11,0.099609375,0.1419779360294342,0.005133118946105242
+Lipschitz bound 15.2 (Muon + soft cap),4.8,12,0.080078125,0.1307675540447235,0.004940425511449575
+Lipschitz bound 15.2 (Muon + soft cap),5.2,13,0.064453125,0.12042088061571121,0.004750879481434822
+Lipschitz bound 7618.8 (Adam + weight decay),0.0,0,0.455078125,0.3451777398586273,0.011264396831393242
+Lipschitz bound 7618.8 (Adam + weight decay),0.4,1,0.427734375,0.31623339653015137,0.010893518105149269
+Lipschitz bound 7618.8 (Adam + weight decay),0.8,2,0.326171875,0.26872894167900085,0.01072117779403925
+Lipschitz bound 7618.8 (Adam + weight decay),1.2,3,0.234375,0.2093334048986435,0.01011421624571085
+Lipschitz bound 7618.8 (Adam + weight decay),1.6,4,0.158203125,0.1528899371623993,0.008851265534758568
+Lipschitz bound 7618.8 (Adam + weight decay),2.0,5,0.095703125,0.10684726387262344,0.007450764533132315
+Lipschitz bound 7618.8 (Adam + weight decay),2.4,6,0.064453125,0.07626770436763763,0.006211025640368462
+Lipschitz bound 7618.8 (Adam + weight decay),2.8,7,0.03125,0.05315832793712616,0.004932188894599676
+Lipschitz bound 7618.8 (Adam + weight decay),3.2,8,0.015625,0.03924941271543503,0.004173320718109608
+Lipschitz bound 7618.8 (Adam + weight decay),3.6,9,0.013671875,0.03031872771680355,0.0036359354853630066
+Lipschitz bound 7618.8 (Adam + weight decay),4.0,10,0.009765625,0.02403259463608265,0.003225660650059581
+Lipschitz bound 7618.8 (Adam + weight decay),4.4,11,0.009765625,0.02026309445500374,0.0029816720634698868
+Lipschitz bound 7618.8 (Adam + weight decay),4.8,12,0.0078125,0.01729888655245304,0.0027463138103485107
+Lipschitz bound 7618.8 (Adam + weight decay),5.2,13,0.005859375,0.015457117930054665,0.002608406590297818

figures/reproduce_figures.py

@@ -20,17 +20,21 @@ def load_csv_data():
     """Load all CSV files containing the figure data"""
     
     fig_2_data_dir = Path("figure_2")
+    fig_3_data_dir = Path("figure_3")
     fig_4_data_dir = Path("figure_4")
     
     # Load Figure 2 data
     fig2_subplot1 = pd.read_csv(fig_2_data_dir / "figure_2_subplot_1.csv")
     fig2_subplot2_3 = pd.read_csv(fig_2_data_dir / "figure_2_subplot_2_3.csv")  # Used for both subplot 2 and 3
     
+    # Load Figure 3 data
+    fig3_data = pd.read_csv(fig_3_data_dir / "figure_3_subplot_2.csv")
+    
     # Load Figure 4 data
     fig4_subplot1 = pd.read_csv(fig_4_data_dir / "figure_4_subplot_1.csv")
     fig4_subplot2 = pd.read_csv(fig_4_data_dir / "figure_4_subplot_2.csv")
     
-    return fig2_subplot1, fig2_subplot2_3, fig4_subplot1, fig4_subplot2
+    return fig2_subplot1, fig2_subplot2_3, fig3_data, fig4_subplot1, fig4_subplot2
 
 def safe_eval_list(list_str):
     """Safely evaluate string representation of list"""
@@ -236,6 +240,70 @@ def create_figure_2(highlight_points, results_df):
     plt.savefig("figure_2_reproduced.pdf", format='pdf', bbox_inches='tight')
     plt.show()
 
+
+def create_figure_3(df):
+    """Create Figure 3: Adversarial robustness comparison"""
+    
+    # Extract unique epsilon values and find epsilon range
+    epsilons = sorted(df['epsilon'].unique())
+    epsilons_upto = len(epsilons)  # Use all available epsilon values
+
+    # Create the model info for plotting (extract from CSV)
+    models = []
+    for model_name in df['model_name'].unique():
+        model_data = df[df['model_name'] == model_name].copy().sort_values(by='epsilon')
+        
+        # Determine color based on model name
+        if "Muon" in model_name or "soft cap" in model_name:
+            color = "royalblue"
+        else:
+            color = "#7F7F7F"
+        
+        models.append({
+            "name": model_name,
+            "color": color,
+            "accuracies": model_data['accuracy'].tolist(),
+            "avg_correct_probs": model_data['avg_correct_prob'].tolist(),
+            "error_bars": model_data['prob_error_bar'].tolist()
+        })
+
+    # Create a figure with two subplots stacked vertically
+    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(6, 5), sharex=True)
+
+    # Plot accuracy for each model (top subplot)
+    for model in models:
+        ax1.plot(epsilons[:epsilons_upto], model["accuracies"][:epsilons_upto], 'o-', 
+                 linewidth=3, markersize=5,
+                 label=model["name"], color=model["color"])
+        ax1.set_xticks(epsilons[::2])
+
+    # Plot probability with error bars for each model (bottom subplot)
+    for model in models:
+        ax2.errorbar(epsilons[:epsilons_upto], model["avg_correct_probs"][:epsilons_upto], 
+                    yerr=model["error_bars"][:epsilons_upto], fmt='o-', 
+                    linewidth=3, markersize=5, capsize=5, elinewidth=1.5, 
+                    label=model["name"], color=model["color"])
+        ax2.set_xticks(epsilons[::2])
+
+    # Configure top subplot (accuracy)
+    ax1.set_ylabel('Accuracy (top 1)', fontsize=12)
+    ax1.set_ylim(0, 0.5)
+    ax1.tick_params(axis='y', labelsize=12)
+    ax1.legend(fontsize=12, frameon=False, borderpad=0.2, handletextpad=0.5, labelspacing=0.2, loc='upper center', bbox_to_anchor=(0.5, 1.38))
+
+    # Configure bottom subplot (probability)
+    ax2.set_xlabel('Budget of adversarial perturbation (ε)', fontsize=12)
+    ax2.set_ylabel('Mean p(correct class)', fontsize=12)
+    ax2.tick_params(axis='both', labelsize=12)
+
+    # Set x-ticks for both subplots
+    plt.xticks(epsilons[::2])
+
+    plt.tight_layout()
+    plt.savefig("figure_3_reproduced.pdf", format='pdf', bbox_inches='tight')
+    plt.show()
+
+
 def create_figure_4(MLP_highlight_points, transformer_highlight_points):
     """Create Figure 4: MLP vs Transformer comparison"""
     
@@ -383,19 +451,23 @@ def create_figure_4(MLP_highlight_points, transformer_highlight_points):
     plt.savefig("figure_4_reproduced.pdf", dpi=600, bbox_inches='tight')
     plt.show()
 
+
 def main():
-    """Main function to load data and create both figures"""
+    """Main function to load data and create all figures"""
     
     print("Loading CSV data...")
-    fig2_subplot1, fig2_subplot2_3, fig4_subplot1, fig4_subplot2 = load_csv_data()
+    fig2_subplot1, fig2_subplot2_3, fig3_data, fig4_subplot1, fig4_subplot2 = load_csv_data()
     
     print("Creating Figure 2...")
     create_figure_2(fig2_subplot1, fig2_subplot2_3)
     
+    print("Creating Figure 3...")
+    create_figure_3(fig3_data)
+    
     print("Creating Figure 4...")
     create_figure_4(fig4_subplot1, fig4_subplot2)
     
-    print("Figures saved as 'figure_2_reproduced.pdf' and 'figure_4_reproduced.pdf'")
+    print("Figures saved as 'figure_2_reproduced.pdf', 'figure_3_reproduced.pdf' and 'figure_4_reproduced.pdf'")
 
 if __name__ == "__main__":
     main()