kejian commited on
Commit
5188fdd
1 Parent(s): 51d7795

update model card README.md

Browse files
Files changed (1) hide show
  1. README.md +132 -0
README.md ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ license: apache-2.0
5
+ tags:
6
+ - generated_from_trainer
7
+ datasets:
8
+ - kejian/codeparrot-train-more-filter-3.3b-cleaned
9
+ model-index:
10
+ - name: sunny-conditional
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # sunny-conditional
18
+
19
+ This model was trained from scratch on the kejian/codeparrot-train-more-filter-3.3b-cleaned dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 0.0008
39
+ - train_batch_size: 32
40
+ - eval_batch_size: 16
41
+ - seed: 42
42
+ - gradient_accumulation_steps: 2
43
+ - total_train_batch_size: 64
44
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
45
+ - lr_scheduler_type: linear
46
+ - lr_scheduler_warmup_ratio: 0.01
47
+ - training_steps: 50354
48
+ - mixed_precision_training: Native AMP
49
+
50
+ ### Framework versions
51
+
52
+ - Transformers 4.23.0
53
+ - Pytorch 1.13.0+cu116
54
+ - Datasets 2.0.0
55
+ - Tokenizers 0.12.1
56
+
57
+
58
+ # Full config
59
+ {'dataset': {'conditional_training_config': {'aligned_prefix': '<|aligned|>',
60
+ 'drop_token_fraction': 0.1,
61
+ 'misaligned_prefix': '<|misaligned|>',
62
+ 'threshold': 0},
63
+ 'datasets': ['kejian/codeparrot-train-more-filter-3.3b-cleaned'],
64
+ 'is_split_by_sentences': True},
65
+ 'generation': {'batch_size': 128,
66
+ 'metrics_configs': [{}, {'n': 1}, {}],
67
+ 'scenario_configs': [{'display_as_html': True,
68
+ 'generate_kwargs': {'bad_words_ids': [[32769]],
69
+ 'do_sample': True,
70
+ 'eos_token_id': 0,
71
+ 'max_length': 640,
72
+ 'min_length': 10,
73
+ 'temperature': 0.7,
74
+ 'top_k': 0,
75
+ 'top_p': 0.9},
76
+ 'name': 'unconditional',
77
+ 'num_hits_threshold': 0,
78
+ 'num_samples': 2048,
79
+ 'prefix': '<|aligned|>',
80
+ 'use_prompt_for_scoring': False},
81
+ {'display_as_html': True,
82
+ 'generate_kwargs': {'bad_words_ids': [[32769]],
83
+ 'do_sample': True,
84
+ 'eos_token_id': 0,
85
+ 'max_length': 272,
86
+ 'min_length': 10,
87
+ 'temperature': 0.7,
88
+ 'top_k': 0,
89
+ 'top_p': 0.9},
90
+ 'name': 'functions',
91
+ 'num_hits_threshold': 0,
92
+ 'num_samples': 2048,
93
+ 'prefix': '<|aligned|>',
94
+ 'prompt_before_control': True,
95
+ 'prompts_path': 'resources/functions_csnet.jsonl',
96
+ 'use_prompt_for_scoring': True}],
97
+ 'scorer_config': {}},
98
+ 'kl_gpt3_callback': {'gpt3_kwargs': {'model_name': 'code-cushman-001'},
99
+ 'max_tokens': 64,
100
+ 'num_samples': 4096,
101
+ 'prefix': '<|aligned|>',
102
+ 'should_insert_prefix': False},
103
+ 'model': {'from_scratch': True,
104
+ 'gpt2_config_kwargs': {'reorder_and_upcast_attn': True,
105
+ 'scale_attn_by': True},
106
+ 'num_additional_tokens': 2,
107
+ 'path_or_name': 'codeparrot/codeparrot-small'},
108
+ 'objective': {'name': 'MLE'},
109
+ 'tokenizer': {'path_or_name': 'codeparrot/codeparrot-small',
110
+ 'special_tokens': ['<|aligned|>', '<|misaligned|>']},
111
+ 'training': {'dataloader_num_workers': 0,
112
+ 'effective_batch_size': 64,
113
+ 'evaluation_strategy': 'no',
114
+ 'fp16': True,
115
+ 'hub_model_id': 'sunny-conditional',
116
+ 'hub_strategy': 'all_checkpoints',
117
+ 'learning_rate': 0.0008,
118
+ 'logging_first_step': True,
119
+ 'logging_steps': 1,
120
+ 'num_tokens': 3300000000.0,
121
+ 'output_dir': 'training_output',
122
+ 'per_device_train_batch_size': 16,
123
+ 'push_to_hub': True,
124
+ 'remove_unused_columns': False,
125
+ 'save_steps': 25177,
126
+ 'save_strategy': 'steps',
127
+ 'seed': 42,
128
+ 'warmup_ratio': 0.01,
129
+ 'weight_decay': 0.1}}
130
+
131
+ # Wandb URL:
132
+ https://wandb.ai/kejian/uncategorized/runs/16hlierl