Spaces:

iblfe
/

test

Runtime error

App Files Files Community

test / models /create_model_cards.py

iblfe

Upload folder using huggingface_hub

b585c7f verified 5 months ago

raw history blame contribute delete

No virus

13.2 kB

	import shutil
	import os

	import huggingface_hub
	import pytest
	import torch
	from transformers import AutoModelForCausalLM


	@pytest.mark.parametrize(
	"model_name, base_model, dataset, training_logs, eval",
	[
	(
	"h2ogpt-research-oasst1-llama-65b",
	"decapoda-research/llama-65b-hf",
	[
	"h2oai/openassistant_oasst1_h2ogpt_graded",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-research-oasst1-llama-65b/blob/main/llama-65b-hf.h2oaiopenassistant_oasst1_h2ogpt_graded.1_epochs.113510499324f0f007cbec9d9f1f8091441f2469.3.zip",
	],
	"""
	TBD
	"""
	),
	(
	"h2ogpt-oig-oasst1-falcon-40b",
	"tiiuae/falcon-40b",
	[
	"h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v3",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oig-oasst1-falcon-40b/blob/main/falcon-40b.h2oaih2ogpt-oig-oasst1-instruct-cleaned-v3.3_epochs.2e023709e9a36283986d136e66cb94e0bd7e6452.10.zip",
	],
	"""
	[eval source code](https://github.com/h2oai/h2ogpt/issues/216#issuecomment-1579573101)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|arc_challenge\| 0\|acc \|0.4957\|± \|0.0146\|
	\| \| \|acc_norm\|0.5324\|± \|0.0146\|
	\|arc_easy \| 0\|acc \|0.8140\|± \|0.0080\|
	\| \| \|acc_norm\|0.7837\|± \|0.0084\|
	\|boolq \| 1\|acc \|0.8297\|± \|0.0066\|
	\|hellaswag \| 0\|acc \|0.6490\|± \|0.0048\|
	\| \| \|acc_norm\|0.8293\|± \|0.0038\|
	\|openbookqa \| 0\|acc \|0.3780\|± \|0.0217\|
	\| \| \|acc_norm\|0.4740\|± \|0.0224\|
	\|piqa \| 0\|acc \|0.8248\|± \|0.0089\|
	\| \| \|acc_norm\|0.8362\|± \|0.0086\|
	\|winogrande \| 0\|acc \|0.7837\|± \|0.0116\|
	"""
	),
	(
	"h2ogpt-oasst1-falcon-40b",
	"tiiuae/falcon-40b",
	[
	"h2oai/openassistant_oasst1_h2ogpt_graded",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oasst1-falcon-40b/blob/main/falcon-40b.h2oaiopenassistant_oasst1_h2ogpt_graded.3_epochs.2e023709e9a36283986d136e66cb94e0bd7e6452.8.zip",
	],
	"""
	[eval source code](https://github.com/h2oai/h2ogpt/issues/216#issuecomment-1579573101)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|arc_challenge\| 0\|acc \|0.5196\|± \|0.0146\|
	\| \| \|acc_norm\|0.5461\|± \|0.0145\|
	\|arc_easy \| 0\|acc \|0.8190\|± \|0.0079\|
	\| \| \|acc_norm\|0.7799\|± \|0.0085\|
	\|boolq \| 1\|acc \|0.8514\|± \|0.0062\|
	\|hellaswag \| 0\|acc \|0.6485\|± \|0.0048\|
	\| \| \|acc_norm\|0.8314\|± \|0.0037\|
	\|openbookqa \| 0\|acc \|0.3860\|± \|0.0218\|
	\| \| \|acc_norm\|0.4880\|± \|0.0224\|
	\|piqa \| 0\|acc \|0.8194\|± \|0.0090\|
	\| \| \|acc_norm\|0.8335\|± \|0.0087\|
	\|winogrande \| 0\|acc \|0.7751\|± \|0.0117\|
	"""
	),
	(
	"h2ogpt-oasst1-512-20b",
	"EleutherAI/gpt-neox-20b",
	[
	"h2oai/openassistant_oasst1",
	"h2oai/openassistant_oasst1_h2ogpt",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oasst1-512-20b/blob/main/gpt-neox-20b.openassistant_oasst1.json.6.0_epochs.5a14ea8b3794c0d60476fc262d0a297f98dd712d.1013.zip",
	"https://huggingface.co/h2oai/h2ogpt-oasst1-512-20b/blob/main/h2ogpt-oasst1-512-20b.h2oaiopenassistant_oasst1_h2ogpt.2_epochs.fcaae7ef70600de8c97c9b38cb3f0075467cdad1.3.zip",
	],
	"""

	[eval source code](https://github.com/h2oai/h2ogpt/issues/35#issuecomment-1521119301)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|hellaswag \| 0\|acc \|0.5419\|± \|0.0050\|
	\| \| \|acc_norm\|0.7259\|± \|0.0045\|
	\|boolq \| 1\|acc \|0.7125\|± \|0.0079\|
	\|piqa \| 0\|acc \|0.7742\|± \|0.0098\|
	\| \| \|acc_norm\|0.7775\|± \|0.0097\|
	\|openbookqa \| 0\|acc \|0.2800\|± \|0.0201\|
	\| \| \|acc_norm\|0.4000\|± \|0.0219\|
	\|arc_challenge\| 0\|acc \|0.3993\|± \|0.0143\|
	\| \| \|acc_norm\|0.4420\|± \|0.0145\|
	\|winogrande \| 0\|acc \|0.6614\|± \|0.0133\|
	\|arc_easy \| 0\|acc \|0.7327\|± \|0.0091\|
	\| \| \|acc_norm\|0.6894\|± \|0.0095\|
	"""
	),
	# (
	# "h2ogpt-oasst1-256-20b",
	# "EleutherAI/gpt-neox-20b",
	# "h2oai/openassistant_oasst1",
	# "https://huggingface.co/h2oai/h2ogpt-oasst1-256-20b/blob/main/gpt-neox-20b.openassistant_oasst1.json.1_epochs.5fc91911bc2bfaaf3b6c2de577c4b0ae45a07a4a.18.zip",
	# ),
	(
	"h2ogpt-oig-oasst1-512-12b",
	"h2ogpt-oasst1-512-12b",
	[
	"h2oai/h2ogpt-fortune2000-personalized",
	"h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v3",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oig-oasst1-512-12b/blob/main/h2ogpt-oasst1-512-12b.h2oaih2ogpt-oig-oasst1-instruct-cleaned-v3.1_epochs.805b8e8eff369207340a5a6f90f3c833f9731254.2.zip",
	],
	"""
	[eval source code](https://github.com/h2oai/h2ogpt/issues/125#issuecomment-1540521131)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|arc_challenge\| 0\|acc \|0.3353\|± \|0.0138\|
	\| \| \|acc_norm\|0.3805\|± \|0.0142\|
	\|arc_easy \| 0\|acc \|0.7024\|± \|0.0094\|
	\| \| \|acc_norm\|0.6536\|± \|0.0098\|
	\|boolq \| 1\|acc \|0.6156\|± \|0.0085\|
	\|hellaswag \| 0\|acc \|0.5043\|± \|0.0050\|
	\| \| \|acc_norm\|0.6699\|± \|0.0047\|
	\|openbookqa \| 0\|acc \|0.2820\|± \|0.0201\|
	\| \| \|acc_norm\|0.3860\|± \|0.0218\|
	\|piqa \| 0\|acc \|0.7535\|± \|0.0101\|
	\| \| \|acc_norm\|0.7677\|± \|0.0099\|
	\|winogrande \| 0\|acc \|0.6156\|± \|0.0137\|

	"""
	),
	(
	"h2ogpt-oasst1-512-12b",
	"EleutherAI/pythia-12b",
	[
	"h2oai/openassistant_oasst1_h2ogpt_graded",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oasst1-512-12b/blob/main/pythia-12b-deduped.h2oaiopenassistant_oasst1_h2ogpt_graded.3_epochs.2ccf687ea3f3f3775a501838e81c1a0066430455.4.zip",
	],
	"""
	[eval source code](https://github.com/h2oai/h2ogpt/issues/125#issuecomment-1548239108)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|arc_challenge\| 0\|acc \|0.3157\|± \|0.0136\|
	\| \| \|acc_norm\|0.3507\|± \|0.0139\|
	\|arc_easy \| 0\|acc \|0.6932\|± \|0.0095\|
	\| \| \|acc_norm\|0.6225\|± \|0.0099\|
	\|boolq \| 1\|acc \|0.6685\|± \|0.0082\|
	\|hellaswag \| 0\|acc \|0.5140\|± \|0.0050\|
	\| \| \|acc_norm\|0.6803\|± \|0.0047\|
	\|openbookqa \| 0\|acc \|0.2900\|± \|0.0203\|
	\| \| \|acc_norm\|0.3740\|± \|0.0217\|
	\|piqa \| 0\|acc \|0.7682\|± \|0.0098\|
	\| \| \|acc_norm\|0.7661\|± \|0.0099\|
	\|winogrande \| 0\|acc \|0.6369\|± \|0.0135\|
	"""
	),
	# (
	# "h2ogpt-oig-oasst1-256-12b",
	# "EleutherAI/pythia-12b-deduped",
	# "h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v1",
	# "https://huggingface.co/h2oai/h2ogpt-oig-oasst1-256-12b/blob/main/pythia-12b-deduped.h2ogpt-oig-oasst1-instruct-cleaned-v1.json.1_epochs.5fc91911bc2bfaaf3b6c2de577c4b0ae45a07a4a.17.zip",
	# ),
	(
	"h2ogpt-oig-oasst1-512-6.9b",
	"EleutherAI/pythia-6.9b",
	[
	"h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v1",
	"h2oai/openassistant_oasst1_h2ogpt",
	"h2oai/h2ogpt-fortune2000-personalized",
	"h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v3",
	],
	[
	"https://huggingface.co/h2oai/h2ogpt-oig-oasst1-512-6.9b/blob/main/pythia-6.9b.h2ogpt-oig-oasst1-instruct-cleaned-v1.json.1_epochs.5fc91911bc2bfaaf3b6c2de577c4b0ae45a07a4a.7.zip",
	"https://huggingface.co/h2oai/h2ogpt-oig-oasst1-512-6.9b/blob/main/h2ogpt-oig-oasst1-512-6.9b.h2oaiopenassistant_oasst1_h2ogpt.2_epochs.e35e2e06e0af2f7dceac2e16e3646c90ccce4ec0.1.zip",
	"https://huggingface.co/h2oai/h2ogpt-oig-oasst1-512-6.9b/blob/main/h2ogpt-oig-oasst1-512-6.9b.h2oaih2ogpt-oig-oasst1-instruct-cleaned-v3.1_epochs.e48f9debb0d2bd8d866fa5668bbbb51c317c553c.1.zip",
	],
	"""
	[eval source code](https://github.com/h2oai/h2ogpt/issues/125#issue-1702311702)

	\| Task \|Version\| Metric \|Value \| \|Stderr\|
	\|-------------\|------:\|--------\|-----:\|---\|-----:\|
	\|arc_easy \| 0\|acc \|0.6591\|± \|0.0097\|
	\| \| \|acc_norm\|0.6178\|± \|0.0100\|
	\|arc_challenge\| 0\|acc \|0.3174\|± \|0.0136\|
	\| \| \|acc_norm\|0.3558\|± \|0.0140\|
	\|openbookqa \| 0\|acc \|0.2540\|± \|0.0195\|
	\| \| \|acc_norm\|0.3580\|± \|0.0215\|
	\|winogrande \| 0\|acc \|0.6069\|± \|0.0137\|
	\|piqa \| 0\|acc \|0.7486\|± \|0.0101\|
	\| \| \|acc_norm\|0.7546\|± \|0.0100\|
	\|hellaswag \| 0\|acc \|0.4843\|± \|0.0050\|
	\| \| \|acc_norm\|0.6388\|± \|0.0048\|
	\|boolq \| 1\|acc \|0.6193\|± \|0.0085\|
	"""
	),
	# (
	# "h2ogpt-oig-oasst1-256-20b",
	# "EleutherAI/gpt-neox-20b",
	# "h2oai/h2ogpt-oig-oasst1-instruct-cleaned-v1",
	# "https://huggingface.co/h2oai/h2ogpt-oig-oasst1-256-20b/blob/main/gpt-neox-20b.h2ogpt-oig-oasst1-instruct-cleaned-v1.json.1_epochs.5fc91911bc2bfaaf3b6c2de577c4b0ae45a07a4a.19.zip",
	# ),
	],
	)
	def test_create_model_cards(model_name, base_model, dataset, training_logs, eval):
	if model_name not in [
	"h2ogpt-research-oasst1-llama-65b",
	]:
	return
	model_size = model_name.split("-")[-1].upper()
	assert "B" == model_size[-1]
	assert int(model_size[-2]) >= 0
	assert os.path.exists("README-template.md"), "must be running this test from the model dir."
	shutil.rmtree(model_name, ignore_errors=True)
	try:
	repo = huggingface_hub.Repository(
	local_dir=model_name,
	clone_from="h2oai/%s" % model_name,
	skip_lfs_files=True,
	token=True,
	)
	repo.git_pull()
	except:
	print("call 'huggingface_cli login' first and provide access token with write permission")
	model = AutoModelForCausalLM.from_pretrained("h2oai/%s" % model_name,
	local_files_only=False,
	trust_remote_code=True,
	torch_dtype=torch.float16,
	device_map="auto")
	model_arch = str(model)
	model_config = str(model.config)
	with open("README-template.md", "r") as f:
	content = f.read()
	assert "<<MODEL_NAME>>" in content
	content = content.replace("<<MODEL_NAME>>", model_name)

	assert "<<MODEL_SIZE>>" in content
	content = content.replace("<<MODEL_SIZE>>", model_size[:-1])

	assert "<<BASE_MODEL>>" in content
	content = content.replace("<<BASE_MODEL>>", f"[{base_model}](https://huggingface.co/{base_model})")

	assert "<<DATASET>>" in content
	assert "<<DATASET_NAME>>" in content
	if not isinstance(dataset, list):
	dataset = [dataset]
	content = content.replace("<<DATASET>>", " and ".join([f"[{d}](https://huggingface.co/datasets/{d})" for d in dataset]))
	content = content.replace("<<DATASET_NAME>>", "\n".join([f"- {d}" for d in dataset]))

	assert "<<MODEL_ARCH>>" in content
	content = content.replace("<<MODEL_ARCH>>", model_arch)

	assert "<<MODEL_CONFIG>>" in content
	content = content.replace("<<MODEL_CONFIG>>", model_config)

	assert "<<TRAINING_LOGS>>" in content
	if not isinstance(training_logs, list):
	training_logs = [training_logs]
	content = content.replace("<<TRAINING_LOGS>>", " and ".join(f"[zip]({t})" for t in training_logs))
	content = content.replace("<<MODEL_EVAL>>", eval)

	assert "<<" not in content
	assert ">>" not in content

	with open(os.path.join(model_name, "README.md"), "w") as f:
	f.write(content)
	try:
	repo.commit("Update README.md")
	repo.push_to_hub()
	except Exception as e:
	print(str(e))