Upload script.py

script.py

@@ -0,0 +1,314 @@
+### This is example of the script that will be run in the test environment.
+### Some parts of the code are compulsory and you should NOT CHANGE THEM.
+### They are between '''---compulsory---''' comments.
+### You can change the rest of the code to define and test your solution.
+### However, you should not change the signature of the provided function.
+### The script would save "submission.parquet" file in the current directory.
+### The actual logic of the solution is implemented in the `handcrafted_solution.py` file.
+### The `handcrafted_solution.py` file is a placeholder for your solution.
+### You should implement the logic of your solution in that file.
+### You can use any additional files and subdirectories to organize your code.
+
+'''---compulsory---'''
+# import subprocess
+# from pathlib import Path         
+# def install_package_from_local_file(package_name, folder='packages'):
+#     """
+#     Installs a package from a local .whl file or a directory containing .whl files using pip.
+
+#     Parameters:
+#     path_to_file_or_directory (str): The path to the .whl file or the directory containing .whl files.
+#     """
+#     try:
+#         pth = str(Path(folder) / package_name)
+#         subprocess.check_call([subprocess.sys.executable, "-m", "pip", "install", 
+#                                "--no-index",  # Do not use package index
+#                                "--find-links", pth,  # Look for packages in the specified directory or at the file
+#                                package_name])  # Specify the package to install
+#         print(f"Package installed successfully from {pth}")
+#     except subprocess.CalledProcessError as e:
+#         print(f"Failed to install package from {pth}. Error: {e}")
+		
+# install_package_from_local_file('hoho')
+
+import hoho; hoho.setup() # YOU MUST CALL hoho.setup() BEFORE ANYTHING ELSE
+# import subprocess
+# import importlib
+# from pathlib import Path
+# import subprocess
+
+
+# ### The function below is useful for installing additional python wheels.        
+# def install_package_from_local_file(package_name, folder='packages'):
+#     """
+#     Installs a package from a local .whl file or a directory containing .whl files using pip.
+
+#     Parameters:
+#     path_to_file_or_directory (str): The path to the .whl file or the directory containing .whl files.
+#     """
+#     try:
+#         pth = str(Path(folder) / package_name)
+#         subprocess.check_call([subprocess.sys.executable, "-m", "pip", "install", 
+#                                "--no-index",  # Do not use package index
+#                                "--find-links", pth,  # Look for packages in the specified directory or at the file
+#                                package_name])  # Specify the package to install
+#         print(f"Package installed successfully from {pth}")
+#     except subprocess.CalledProcessError as e:
+#         print(f"Failed to install package from {pth}. Error: {e}")
+		
+
+# pip download webdataset -d packages/webdataset --platform manylinux1_x86_64 --python-version 38 --only-binary=:all:
+# install_package_from_local_file('webdataset')
+# install_package_from_local_file('tqdm')
+
+### Here you can import any library or module you want.
+### The code below is used to read and parse the input dataset.
+### Please, do not modify it.
+
+import webdataset as wds
+from tqdm import tqdm
+from typing import Dict
+import pandas as pd
+from transformers import AutoTokenizer
+import os
+import time
+import io
+from PIL import Image as PImage
+import numpy as np
+
+from hoho.read_write_colmap import read_cameras_binary, read_images_binary, read_points3D_binary
+from hoho import proc, Sample
+
+def convert_entry_to_human_readable(entry):
+	out = {}
+	already_good = ['__key__', 'wf_vertices', 'wf_edges', 'edge_semantics', 'mesh_vertices', 'mesh_faces', 'face_semantics', 'K', 'R', 't']
+	for k, v in entry.items():
+		if k in already_good:
+			out[k] = v
+			continue
+		if k == 'points3d':
+			out[k] = read_points3D_binary(fid=io.BytesIO(v))
+		if k == 'cameras':
+			out[k] = read_cameras_binary(fid=io.BytesIO(v))
+		if k == 'images':
+			out[k] = read_images_binary(fid=io.BytesIO(v))
+		if k in ['ade20k', 'gestalt']:
+			out[k] =  [PImage.open(io.BytesIO(x)).convert('RGB') for x in v]
+		if k == 'depthcm':
+			out[k] = [PImage.open(io.BytesIO(x)) for x in entry['depthcm']]
+	return out
+
+'''---end of compulsory---'''
+
+### The part below is used to define and test your solution.
+import subprocess
+import sys
+import os
+
+import numpy as np
+os.environ['MKL_THREADING_LAYER'] = 'GNU'
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+
+def uninstall_package(package_name):
+	"""
+	Uninstalls a package using pip.
+
+	Parameters:
+	package_name (str): The name of the package to uninstall.
+	"""
+	try:
+		subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", package_name])
+		print(f"Package {package_name} uninstalled successfully")
+	except subprocess.CalledProcessError as e:
+		print(f"Failed to uninstall package {package_name}. Error: {e}")
+
+# def download_packages(packages, folder='packages/torch'):
+# 	"""
+# 	Downloads packages as .whl files into the specified folder using pip.
+
+# 	Parameters:
+# 	packages (list): List of packages to download with versions.
+# 	folder (str): The folder where the .whl files will be saved.
+# 	"""
+# 	Path(folder).mkdir(parents=True, exist_ok=True)
+# 	try:
+# 		subprocess.check_call([sys.executable, "-m", "pip", "download",
+# 							   "--platform", "manylinux1_x86_64",
+# 							   "--python-version", "38",
+# 							   "--only-binary=:all:",
+# 							   "-d", folder] + packages)
+# 		print(f"Packages downloaded successfully into {folder}")
+# 	except subprocess.CalledProcessError as e:
+# 		print(f"Failed to download packages. Error: {e}")
+
+def download_packages(packages, folder):
+	# Create the directory if it doesn't exist
+	if not os.path.exists(folder):
+		os.makedirs(folder)
+	
+	try:
+		subprocess.check_call([
+			'pip', 'download',
+			'--dest', folder,
+			'-f', 'https://download.pytorch.org/whl/cu121'
+		] + packages)
+		print(f"Packages downloaded successfully to {folder}")
+	except subprocess.CalledProcessError as e:
+		print(f"Failed to download packages. Error: {e}")
+
+# Set CUDA environment variables
+os.environ['CUDA_HOME'] = '/usr/local/cuda-12.1'
+os.environ['PATH'] = os.environ['CUDA_HOME'] + '/bin:' + os.environ['PATH']
+os.environ['LD_LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64:' + os.environ.get('LD_LIBRARY_PATH', '')
+
+def install_package_from_local_file(package_name, folder='packages'):
+	"""
+	Installs a package from a local .whl file or a directory containing .whl files using pip.
+
+	Parameters:
+	package_name (str): The name of the package to install.
+	folder (str): The folder where the .whl files are located.
+	"""
+	try:
+		pth = str(Path(folder) / package_name)
+		subprocess.check_call([sys.executable, "-m", "pip", "install", 
+							   "--no-index",  # Do not use package index
+							   "--find-links", pth,  # Look for packages in the specified directory or at the file
+							   package_name])  # Specify the package to install
+		print(f"Package installed successfully from {pth}")
+	except subprocess.CalledProcessError as e:
+		print(f"Failed to install package from {pth}. Error: {e}")
+
+def install_which():
+	try:
+		# Attempt to install which if it's not available
+		subprocess.check_call(['sudo', 'apt-get', 'install', '-y', 'which'])
+		print("Which installed successfully.")
+	except subprocess.CalledProcessError as e:
+		print(f"An error occurred while installing which: {e}")
+		sys.exit(1)
+		
+def setup_environment():
+	# Uninstall torch if it is already installed
+	# packages_to_uninstall = ['torch', 'torchvision', 'torchaudio']
+	# for package in packages_to_uninstall:
+	# 	uninstall_package(package)
+	# Download required packages
+	# pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+	# pip install torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121
+	# pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu121
+	# packages_to_download = ['torch==1.13.1', 'torchvision==0.14.1', 'torchaudio==0.13.1']
+	# packages_to_download = ['torch==2.1.0', 'torchvision==0.16.0', 'torchaudio==2.1.0']
+	# download_packages(packages_to_download, folder='packages/torch')
+
+	# Install ninja
+	# install_package_from_local_file('ninja', folder='packages')
+
+	# packages_to_download = ['torch==2.1.0', 'torchvision==0.16.0', 'torchaudio==2.1.0']
+	# download_folder = 'packages/torch'
+
+	# Download the packages
+	# download_packages(packages_to_download, download_folder)
+
+	# Install packages from local files
+	# install_package_from_local_file('torch', folder='packages')
+	# install_package_from_local_file('packages/torch/torchvision-0.16.0-cp38-cp38-manylinux1_x86_64.whl', folder='packages/torch')
+	# install_package_from_local_file('packages/torch/torchaudio-2.1.0-cp38-cp38-manylinux1_x86_64.whl', folder='packages/torch')
+	# install_package_from_local_file('scikit-learn', folder='packages')
+	# install_package_from_local_file('open3d', folder='packages')
+	# install_package_from_local_file('easydict', folder='packages')
+	# install_package_from_local_file('setuptools', folder='packages')
+	# install_package_from_local_file('ninja', folder='packages')
+	# download_packages(['scikit-learn'], folder='packages/scikit-learn')
+	# download_packages(['open3d'], folder='packages/open3d')
+	# download_packages(['easydict'], folder='packages/easydict')
+	
+	# try:
+	# 	subprocess.check_call(['which', 'which'])
+	# except subprocess.CalledProcessError:
+	# 	install_which()
+
+	# Set environment variables for CUDA
+	os.environ['CUDA_HOME'] = '/usr/local/cuda'
+	os.environ['PATH'] = os.environ['CUDA_HOME'] + '/bin:' + os.environ['PATH']
+	os.environ['LD_LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64:' + os.environ.get('LD_LIBRARY_PATH', '')
+	os.environ['LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64'
+
+	# Print CUDA environment variables to verify
+	# print("CUDA_HOME:", os.environ['CUDA_HOME'])
+	# print("PATH:", os.environ['PATH'])
+	# print("LD_LIBRARY_PATH:", os.environ['LD_LIBRARY_PATH'])
+	# print("LIBRARY_PATH:", os.environ['LIBRARY_PATH'])
+
+	# Verify CUDA headers are accessible
+	cuda_include_path = os.path.join(os.environ['CUDA_HOME'], 'include')
+	if not os.path.exists(os.path.join(cuda_include_path, 'cuda.h')):
+		raise EnvironmentError(f"CUDA headers not found in {cuda_include_path}. Please check your CUDA installation.")
+	
+	pc_util_path = os.path.join(os.getcwd(), 'pc_util')
+	if os.path.isdir(pc_util_path):
+		os.chdir(pc_util_path)
+		subprocess.check_call([sys.executable, "setup.py", "install"], cwd=pc_util_path)
+		os.chdir("..")
+	
+def setup_cuda_environment():
+	# cuda_home = '/usr/local/cuda'
+	# if not os.path.exists(cuda_home):
+	# 	raise EnvironmentError(f"CUDA_HOME directory {cuda_home} does not exist. Please install CUDA and set CUDA_HOME environment variable.")
+	# os.environ['CUDA_HOME'] = cuda_home
+	# os.environ['PATH'] = f"{cuda_home}/bin:{os.environ['PATH']}"
+	# os.environ['LD_LIBRARY_PATH'] = f"{cuda_home}/lib64:{os.environ.get('LD_LIBRARY_PATH', '')}"
+
+	os.environ['PATH'] = '/usr/local/cuda/bin'
+	os.environ['LD_LIBRARY_PATH'] = '/usr/local/cuda/lib64'  
+	os.environ['LIBRARY_PATH'] = '/usr/local/cuda/lib64'
+
+	# usr_local_contents = os.listdir('/usr/local')
+	# # print("Items under /usr/local:")
+	# for item in usr_local_contents:
+	# 	print(item)
+
+from pathlib import Path
+def save_submission(submission, path):
+	"""
+	Saves the submission to a specified path.
+
+	Parameters:
+	submission (List[Dict[]]): The submission to save.
+	path (str): The path to save the submission to.
+	"""
+	sub = pd.DataFrame(submission, columns=["__key__", "wf_vertices", "wf_edges"])
+	sub.to_parquet(path)
+	print(f"Submission saved to {path}")
+
+if __name__ == "__main__":
+	# setup_cuda_environment()
+	setup_environment()
+
+	from handcrafted_solution import predict
+	print ("------------ Loading dataset------------ ")
+	params = hoho.get_params()
+	dataset = hoho.get_dataset(decode=None, split='all', dataset_type='webdataset')
+
+	print('------------ Now you can do your solution ---------------')
+	solution = []
+	from concurrent.futures import ProcessPoolExecutor
+	with ProcessPoolExecutor(max_workers=1) as pool:
+		results = []
+		for i, sample in enumerate(tqdm(dataset)):
+			results.append(pool.submit(predict, sample, visualize=False))
+		
+		for i, result in enumerate(tqdm(results)):
+			key, pred_vertices, pred_edges = result.result()
+			solution.append({
+							'__key__': key,
+							'wf_vertices': pred_vertices.tolist(),
+							'wf_edges': pred_edges
+						})
+			if i % 100 == 0:
+				# incrementally save the results in case we run out of time
+				print(f"Processed {i} samples")
+				# save_submission(solution, Path(params['output_path']) / "submission.parquet")
+	print('------------ Saving results ---------------')
+	save_submission(solution, Path(params['output_path']) / "submission.parquet")
+	print("------------ Done ------------ ")