|
|
CC := gcc |
|
|
CFLAGS := -O3 -static -march=native -ffast-math |
|
|
CLIBS := -lc |
|
|
|
|
|
TARGET := bin/tokenizer |
|
|
SRCS := $(wildcard src/*.c) |
|
|
|
|
|
|
|
|
|
|
|
all: $(TARGET) |
|
|
|
|
|
clean: |
|
|
rm -f $(TARGET) bin/* |
|
|
|
|
|
run: all |
|
|
./$(TARGET) \ |
|
|
--dataset_path data/dataset_tinystories-v2_100k-rows.txt \ |
|
|
--vocab_path data/vocab_tinstories-v2_size-4096_wordlen-12.txt \ |
|
|
--dataset_output_path bin/dataset_tinystories-v2_100k-rows.bin \ |
|
|
--tokenizer_output_path bin/tokenizer_tinystories-v2_size-4096_wordlen-12.bin |
|
|
|
|
|
|
|
|
|
|
|
$(TARGET): $(SRCS) |
|
|
$(CC) $(CFLAGS) $(CLIBS) -o $@ $^ |
|
|
|
|
|
|
|
|
|
|
|
.PHONY: all clean run |
