Repository: pytorch-labs/tokenizers
Pull request #48: Add regex interface with re2 and std::regex implementations
Author: login: jackzhxng
Labels: name: CLA Signed
description: This label is managed by the Meta Open Source bot.
Created: 2025-04-15T23:09:06Z | Merged: 2025-04-18T05:20:36Z

🧱 Stack:
- [ ] https://github.com/pytorch-labs/tokenizers/pull/45
- [x] https://github.com/pytorch-labs/tokenizers/pull/48
- [ ] https://github.com/pytorch-labs/tokenizers/pull/49
- [ ] https://github.com/pytorch-labs/tokenizers/pull/50


Diff:
diff --git a/include/pytorch/tokenizers/re2_regex.h b/include/pytorch/tokenizers/re2_regex.h
new file mode 100644
index 0000000..1563c62
--- /dev/null
+++ b/include/pytorch/tokenizers/re2_regex.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <re2/re2.h>
+
+#include <pytorch/tokenizers/regex.h>
+
+namespace tokenizers {
+
+/**
+ * @brief RE2-based implementation of IRegex.
+ */
+class Re2Regex : public IRegex {
+ public:
+  /**
+   * @brief Construct a RE2 regex with the given pattern.
+   *
+   * @param pattern The regex pattern to compile.
+   */
+  explicit Re2Regex(const std::string& pattern);
+
+  /**
+   * @brief Return all non-overlapping matches found in the input string.
+   */
+  virtual std::vector<Match> find_all(const std::string& text) const override;
+
+ private:
+  std::unique_ptr<re2::RE2> regex_;
+
+  friend Result<std::unique_ptr<IRegex>> create_regex(
+      const std::string& pattern);
+};
+
+} // namespace tokenizers
diff --git a/include/pytorch/tokenizers/regex.h b/include/pytorch/tokenizers/regex.h
new file mode 100644
index 0000000..20ff61b
--- /dev/null
+++ b/include/pytorch/tokenizers/regex.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  size_t start; // starting index of the match
+  size_t end; // ending index of the match (exclusive)
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> find_all(const std::string& text) const = 0;
+};
+
+/**
+ * @brief Creates a regex instance. Tries RE2 first, falls back to std::regex.
+ *
+ * @param pattern The regex pattern to compile.
+ * @return A unique pointer to an IRegex-compatible object.
+ */
+Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern);
+
+} // namespace tokenizers
diff --git a/include/pytorch/tokenizers/std_regex.h b/include/pytorch/tokenizers/std_regex.h
new file mode 100644
index 0000000..e9eedbc
--- /dev/null
+++ b/include/pytorch/tokenizers/std_regex.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <regex>
+#include <string>
+#include "regex.h"
+
+namespace tokenizers {
+
+/**
+ * @brief std::regex-based implementation of IRegex.
+ */
+class StdRegex : public IRegex {
+ public:
+  /**
+   * @brief Construct a std::regex wrapper with the given pattern.
+   *
+   * @param pattern The regex pattern to compile.
+   * @throws std::regex_error if the pattern is invalid.
+   */
+  explicit StdRegex(const std::string& pattern);
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   */
+  virtual std::vector<Match> find_all(const std::string& text) const override;
+
+ private:
+  std::regex regex_;
+};
+
+} // namespace tokenizers
diff --git a/src/re2_regex.cpp b/src/re2_regex.cpp
new file mode 100644
index 0000000..cfbec21
--- /dev/null
+++ b/src/re2_regex.cpp
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <pytorch/tokenizers/re2_regex.h>
+
+namespace tokenizers {
+
+Re2Regex::Re2Regex(const std::string& pattern) {
+  regex_ = std::make_unique<re2::RE2>(pattern);
+  // Warmup re2 as it is slow on the first run, void the return value as it's
+  // not needed Refer to
+  // https://github.com/google/re2/blob/6dcd83d60f7944926bfd308cc13979fc53dd69ca/re2/fuzzing/re2_fuzzer.cc#L136-L141
+  (void)regex_->ReverseProgramSize();
+}
+
+std::vector<Match> Re2Regex::find_all(const std::string& text) const {
+  std::vector<Match> result;
+  re2::StringPiece input(text);
+  re2::StringPiece piece;
+
+  const char* base = input.data();
+
+  while (RE2::FindAndConsume(&input, *regex_, &piece)) {
+    size_t start = piece.data() - base;
+    result.push_back({start, start + piece.size()});
+  }
+
+  return result;
+}
+
+} // namespace tokenizers
diff --git a/src/regex.cpp b/src/regex.cpp
new file mode 100644
index 0000000..6b26b72
--- /dev/null
+++ b/src/regex.cpp
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <pytorch/tokenizers/re2_regex.h>
+#include <pytorch/tokenizers/regex.h>
+#include <pytorch/tokenizers/std_regex.h>
+
+#include <re2/re2.h>
+#include <iostream>
+#include <memory>
+
+namespace tokenizers {
+
+/**
+ * @brief Factory function that creates a regex object using RE2 if possible.
+ *        Falls back to std::regex if RE2 rejects the pattern with
+ *        ErrorBadPerlOp.
+ */
+Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern) {
+  // Try RE2 first
+  auto re2 = std::make_unique<Re2Regex>("(" + pattern + ")");
+
+  if (re2->regex_->ok()) {
+    return static_cast<std::unique_ptr<IRegex>>(std::move(re2));
+  }
+
+  if (re2->regex_->error_code() == re2::RE2::ErrorBadPerlOp) {
+    try {
+      std::cout
+          << "RE2 is unable to support things such as negative lookaheads in "
+          << pattern << ", defaulting to std::regex.";
+      auto std_regex = std::make_unique<StdRegex>("(" + pattern + ")");
+      return static_cast<std::unique_ptr<IRegex>>(std::move(std_regex));
+    } catch (const std::regex_error& e) {
+      std::cerr << "std::regex failed: " << e.what() << std::endl;
+      return tokenizers::Error::LoadFailure;
+    }
+  } else {
+    std::cerr << "RE2 failed to compile pattern: " << pattern << "\n";
+    std::cerr << "Error: " << (re2->regex_->error()) << std::endl;
+    return tokenizers::Error::LoadFailure;
+  }
+}
+
+} // namespace tokenizers
diff --git a/src/std_regex.cpp b/src/std_regex.cpp
new file mode 100644
index 0000000..3121bac
--- /dev/null
+++ b/src/std_regex.cpp
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <pytorch/tokenizers/std_regex.h>
+#include <regex>
+
+namespace tokenizers {
+
+StdRegex::StdRegex(const std::string& pattern) : regex_(pattern) {}
+
+std::vector<Match> StdRegex::find_all(const std::string& text) const {
+  std::vector<Match> result;
+  std::sregex_iterator iter(text.begin(), text.end(), regex_);
+  std::sregex_iterator end;
+
+  for (; iter != end; ++iter) {
+    const auto& match = *iter;
+    size_t start = match.position(1);
+    result.push_back({start, start + match[1].length()});
+  }
+
+  return result;
+}
+
+} // namespace tokenizers


Reviews:
user: login: larryliu0820
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:43:29Z
user: login: larryliu0820
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:44:09Z
user: login: larryliu0820
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:44:41Z
user: login: larryliu0820
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T18:46:28Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T19:40:44Z
user: login: shoumikhin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T20:17:53Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T20:33:08Z
user: login: shoumikhin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T20:49:15Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T22:58:43Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T22:59:26Z
user: login: larryliu0820
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T16:39:46Z
user: login: shoumikhin
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T17:37:24Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T21:31:39Z
user: login: jackzhxng
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T22:28:03Z
user: login: shoumikhin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T01:08:26Z

Review comments:
diff_hunk: @@ -0,0 +1,42 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include "regex.h"
+
+// Third Party
+#include <re2/re2.h>
+
+/**
+ * @brief RE2-based implementation of IRegex.
+ */
+class Re2Regex : public IRegex {
+ public:
+  /**
+   * @brief Construct a RE2 regex with the given pattern.
+   *
+   * @param pattern The regex pattern to compile.
+   */
+  explicit Re2Regex(const std::string& pattern);
+
+  /**
+   * @brief Return all non-overlapping matches found in the input string.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const override;
path: include/pytorch/tokenizers/re2_regex.h
user: login: larryliu0820
body: This should be a `Result`? Is this changed in the top PR?
created_at: 2025-04-16T18:43:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,33 @@
+#include "pytorch/tokenizers/re2_regex.h"
path: src/re2_regex.cpp 
user: login: larryliu0820
body: Use `<>`
created_at: 2025-04-16T18:44:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,28 @@
+#pragma once
+
+#include <memory>
+#include <regex>
+#include <string>
+#include "regex.h"
path: include/pytorch/tokenizers/std_regex.h
user: login: larryliu0820
body: Hmm should it be <pytorch/tokenizers/regex.h>? It needs to work internally
created_at: 2025-04-16T18:44:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,37 @@
+#include "pytorch/tokenizers/regex.h"
+#include "pytorch/tokenizers/re2_regex.h"
+#include "pytorch/tokenizers/std_regex.h"
path: src/regex.cpp
user: login: larryliu0820
body: Use `<>`
created_at: 2025-04-16T18:46:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,42 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include "regex.h"
+
+// Third Party
+#include <re2/re2.h>
+
+/**
+ * @brief RE2-based implementation of IRegex.
+ */
+class Re2Regex : public IRegex {
+ public:
+  /**
+   * @brief Construct a RE2 regex with the given pattern.
+   *
+   * @param pattern The regex pattern to compile.
+   */
+  explicit Re2Regex(const std::string& pattern);
+
+  /**
+   * @brief Return all non-overlapping matches found in the input string.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const override;
path: include/pytorch/tokenizers/re2_regex.h
user: login: jackzhxng
body: Don't expect this to error
created_at: 2025-04-16T19:40:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
+};
+
+/**
+ * @brief Creates a regex instance. Tries RE2 first, falls back to std::regex.
+ *
+ * @param pattern The regex pattern to compile.
+ * @return A unique pointer to an IRegex-compatible object.
+ */
+Result<std::unique_ptr<IRegex>> createRegex(const std::string& pattern);
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: Do we need to have a constructor in this abstract class?
created_at: 2025-04-16T20:10:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: Ideally regex should either fail at constructor, or stay valid forever once created. Do we need a standalone check like this?
created_at: 2025-04-16T20:12:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: nit: do we follow ET style conventions or this repo has diverged? Eg. in ET there's no camel case, but underscores to separate words in identifiers.
created_at: 2025-04-16T20:14:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: nit: do we add the I prefix to abstract classes in ET?
created_at: 2025-04-16T20:14:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: Can we just return a pair of indices instead of a custom struct?
created_at: 2025-04-16T20:15:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: May like something like this:

```cpp
#pragma once

#include <string>
#include <vector>

class Regex {
public:
  virtual ~Regex() = default;

  // The only method subclasses have to implement.
  virtual std::pair<size_t, size_t> match(const std::string& text, size_t start) const = 0;

  // Convenience overload to match from the beginning.
  std::pair<size_t, size_t> match(const std::string& text) const {
    return match(text, 0);
  }

  // General implementation to match all.
  std::vector<std::pair<size_t, size_t>> match_all(const std::string& text, size_t start = 0) const {
    std::vector<std::pair<size_t, size_t>> matches;
    for (size_t length = 0;; start += length) {
      std::tie(start, length) = match(text, start);
      if (length == 0) {
        break;
      }
      matches.emplace_back(start, length);
    }
    return matches;
  }
};
```
created_at: 2025-04-16T20:17:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: jackzhxng
body: To see if it failed during construction so it can then fallback on another regex implementation
created_at: 2025-04-16T20:33:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: What if we make this class construction-agnostic and let the subclasses deal with errors during construction? Eg. different regex impls may approach it differently: throw exceptions from constructor, return an error, or have a dedicated "compile" method, etc. And the users of this interface shouldn't care how exactly a regex has been constructed, they just want to get a match using an apriori valid interface. Normally, it's up to a "regex factory" or some higher level concept to deal with failures at construction (eg. pick a proper impl as a fallback according to some logic) and then provide a valid pointer to this interface. Like a platform-specific impl could leverage Apple's `NSRegularExpression`, but then fallback to `std::regex` or something if the former fails. But whoever gets a pointer to this interface would never need to reason if it's valid or not, but just use it.
created_at: 2025-04-16T20:49:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: jackzhxng
body: Yeah that makes sense, I can make this protected? I just added this to use in the factory function `create_regex`
created_at: 2025-04-16T22:58:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
+};
+
+/**
+ * @brief Creates a regex instance. Tries RE2 first, falls back to std::regex.
+ *
+ * @param pattern The regex pattern to compile.
+ * @return A unique pointer to an IRegex-compatible object.
+ */
+Result<std::unique_ptr<IRegex>> createRegex(const std::string& pattern);
path: include/pytorch/tokenizers/regex.h
user: login: jackzhxng
body: This is the factory function mentioned below
created_at: 2025-04-16T22:59:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
path: include/pytorch/tokenizers/regex.h
user: login: larryliu0820
body: There's IRunner
created_at: 2025-04-17T16:39:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: shoumikhin
body: Let's consider how it's gonna be used. I guess `IRegex` will be injected as a dep to some higher level concept like some concrete impl of `ITokenizer`?

Some pseudocode:

```cpp
// MyModelTokenizer is such that requires regex, so it'll use IRegex.
// Other tokenizers may not need regex at all, btw.
class MyModelTokenizer : public ITokenizer {
  MyModelTokenizer(const std::string& filepath, std::unique_ptr<IRegex> regex)
  : regex_(std::move(regex)) {
    // open file, initialize everything else
  }

  std::vector<size_t> encode(const std::string& text) override {
    // Use regex_ to parse the text, etc.
    // It's guaranteed the injected IRegex is ready to use and there's no need to validate it again
    // Tokenizer doesn't need use anything of IRegex beyond matching text
    auto tokens = regex_->match_all(text);
  }
};

// MyModelRunner is such that required text tokenization, so it'll use ITokenizer.
// Other runners may not need tokenization at all, btw, or expect some other components do tokenization and provide them with already ready tokens.
class MyModelRunner : public IRunner {
  MyModelRunner(std::unique_ptr<ITokenizer> tokenizer) 
  : tokenizer_(std::move(tokenizer)) {...}

  std::vector<size_t> preprocess(const std::string& text) override {
    return tokenizer_->encode(text);
  }

  size_t generate(const std::vector<size_t>& tokens) override { ... }
};
```

So we can inject various regex implementations into tokenizers that do need regexp, and the latter never have to deal with regex creation or check its validity.
created_at: 2025-04-17T17:37:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
path: include/pytorch/tokenizers/regex.h
user: login: jackzhxng
body: I feel like we should just leave this API as is. We can get into a more granular API design later if necessary but the main point of all of this was to simply just provide a pcre2 fallback if re2 didn't work. I don't really expect people to be adding different regex implementations to be honest so don't want to overengineer too much. Another reason is I'd like to not mess with the current re2 code which uses `FindAndConsume`, which is stateful and would not fit into the proposed `match` API
created_at: 2025-04-17T21:31:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <pytorch/tokenizers/result.h>
+
+namespace tokenizers {
+
+struct Match {
+  std::string text;
+  size_t position;
+};
+
+/**
+ * @brief Abstract interface for regex wrappers.
+ */
+class IRegex {
+ public:
+  virtual ~IRegex() = default;
+
+  /**
+   * @brief Find all non-overlapping matches in the input string.
+   *
+   * @param text The input string to search.
+   * @return A vector of strings containing all matched substrings.
+   */
+  virtual std::vector<Match> findAll(const std::string& text) const = 0;
+
+  /**
+   * @brief Check if the regex pattern was compiled successfully.
+   *
+   * @return true if the pattern is valid and ready to use, false otherwise.
+   */
+  virtual bool ok() const = 0;
path: include/pytorch/tokenizers/regex.h
user: login: jackzhxng
body: I ended up removing it
created_at: 2025-04-17T22:28:03Z
author_association: CONTRIBUTOR

PR comments:
user: login: facebook-github-bot
created_at: 2025-04-15T23:13:20Z
body: @jackzhxng has imported this pull request. If you are a Meta employee, you can view this diff [on Phabricator](https://www.internalfb.com/diff/D73071817).
author_association: CONTRIBUTOR
user: login: larryliu0820
created_at: 2025-04-16T18:47:26Z
body: You will need to add license header to all source files, similar to https://github.com/pytorch-labs/tokenizers/blob/main/include/pytorch/tokenizers/tiktoken.h
author_association: CONTRIBUTOR
user: login: facebook-github-bot
created_at: 2025-04-17T23:11:47Z
body: @jackzhxng has imported this pull request. If you are a Meta employee, you can view this diff [on Phabricator](https://www.internalfb.com/diff/D73071817).
author_association: CONTRIBUTOR