Repository: redpanda-data/redpanda
Pull request #25811: [CORE-9797] schema_registry: Improve diagnostics when references are missing
Author: login: BenPope
Labels: name: area/redpanda
Created: 2025-04-15T11:07:11Z | Merged: 2025-04-22T15:47:22Z

Improve diagnostics when references are missing during:
* `POST /subjects/{subject}/`
* `POST /subjects/{subject}/versions`
* `POST /compatibility/subjects/{subject}/versions`

## Backports Required

<!-- Checking at least one of the checkboxes is REQUIRED if this PR is not a backport. -->

- [ ] none - not a bug fix
- [ ] none - this is a backport
- [ ] none - issue does not exist in previous branches
- [ ] none - papercut/not impactful enough to backport
- [x] v25.1.x
- [x] v24.3.x
- [ ] v24.2.x
- [ ] v24.1.x

## Release Notes

### Improvements

* Schema Registry: Improve diagnostics when POST a schema with missing references




Diff:
diff --git a/src/v/pandaproxy/reply.h b/src/v/pandaproxy/reply.h
index 590136f2cf163..b3218cc8faa4e 100644
--- a/src/v/pandaproxy/reply.h
+++ b/src/v/pandaproxy/reply.h
@@ -161,7 +161,7 @@ inline auto exception_reply(ss::logger& log, std::exception_ptr e) {
         auto& content = eb.get_json_body();
         vlog(
           log.error,
-          "exception_reply: {:?}, exception: {}",
+          "exception_reply: {:?}, exception: {:?}",
           content ? json::rjson_serialize_str(*content) : ss::sstring{},
           std::current_exception());
         return eb;
diff --git a/src/v/pandaproxy/schema_registry/avro.cc b/src/v/pandaproxy/schema_registry/avro.cc
index 8d3b569f84740..d4bff4b4ce26a 100644
--- a/src/v/pandaproxy/schema_registry/avro.cc
+++ b/src/v/pandaproxy/schema_registry/avro.cc
@@ -560,11 +560,19 @@ ss::future<collected_schema> collect_schema(
   ss::sstring name,
   subject_schema schema) {
     for (const auto& ref : schema.def().refs()) {
-        if (!collected.contains(ref.name)) {
-            auto ss = co_await store.get_subject_schema(
-              ref.sub, ref.version, include_deleted::yes);
-            collected = co_await collect_schema(
-              store, std::move(collected), ref.name, std::move(ss.schema));
+        if (name != ref.name && !collected.contains(ref.name)) {
+            try {
+                auto ss = co_await store.get_subject_schema(
+                  ref.sub, ref.version, include_deleted::yes);
+                collected = co_await collect_schema(
+                  store, std::move(collected), ref.name, std::move(ss.schema));
+            } catch (const exception& e) {
+                if (failed_subject_schema_lookup(e.code())) {
+                    throw as_exception(
+                      no_reference_found_for(schema, ref.sub, ref.version));
+                }
+                throw;
+            }
         }
     }
     collected.insert(std::move(name), std::move(schema).def());
@@ -636,6 +644,22 @@ sanitize_avro_schema_definition(schema_definition def) {
       def.refs()};
 }
 
+ss::future<subject_schema> make_canonical_avro_schema(
+  schema_getter&, subject_schema unparsed_schema, normalize norm) {
+    auto [sub, unparsed] = std::move(unparsed_schema).destructure();
+    auto [def, type, refs] = std::move(unparsed).destructure();
+    if (norm) {
+        std::sort(refs.begin(), refs.end());
+        refs.erase(std::unique(refs.begin(), refs.end()), refs.end());
+    }
+    schema_definition schema{std::move(def), type, std::move(refs)};
+    // TODO: Check references
+    // co_await collect_schema(store, {}, sub, {sub, schema.share()});
+    co_return subject_schema{
+      std::move(sub),
+      sanitize_avro_schema_definition(std::move(schema)).value()};
+}
+
 compatibility_result check_compatible(
   const avro_schema_definition& reader,
   const avro_schema_definition& writer,
diff --git a/src/v/pandaproxy/schema_registry/avro.h b/src/v/pandaproxy/schema_registry/avro.h
index f66f703d10a0d..d706231a7ff37 100644
--- a/src/v/pandaproxy/schema_registry/avro.h
+++ b/src/v/pandaproxy/schema_registry/avro.h
@@ -23,6 +23,9 @@ make_avro_schema_definition(schema_getter& store, subject_schema schema);
 result<schema_definition>
 sanitize_avro_schema_definition(schema_definition def);
 
+ss::future<subject_schema> make_canonical_avro_schema(
+  schema_getter& store, subject_schema schema, normalize norm = normalize::no);
+
 compatibility_result check_compatible(
   const avro_schema_definition& reader,
   const avro_schema_definition& writer,
diff --git a/src/v/pandaproxy/schema_registry/error.cc b/src/v/pandaproxy/schema_registry/error.cc
index 7f00685feb44a..133e81f6a41b8 100644
--- a/src/v/pandaproxy/schema_registry/error.cc
+++ b/src/v/pandaproxy/schema_registry/error.cc
@@ -11,6 +11,7 @@
 
 #include "error.h"
 
+#include "bytes/iobuf_parser.h"
 #include "pandaproxy/error.h"
 #include "pandaproxy/schema_registry/error.h"
 #include "pandaproxy/schema_registry/errors.h"
@@ -34,6 +35,8 @@ struct error_category final : std::error_category {
             return "Invalid schema";
         case error_code::schema_empty:
             return "Empty schema";
+        case error_code::schema_missing_reference:
+            return "Schema references a schema that doesn't exist";
         case error_code::schema_incompatible:
             return "Schema being registered is incompatible with an earlier "
                    "schema for subject";
@@ -112,6 +115,7 @@ struct error_category final : std::error_category {
         case error_code::schema_invalid:
             return reply_error_code::unprocessable_entity;
         case error_code::schema_empty:
+        case error_code::schema_missing_reference:
             return reply_error_code::schema_empty; // 42201
         case error_code::schema_version_invalid:
             return reply_error_code::schema_version_invalid; // 42202
@@ -153,8 +157,9 @@ error_info no_reference_found_for(
                     | std::views::transform([](const auto& ref) {
                           return fmt::format("{{{:e}}}", ref);
                       });
+    iobuf_const_parser parser{schema.def().raw()};
     return {
-      error_code::schema_empty,
+      error_code::schema_missing_reference,
       fmt::format(
         "Invalid schema "
         "{{subject={},version=0,id=-1,schemaType={},references=[{}],metadata="
@@ -163,7 +168,7 @@ error_info no_reference_found_for(
         schema.sub()(),
         to_string_view(schema.def().type()),
         fmt::join(fmt_refs, ", "),
-        schema.def().raw()(),
+        parser.read_string(parser.bytes_left()),
         fmt::join(fmt_refs, ", "),
         to_string_view(schema.type()),
         sub(),
diff --git a/src/v/pandaproxy/schema_registry/error.h b/src/v/pandaproxy/schema_registry/error.h
index 37276284bdd82..30b88eed6a1b2 100644
--- a/src/v/pandaproxy/schema_registry/error.h
+++ b/src/v/pandaproxy/schema_registry/error.h
@@ -20,6 +20,7 @@ enum class error_code {
     schema_id_not_found = 1,
     schema_invalid,
     schema_empty,
+    schema_missing_reference,
     schema_incompatible,
     schema_version_invalid,
     subject_not_found,
diff --git a/src/v/pandaproxy/schema_registry/errors.h b/src/v/pandaproxy/schema_registry/errors.h
index 50fb9f9c4496c..9d535bc1985c6 100644
--- a/src/v/pandaproxy/schema_registry/errors.h
+++ b/src/v/pandaproxy/schema_registry/errors.h
@@ -201,4 +201,9 @@ inline error_info versions_exhausted(const subject& sub) {
       fmt::format("Versions exhausted for subject {}", sub())};
 }
 
+inline bool failed_subject_schema_lookup(std::error_code ec) {
+    return ec == error_code::subject_not_found
+           || ec == error_code::subject_version_not_found;
+}
+
 } // namespace pandaproxy::schema_registry
diff --git a/src/v/pandaproxy/schema_registry/handlers.cc b/src/v/pandaproxy/schema_registry/handlers.cc
index 559d942b45452..a0cc7b60cc7cb 100644
--- a/src/v/pandaproxy/schema_registry/handlers.cc
+++ b/src/v/pandaproxy/schema_registry/handlers.cc
@@ -716,7 +716,9 @@ compatibility_subject_version(server::request_t rq, server::reply_t rp) {
     } catch (exception& e) {
         constexpr auto reportable = [](std::error_code ec) {
             constexpr std::array errors{
-              error_code::schema_invalid, error_code::schema_empty};
+              error_code::schema_invalid,
+              error_code::schema_empty,
+              error_code::schema_missing_reference};
             return absl::c_any_of(
               errors, [ec](error_code e) { return ec == e; });
         };
diff --git a/src/v/pandaproxy/schema_registry/json.cc b/src/v/pandaproxy/schema_registry/json.cc
index 4a6f9b838d87f..b5e9d3487e457 100644
--- a/src/v/pandaproxy/schema_registry/json.cc
+++ b/src/v/pandaproxy/schema_registry/json.cc
@@ -233,9 +233,7 @@ ss::future<> check_references(sharded_store& store, subject_schema schema) {
     for (const auto& ref : schema.def().refs()) {
         co_await store.get_id(ref.sub, ref.version)
           .handle_exception_type([&](const exception& e) -> schema_id {
-              if (
-                e.code() == error_code::subject_not_found
-                || e.code() == error_code::schema_id_not_found) {
+              if (failed_subject_schema_lookup(e.code())) {
                   throw as_exception(
                     no_reference_found_for(schema, ref.sub, ref.version));
               }
diff --git a/src/v/pandaproxy/schema_registry/protobuf.cc b/src/v/pandaproxy/schema_registry/protobuf.cc
index 9f69c59a77cb3..302da5a4adb3f 100644
--- a/src/v/pandaproxy/schema_registry/protobuf.cc
+++ b/src/v/pandaproxy/schema_registry/protobuf.cc
@@ -437,13 +437,21 @@ ss::future<pb::FileDescriptorProto> build_file_with_refs(
         if (dp.FindFileByName(ref.name)) {
             continue;
         }
-        auto dep = co_await store.get_subject_schema(
-          ref.sub, ref.version, include_deleted::yes);
-        co_await build_file_with_refs(
-          dp,
-          store,
-          subject_schema{subject{ref.name}, std::move(dep.schema).def()},
-          normalize::no);
+        try {
+            auto dep = co_await store.get_subject_schema(
+              ref.sub, ref.version, include_deleted::yes);
+            co_await build_file_with_refs(
+              dp,
+              store,
+              subject_schema{subject{ref.name}, std::move(dep.schema).def()},
+              normalize::no);
+        } catch (const exception& e) {
+            if (failed_subject_schema_lookup(e.code())) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
+            throw;
+        }
     }
 
     parser p;
@@ -467,8 +475,17 @@ ss::future<pb::FileDescriptorProto> import_schema(
         co_return co_await build_file_with_refs(
           dp, store, schema.share(), norm);
     } catch (const exception& e) {
+        // Rethrow if the schema is missing references
+        if (e.code() == error_code::schema_missing_reference) {
+            throw;
+        }
+        // Otherwise log the error details and throw an appropriate error for
+        // the response
         vlog(
-          srlog.warn, "Failed to decode schema {}: {}", schema.sub(), e.what());
+          srlog.warn,
+          "Failed to decode schema {}: {:?}",
+          schema.sub(),
+          e.what());
         throw as_exception(invalid_schema(schema));
     }
 }
diff --git a/src/v/pandaproxy/schema_registry/sharded_store.cc b/src/v/pandaproxy/schema_registry/sharded_store.cc
index 27b748d7ccf84..1867465770ed9 100644
--- a/src/v/pandaproxy/schema_registry/sharded_store.cc
+++ b/src/v/pandaproxy/schema_registry/sharded_store.cc
@@ -89,12 +89,9 @@ ss::future<subject_schema> sharded_store::make_canonical_schema(
                  config::shard_local_cfg().schema_registry_always_normalize()};
     }
     switch (schema.type()) {
-    case schema_type::avro: {
-        auto [sub, unparsed] = std::move(schema).destructure();
-        co_return subject_schema{
-          std::move(sub),
-          sanitize_avro_schema_definition(std::move(unparsed)).value()};
-    }
+    case schema_type::avro:
+        co_return co_await make_canonical_avro_schema(
+          *this, std::move(schema), norm);
     case schema_type::protobuf:
         co_return co_await make_canonical_protobuf_schema(
           *this, std::move(schema), norm);
@@ -334,9 +331,7 @@ sharded_store::has_schema(subject_schema schema, include_deleted inc_del) {
                 break;
             }
         } catch (const exception& e) {
-            if (
-              e.code() == error_code::subject_not_found
-              || e.code() == error_code::subject_version_not_found) {
+            if (failed_subject_schema_lookup(e.code())) {
             } else if (
               // Stored schemas might be invalid if imported improperly
               e.code() == error_code::schema_invalid) {
diff --git a/src/v/pandaproxy/schema_registry/storage.h b/src/v/pandaproxy/schema_registry/storage.h
index 9571c2951635c..31e376f4a18de 100644
--- a/src/v/pandaproxy/schema_registry/storage.h
+++ b/src/v/pandaproxy/schema_registry/storage.h
@@ -22,6 +22,7 @@
 #include "pandaproxy/json/rjson_util.h"
 #include "pandaproxy/logger.h"
 #include "pandaproxy/schema_registry/error.h"
+#include "pandaproxy/schema_registry/errors.h"
 #include "pandaproxy/schema_registry/exceptions.h"
 #include "pandaproxy/schema_registry/seq_writer.h"
 #include "pandaproxy/schema_registry/sharded_store.h"
@@ -1479,9 +1480,7 @@ struct consume_to_store {
                     // tombstone all the records referring to a particular
                     // version, we will see more than one get applied, and
                     // after the first one, the rest will not find it.
-                    if (
-                      e.code() == error_code::subject_not_found
-                      || e.code() == error_code::subject_version_not_found) {
+                    if (failed_subject_schema_lookup(e.code())) {
                         vlog(
                           srlog.debug,
                           "Ignoring tombstone at offset={}, subject or version "
diff --git a/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc b/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc
index 01bd5fe98c977..93fa60d36f583 100644
--- a/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc
+++ b/src/v/pandaproxy/schema_registry/test/compatibility_protobuf.cc
@@ -135,7 +135,9 @@ SEASTAR_THREAD_TEST_CASE(test_protobuf_imported_failure) {
       pps::make_protobuf_schema_definition(store.store, schema1.share()).get(),
       pps::exception,
       [](const pps::exception& ex) {
-          return ex.code() == pps::error_code::schema_invalid;
+          return ex.code() == pps::error_code::schema_missing_reference
+                 && std::string_view(ex.message())
+                      .contains("No schema reference found for subject");
       });
 }
 
diff --git a/src/v/pandaproxy/schema_registry/test/test_json_schema.cc b/src/v/pandaproxy/schema_registry/test/test_json_schema.cc
index 3ec1c3728e471..a816fb4f94a40 100644
--- a/src/v/pandaproxy/schema_registry/test/test_json_schema.cc
+++ b/src/v/pandaproxy/schema_registry/test/test_json_schema.cc
@@ -360,7 +360,7 @@ const std::array test_reference_cases = {
   test_references_data{
     {{{referenced.share(), {}},
       {referencer_wrong_sub.share(),
-       {pps::error_code::schema_empty,
+       {pps::error_code::schema_missing_reference,
         R"(Invalid schema {subject=referencer,version=0,id=-1,schemaType=JSON,references=[{name='example.com/referenced.json', subject='wrong_sub', version=1}],metadata=null,ruleSet=null,schema={
   "description": "A schema that references the base schema",
   "type": "object",
diff --git a/src/v/pandaproxy/server.cc b/src/v/pandaproxy/server.cc
index 98463a4a941a1..9898d9a2c95a8 100644
--- a/src/v/pandaproxy/server.cc
+++ b/src/v/pandaproxy/server.cc
@@ -148,7 +148,7 @@ struct handler_adaptor : ss::httpd::handler_base {
             auto ex = std::current_exception();
             vlog(
               _log.warn,
-              "Request: {} {} failed: {}",
+              "Request: {} {} failed: {:?}",
               method,
               url,
               std::current_exception());
diff --git a/tests/rptest/tests/schema_registry_test.py b/tests/rptest/tests/schema_registry_test.py
index baa6b0c6987e2..abf3939659be9 100644
--- a/tests/rptest/tests/schema_registry_test.py
+++ b/tests/rptest/tests/schema_registry_test.py
@@ -666,6 +666,22 @@ def get_subject_name(sns: str, topic: str, field: MessageField,
         "type": "AVRO",
     }
 }
+
+not_dependent_schemas = {
+    "proto": {
+        "schema": schema_proto_def,
+        "type": "PROTOBUF",
+    },
+    "json": {
+        "schema": json_number_schema_def,
+        "type": "JSON",
+    },
+    "avro": {
+        "schema": schema_avro_def,
+        "type": "AVRO",
+    },
+}
+
 dependent_schemas = {
     "proto": {
         "subject":
@@ -3451,6 +3467,50 @@ def test_subjects_subject(schema, expected_version=None, norm=False):
         test_subjects_subject(normalized, expected_version=2)
         test_subjects_subject(normalized, expected_version=2, norm=True)
 
+    @cluster(num_nodes=3)
+    @matrix(stype=["json", "proto"])
+    def test_missing_references(self, stype):
+        """
+        Missing references should return an error message with details of the missing reference
+        """
+
+        self._set_config(data=json.dumps({"compatibility": "NONE"}))
+
+        base_schema = not_dependent_schemas[stype]
+        schema = dependent_schemas[stype]
+        subject = schema["subject"]
+        ref_subject = schema["references"][0]["subject"]
+        ref_version = schema["references"][0]["version"]
+
+        result_raw = self._post_subjects_subject_versions(
+            subject=subject,
+            data=json.dumps({
+                "schema": base_schema["schema"],
+                "schemaType": base_schema["type"],
+            }))
+        assert result_raw.status_code == requests.codes.ok, \
+                f"Expected {requests.codes.ok} but got {result_raw.status_code} during test setup. "\
+                f"Request content: {result_raw.content}. Processing {stype}."
+
+        for endpoint in [
+                self._post_subjects_subject_versions,
+                self._post_subjects_subject
+        ]:
+            result_raw = endpoint(subject=subject,
+                                  data=json.dumps({
+                                      "schema":
+                                      schema["schema"],
+                                      "schemaType":
+                                      schema["type"],
+                                      "references":
+                                      schema["references"],
+                                  }))
+            assert result_raw.status_code == requests.codes.unprocessable_entity, \
+                    f"Expected {requests.codes.unprocessable_entity} but got {result_raw.status_code}. "\
+                    f"Request content: {result_raw.content}. Processing {type}."
+            assert f"No schema reference found for subject \"{ref_subject}\" and version {ref_version}" in result_raw.json()["message"], \
+                f"Expected result to contain missing references, but got {result_raw.json()}."
+
     @cluster(num_nodes=3)
     def test_soft_deleted_references(self):
         """


Reviews:
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T12:20:30Z
user: login: pgellert
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T17:05:13Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:17:39Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:18:17Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T17:19:02Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-16T16:25:12Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-16T16:25:29Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-16T16:26:06Z
user: login: pgellert
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-04-16T21:11:00Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T07:00:24Z
user: login: IoannisRP
body: lgtm
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T12:39:15Z
user: login: BenPope
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T13:11:36Z
user: login: IoannisRP
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T13:26:12Z
user: login: IoannisRP
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-17T13:26:42Z

Review comments:
diff_hunk: @@ -463,14 +473,7 @@ ss::future<pb::FileDescriptorProto> import_schema(
   schema_getter& store,
   subject_schema schema,
   normalize norm) {
-    try {
-        co_return co_await build_file_with_refs(
-          dp, store, schema.share(), norm);
-    } catch (const exception& e) {
-        vlog(
-          plog.warn, "Failed to decode schema {}: {}", schema.sub(), e.what());
-        throw as_exception(invalid_schema(schema));
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: BenPope
body: This is actually important to prevent a lot of noise in the response message
```suggestion
        // Rethrow if the schema is missing references
        if (e.code() == error_code::schema_empty) {
            throw;
        }
        // Otherwise log the error details and throw an appropriate error for
        // the response
        vlog(
          plog.warn, "Failed to decode schema {}: {}", schema.sub(), e.what());
        throw as_exception(invalid_schema(schema));
created_at: 2025-04-15T12:20:30Z
author_association: MEMBER
diff_hunk: @@ -427,13 +427,15 @@ post_subject(server::request_t rq, server::reply_t rp) {
     co_await rq.service().schema_store().get_versions(sub, inc_del);
 
     subject_schema schema;
+    size_t len = 0;
     try {
         auto unparsed = co_await ppj::rjson_parse(
           std::move(rq.req), post_subject_versions_request_handler<>{sub});
+        len = unparsed.def.def().raw()().size_bytes();
         schema = co_await rq.service().schema_store().make_canonical_schema(
           std::move(unparsed.def), norm);
     } catch (const exception& e) {
-        if (e.code() == error_code::schema_empty) {
+        if (e.code() == error_code::schema_empty && len == 0) {
path: src/v/pandaproxy/schema_registry/handlers.cc
user: login: pgellert
body: The ducktape test makes sense but I'm not following this part. Why is this part necessary?
created_at: 2025-04-15T12:54:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -467,6 +477,12 @@ ss::future<pb::FileDescriptorProto> import_schema(
         co_return co_await build_file_with_refs(
           dp, store, schema.share(), norm);
     } catch (const exception& e) {
+        // Rethrow if the schema is missing references
+        if (e.code() == error_code::schema_empty) {
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: pgellert
body: Shouldn't this be `no_reference_found_for` based on the comment? But also, could you elaborate while this special casing is needed? Wouldn't we want to log either way and wrap the exception?
created_at: 2025-04-15T13:00:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -560,11 +560,16 @@ ss::future<collected_schema> collect_schema(
   ss::sstring name,
   subject_schema schema) {
     for (const auto& ref : schema.def().refs()) {
-        if (!collected.contains(ref.name)) {
-            auto ss = co_await store.get_subject_schema(
-              ref.sub, ref.version, include_deleted::yes);
-            collected = co_await collect_schema(
-              store, std::move(collected), ref.name, std::move(ss.schema));
+        if (name != ref.name && !collected.contains(ref.name)) {
+            try {
+                auto ss = co_await store.get_subject_schema(
+                  ref.sub, ref.version, include_deleted::yes);
+                collected = co_await collect_schema(
+                  store, std::move(collected), ref.name, std::move(ss.schema));
+            } catch (const exception& e) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/avro.cc
user: login: pgellert
body: Should we check the error code here as well?
```
if (
              e.code() == error_code::subject_not_found
              || e.code() == error_code::subject_version_not_found) {
```
created_at: 2025-04-15T15:05:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -636,6 +641,16 @@ sanitize_avro_schema_definition(schema_definition def) {
       def.refs()};
 }
 
+ss::future<subject_schema> make_canonical_avro_schema(
+  schema_getter& store, subject_schema unparsed_schema, normalize) {
+    auto [sub, schema] = std::move(unparsed_schema).destructure();
+    // Check references
+    co_await collect_schema(store, {}, sub, {sub, schema.share()});
path: src/v/pandaproxy/schema_registry/avro.cc
user: login: pgellert
body: This seems to be new behaviour. Would it make sense to extract this to a separate PR and backport everything else but this?
created_at: 2025-04-15T15:15:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -427,13 +427,15 @@ post_subject(server::request_t rq, server::reply_t rp) {
     co_await rq.service().schema_store().get_versions(sub, inc_del);
 
     subject_schema schema;
+    size_t len = 0;
     try {
         auto unparsed = co_await ppj::rjson_parse(
           std::move(rq.req), post_subject_versions_request_handler<>{sub});
+        len = unparsed.def.def().raw()().size_bytes();
         schema = co_await rq.service().schema_store().make_canonical_schema(
           std::move(unparsed.def), norm);
     } catch (const exception& e) {
-        if (e.code() == error_code::schema_empty) {
+        if (e.code() == error_code::schema_empty && len == 0) {
path: src/v/pandaproxy/schema_registry/handlers.cc
user: login: BenPope
body: >  Why is this part necessary?
The `error_code` for an empty schema (which can come from avro) is shared with `no_reference_found_for`, so this change allows both errors to be distinguished.

I'll rename it to `schema_empty_or_missing_reference`, and add some helpers.
created_at: 2025-04-15T17:17:38Z
author_association: MEMBER
diff_hunk: @@ -467,6 +477,12 @@ ss::future<pb::FileDescriptorProto> import_schema(
         co_return co_await build_file_with_refs(
           dp, store, schema.share(), norm);
     } catch (const exception& e) {
+        // Rethrow if the schema is missing references
+        if (e.code() == error_code::schema_empty) {
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: BenPope
body: > Wouldn't we want to log either way and wrap the exception?

The error response has sufficient detail, and is logged elsewhere.
created_at: 2025-04-15T17:18:17Z
author_association: MEMBER
diff_hunk: @@ -560,11 +560,16 @@ ss::future<collected_schema> collect_schema(
   ss::sstring name,
   subject_schema schema) {
     for (const auto& ref : schema.def().refs()) {
-        if (!collected.contains(ref.name)) {
-            auto ss = co_await store.get_subject_schema(
-              ref.sub, ref.version, include_deleted::yes);
-            collected = co_await collect_schema(
-              store, std::move(collected), ref.name, std::move(ss.schema));
+        if (name != ref.name && !collected.contains(ref.name)) {
+            try {
+                auto ss = co_await store.get_subject_schema(
+                  ref.sub, ref.version, include_deleted::yes);
+                collected = co_await collect_schema(
+                  store, std::move(collected), ref.name, std::move(ss.schema));
+            } catch (const exception& e) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/avro.cc
user: login: BenPope
body: There are a few of these, I'll add a helper.
created_at: 2025-04-15T17:19:02Z
author_association: MEMBER
diff_hunk: @@ -427,13 +427,15 @@ post_subject(server::request_t rq, server::reply_t rp) {
     co_await rq.service().schema_store().get_versions(sub, inc_del);
 
     subject_schema schema;
+    size_t len = 0;
     try {
         auto unparsed = co_await ppj::rjson_parse(
           std::move(rq.req), post_subject_versions_request_handler<>{sub});
+        len = unparsed.def.def().raw()().size_bytes();
         schema = co_await rq.service().schema_store().make_canonical_schema(
           std::move(unparsed.def), norm);
     } catch (const exception& e) {
-        if (e.code() == error_code::schema_empty) {
+        if (e.code() == error_code::schema_empty && len == 0) {
path: src/v/pandaproxy/schema_registry/handlers.cc
user: login: BenPope
body: I added a new `error_code`
created_at: 2025-04-16T16:25:12Z
author_association: MEMBER
diff_hunk: @@ -560,11 +560,16 @@ ss::future<collected_schema> collect_schema(
   ss::sstring name,
   subject_schema schema) {
     for (const auto& ref : schema.def().refs()) {
-        if (!collected.contains(ref.name)) {
-            auto ss = co_await store.get_subject_schema(
-              ref.sub, ref.version, include_deleted::yes);
-            collected = co_await collect_schema(
-              store, std::move(collected), ref.name, std::move(ss.schema));
+        if (name != ref.name && !collected.contains(ref.name)) {
+            try {
+                auto ss = co_await store.get_subject_schema(
+                  ref.sub, ref.version, include_deleted::yes);
+                collected = co_await collect_schema(
+                  store, std::move(collected), ref.name, std::move(ss.schema));
+            } catch (const exception& e) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/avro.cc
user: login: BenPope
body: Done
created_at: 2025-04-16T16:25:29Z
author_association: MEMBER
diff_hunk: @@ -636,6 +641,16 @@ sanitize_avro_schema_definition(schema_definition def) {
       def.refs()};
 }
 
+ss::future<subject_schema> make_canonical_avro_schema(
+  schema_getter& store, subject_schema unparsed_schema, normalize) {
+    auto [sub, schema] = std::move(unparsed_schema).destructure();
+    // Check references
+    co_await collect_schema(store, {}, sub, {sub, schema.share()});
path: src/v/pandaproxy/schema_registry/avro.cc
user: login: BenPope
body: > This seems to be new behaviour. Would it make sense to extract this to a separate PR and backport everything else but this?

I've made it a TODO, it improves the error message when a compatibility check is required.
created_at: 2025-04-16T16:26:05Z
author_association: MEMBER
diff_hunk: @@ -112,6 +114,7 @@ struct error_category final : std::error_category {
         case error_code::schema_invalid:
             return reply_error_code::unprocessable_entity;
         case error_code::schema_empty:
+        case error_code::schema_missing_reference:
path: src/v/pandaproxy/schema_registry/error.cc
user: login: pgellert
body: Nice, looks great
created_at: 2025-04-16T20:52:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -897,7 +897,7 @@ ss::future<compatibility_result> sharded_store::do_is_compatible(
             version_messages.emplace_back(
               fmt::format("{{oldSchemaVersion: {}}}", old_schema.version));
             version_messages.emplace_back(
-              fmt::format("{{oldSchema: '{}'}}", to_string(old_valid.raw())));
+              fmt::format("{{oldSchema: '{}'}}", old_valid.raw()));
path: src/v/pandaproxy/schema_registry/sharded_store.cc
user: login: pgellert
body: The commit message says:
> Change operator<< so that newlines are escaped.

This corresponds to the last commit, right? I think for that, I don't think it matters whether you stream the output or format a string. Am I missing something here? (Mostly just checking that I'm not missing anything here + that you didn't intend to change anything more here.)
created_at: 2025-04-16T21:00:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -897,7 +897,7 @@ ss::future<compatibility_result> sharded_store::do_is_compatible(
             version_messages.emplace_back(
               fmt::format("{{oldSchemaVersion: {}}}", old_schema.version));
             version_messages.emplace_back(
-              fmt::format("{{oldSchema: '{}'}}", to_string(old_valid.raw())));
+              fmt::format("{{oldSchema: '{}'}}", old_valid.raw()));
path: src/v/pandaproxy/schema_registry/sharded_store.cc
user: login: BenPope
body: You are correct, I had previously implemented operator<< in terms of a `fmt::formatter`, that was able to escape characters, but the final commit is simpler.

I'll drop this commit and use `iobuf_const_parser::read_string` in the next, so that utf-8 validation occurs.
created_at: 2025-04-17T07:00:23Z
author_association: MEMBER
diff_hunk: @@ -437,13 +437,21 @@ ss::future<pb::FileDescriptorProto> build_file_with_refs(
         if (dp.FindFileByName(ref.name)) {
             continue;
         }
-        auto dep = co_await store.get_subject_schema(
-          ref.sub, ref.version, include_deleted::yes);
-        co_await build_file_with_refs(
-          dp,
-          store,
-          subject_schema{subject{ref.name}, std::move(dep.schema).def()},
-          normalize::no);
+        try {
+            auto dep = co_await store.get_subject_schema(
+              ref.sub, ref.version, include_deleted::yes);
+            co_await build_file_with_refs(
+              dp,
+              store,
+              subject_schema{subject{ref.name}, std::move(dep.schema).def()},
+              normalize::no);
+        } catch (const exception& e) {
+            if (failed_subject_schema_lookup(e.code())) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: IoannisRP
body: This can fail for 2 reasons.
- Schema `{sub, version}` not found
- Ref schema failing to build.
In both cases we return the same error that the schema cannot be found.

I think it would be better if we had a different error for each type of reference failure.

It would be nice to also have some recursive info on where the failure occured, in case it happened further in the dep chain, but I don't think it is worth the effort.

This affects only `proto`. `avro` and `json` don't have any processing once the schema is retrieved.
created_at: 2025-04-17T12:38:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -437,13 +437,21 @@ ss::future<pb::FileDescriptorProto> build_file_with_refs(
         if (dp.FindFileByName(ref.name)) {
             continue;
         }
-        auto dep = co_await store.get_subject_schema(
-          ref.sub, ref.version, include_deleted::yes);
-        co_await build_file_with_refs(
-          dp,
-          store,
-          subject_schema{subject{ref.name}, std::move(dep.schema).def()},
-          normalize::no);
+        try {
+            auto dep = co_await store.get_subject_schema(
+              ref.sub, ref.version, include_deleted::yes);
+            co_await build_file_with_refs(
+              dp,
+              store,
+              subject_schema{subject{ref.name}, std::move(dep.schema).def()},
+              normalize::no);
+        } catch (const exception& e) {
+            if (failed_subject_schema_lookup(e.code())) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: BenPope
body: I don't understand your reasoning.

If it fails due to a reference not being found, it will translate to a `no_reference_found_for`.
If it fails because a reference failed to build, that would be returned as the original failure.
created_at: 2025-04-17T13:11:36Z
author_association: MEMBER
diff_hunk: @@ -437,13 +437,21 @@ ss::future<pb::FileDescriptorProto> build_file_with_refs(
         if (dp.FindFileByName(ref.name)) {
             continue;
         }
-        auto dep = co_await store.get_subject_schema(
-          ref.sub, ref.version, include_deleted::yes);
-        co_await build_file_with_refs(
-          dp,
-          store,
-          subject_schema{subject{ref.name}, std::move(dep.schema).def()},
-          normalize::no);
+        try {
+            auto dep = co_await store.get_subject_schema(
+              ref.sub, ref.version, include_deleted::yes);
+            co_await build_file_with_refs(
+              dp,
+              store,
+              subject_schema{subject{ref.name}, std::move(dep.schema).def()},
+              normalize::no);
+        } catch (const exception& e) {
+            if (failed_subject_schema_lookup(e.code())) {
+                throw as_exception(
+                  no_reference_found_for(schema, ref.sub, ref.version));
+            }
path: src/v/pandaproxy/schema_registry/protobuf.cc
user: login: IoannisRP
body: You are right. I misread this.
Ignore my comment
created_at: 2025-04-17T13:26:12Z
author_association: CONTRIBUTOR

PR comments:
user: login: vbotbuildovich
created_at: 2025-04-15T16:39:39Z
body: <!--TEST RESULTS-->
<!--d162ff49-093a-459b-8f67-4c94f44e95ce-->
<!--TEST RESULTS-->
<!--026aa899-cce0-4f5d-9e97-5163d828c7c6-->
<!--TEST RESULTS-->
<!--f84f6632-c242-4879-8512-6ac29669fb20-->
<!--TEST RESULTS-->
<!--65e13223-0b7c-472c-b1e0-0850199d47d6-->
<!--title-->
## CI test results
<!--/title-->
<!--body-->
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/64548">build#64548</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.archival_test.ArchivalTest.test_all_partitions_leadership_transfer.cloud_storage_type=CloudStorageType.S3 | ducktape | https://buildkite.com/redpanda/redpanda/builds/64548#019639c5-62ca-4741-af31-de1386786218 | FLAKY | 1/2 |
| rptest.tests.cloud_storage_scrubber_test.CloudStorageScrubberTest.test_scrubber.cloud_storage_type=CloudStorageType.S3 | ducktape | https://buildkite.com/redpanda/redpanda/builds/64548#019639c2-7805-44ca-90bb-43eed9903f35 | FLAKY | 1/2 |
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_migrated_topic_data_integrity.transfer_leadership=True.params=.cancellation.dir.in.stage.preparing.use_alias.True | ducktape | https://buildkite.com/redpanda/redpanda/builds/64548#019639c2-7804-41a0-a47c-ba4a8aa74d55 | FLAKY | 1/2 |
| rptest.tests.upgrade_test.UpgradeBackToBackTest.test_upgrade_with_all_workloads.single_upgrade=False | ducktape | https://buildkite.com/redpanda/redpanda/builds/64548#019639c2-7806-44aa-bfe9-c849bc84ef58 | FLAKY | 1/3 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/64624">build#64624</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.datalake.rest_catalog_connection_test.RestCatalogConnectionTest.test_redpanda_connection_to_rest_catalog.cloud_storage_type=CloudStorageType.S3 | ducktape | https://buildkite.com/redpanda/redpanda/builds/64624#01963fd6-b540-4601-9f9f-fad9452eed07 | FLAKY | 20/21 |
| rptest.tests.node_pool_migration_test.NodePoolMigrationTest.test_migrating_redpanda_nodes_to_new_pool.balancing_mode=node_add.test_mode=TestMode.TIRED_STORAGE.cleanup_policy=compact.delete | ducktape | https://buildkite.com/redpanda/redpanda/builds/64624#01963fd6-b545-41a4-b68f-8bdc13ca1eb9 | FLAKY | 16/17 |
| rptest.tests.scaling_up_test.ScalingUpTest.test_scaling_up_with_recovered_topic | ducktape | https://buildkite.com/redpanda/redpanda/builds/64624#01963fd6-b540-4867-a22b-1dbf99109a89 | FLAKY | 15/21 |
| rptest.transactions.producers_api_test.ProducersAdminAPITest.test_producers_state_api_during_load | ducktape | https://buildkite.com/redpanda/redpanda/builds/64624#01963fd6-9d10-4fa5-ae62-9ade0e9a6272 | FLAKY | 18/21 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/64678">build#64678</a></summary>


| test_id | test_kind | job_url | test_status | passed |
|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_migrated_topic_data_integrity.transfer_leadership=False.params=.cancellation.dir.in.stage.preparing.use_alias.False | ducktape | https://buildkite.com/redpanda/redpanda/builds/64678#01964423-950f-49e2-b0bc-d591653c2c56 | FLAKY | 20/21 |
| rptest.tests.data_migrations_api_test.DataMigrationsApiTest.test_migrated_topic_data_integrity.transfer_leadership=True.params=.cancellation.dir.in.stage.preparing.use_alias.False | ducktape | https://buildkite.com/redpanda/redpanda/builds/64678#01964423-9510-4446-8507-cb7568f62548 | FLAKY | 11/21 |
| rptest.tests.datalake.throttling_test.DatalakeThrottlingTest.test_basic_throttling.cloud_storage_type=CloudStorageType.S3.catalog_type=CatalogType.NESSIE | ducktape | https://buildkite.com/redpanda/redpanda/builds/64678#01964427-8bea-441e-bde8-9c751d8a9820 | FLAKY | 17/21 |
| rptest.tests.upgrade_test.UpgradeBackToBackTest.test_upgrade_with_all_workloads.single_upgrade=False | ducktape | https://buildkite.com/redpanda/redpanda/builds/64678#01964423-9510-4446-8507-cb7568f62548 | FLAKY | 8/21 |
</details>
<details>
<summary>test results on <a href="https://buildkite.com/redpanda/redpanda/builds/64838">build#64838</a></summary>


| test_id                             | test_kind   | job_url                                                                                   | test_status   | passed   |
|:------------------------------------|:------------|:------------------------------------------------------------------------------------------|:--------------|:---------|
| gtest_raft_rpunit.gtest_raft_rpunit | unit        | https://buildkite.com/redpanda/redpanda/builds/64838#01965c7c-7e55-43ed-86bb-edea76a87757 | FLAKY         | 1/2      |
</details>
<!--/body-->
author_association: COLLABORATOR
user: login: BenPope
created_at: 2025-04-16T16:41:39Z
body: Changes in [force-push](https://github.com/redpanda-data/redpanda/compare/2e3fb38ee91f55639ff7d16381fe2dd2ae38a04a..004ece030171d2d6546dc6ed1ab851c626f898b1)
* Split `error_code::schema_empty`
* Add a helper for `failed_subject_schema_lookup`
* Add an `operator<<` for `raw_string`
* Don't force a failure for avro references
author_association: MEMBER
user: login: BenPope
created_at: 2025-04-16T16:42:19Z
body: Changees in [force-push](https://github.com/redpanda-data/redpanda/compare/3b871daa0cd5ae32e9b883215e4406cb29487558..f771174d77adfe8457cd0604a464b316aeb619d0)
* Rebase to fix logging conflicts
author_association: MEMBER
user: login: vbotbuildovich
created_at: 2025-04-16T20:15:26Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/64624-->
<!--739dc4e3-6903-4ae8-824d-818961ddb9df-->
<!--d2d1bb3c-b6d2-42ef-b125-2a3064b5dcfc-->
<!--title-->
## Retry command for [Build#64624](https://buildkite.com/redpanda/redpanda/builds/64624)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```

/ci-repeat 1
tests/rptest/transactions/producers_api_test.py::ProducersAdminAPITest.test_producers_state_api_during_load
tests/rptest/tests/datalake/rest_catalog_connection_test.py::RestCatalogConnectionTest.test_redpanda_connection_to_rest_catalog@{"cloud_storage_type":1}
```
<!--/body-->
author_association: COLLABORATOR
user: login: BenPope
created_at: 2025-04-17T07:03:42Z
body: Changes in [force-push](https://github.com/redpanda-data/redpanda/compare/f771174d77adfe8457cd0604a464b316aeb619d0..48d615e82cd007b059c0922dce008bf5ba500f3e)
* Drop the commit implementing `operator<<` for `raw_string`
* Replace the use with `iobuf_const_parser::read_string`
author_association: MEMBER
user: login: BenPope
created_at: 2025-04-17T12:42:56Z
body: Changes in [force-push](https://github.com/redpanda-data/redpanda/compare/48d615e82cd007b059c0922dce008bf5ba500f3e..db608e1f8292d79f6fce10aee08c50d1a0bc4c0a)
* Rebase to avoid logging conflict in `reply.h`
author_association: MEMBER
user: login: vbotbuildovich
created_at: 2025-04-17T16:25:49Z
body: <!--FAILURES SLASH COMMAND https://buildkite.com/redpanda/redpanda/builds/64678-->
<!--0691ad8e-42df-4ec2-a987-adaf5f6064ba-->
<!--6b3bfb8b-4fac-479b-8d8d-7e012349dfd1-->
<!--b04a8117-cc05-4817-a0d4-eba2b20a8020-->
<!--title-->
## Retry command for [Build#64678](https://buildkite.com/redpanda/redpanda/builds/64678)
please wait until all jobs are finished before running the slash command
<!--/title-->
<!--body-->
```


/ci-repeat 1
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":false},"transfer_leadership":false}
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":false},"transfer_leadership":true}
tests/rptest/tests/upgrade_test.py::UpgradeBackToBackTest.test_upgrade_with_all_workloads@{"single_upgrade":false}
tests/rptest/tests/datalake/throttling_test.py::DatalakeThrottlingTest.test_basic_throttling@{"catalog_type":"nessie","cloud_storage_type":1}
```
<!--/body-->
author_association: COLLABORATOR
user: login: BenPope
created_at: 2025-04-22T07:51:28Z
body: /ci-repeat 1
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":false},"transfer_leadership":false}
tests/rptest/tests/data_migrations_api_test.py::DataMigrationsApiTest.test_migrated_topic_data_integrity@{"params":{"cancellation":{"dir":"in","stage":"preparing"},"use_alias":false},"transfer_leadership":true}
tests/rptest/tests/upgrade_test.py::UpgradeBackToBackTest.test_upgrade_with_all_workloads@{"single_upgrade":false}
tests/rptest/tests/datalake/throttling_test.py::DatalakeThrottlingTest.test_basic_throttling@{"catalog_type":"nessie","cloud_storage_type":1}
author_association: MEMBER
user: login: vbotbuildovich
created_at: 2025-04-22T15:47:43Z
body: /backport v25.1.x
author_association: COLLABORATOR
user: login: vbotbuildovich
created_at: 2025-04-22T15:47:44Z
body: /backport v24.3.x
author_association: COLLABORATOR