CVE-2023-34411: Improper Restriction of XML External Entity Reference (CWE-611)
value: The xml-rs crate before 0.8.14 for Rust and Crab allows a denial of service (panic) via an invalid <! token (such as <!DOCTYPEs/%<!A nesting) in an XML document. The earliest affected version is 0.8.9.
CWE description: The product processes an XML document that can contain XML entities with URIs that resolve to documents outside of the intended sphere of control, causing the product to embed incorrect documents into its output.
CVSS v3 base score: 7.5
Published: 2023-06-05T04:15Z
Repository: https://github.com/00xc/xml-rs
Language: Rust
Files changed: README.md, src/reader/lexer.rs, src/reader/parser/inside_cdata.rs, src/reader/parser/inside_doctype.rs, src/reader/parser/inside_processing_instruction.rs, src/reader/parser/mod.rs, src/reader/parser/outside_tag.rs, tests/xmlconf.rs
Commit date: 2023-05-07 00:16:50 +0100
Commit message: Parse DOCTYPE markup declarations

Diff: diff --git a/README.md b/README.md
index 8cc636c..1a5e73f 100644
--- a/README.md
+++ b/README.md
@@ -23,12 +23,13 @@ This writer consumes its own set of events, but reader events can be converted t
 writer events easily, and so it is possible to write XML transformation chains in a pretty
 clean manner.
 
 This parser is mostly full-featured, however, there are limitations:
 * Only UTF-8 is supported;
-* DTD validation is not supported, `<!DOCTYPE>` declarations are completely ignored; thus no
+* There is only very rudimentary parsing of `<!DOCTYPE>` declarations; thus no
   support for custom entities too; internal DTD declarations are likely to cause parsing errors;
+* DTD validation is not supported;
 * attribute value normalization is not performed, and end-of-line characters are not normalized either.
 
 Other than that the parser tries to be mostly XML-1.1-compliant.
 
 Writer is also mostly full-featured with the following limitations:
diff --git a/src/reader/lexer.rs b/src/reader/lexer.rs
index 661dca6..d6ba8ee 100644
--- a/src/reader/lexer.rs
+++ b/src/reader/lexer.rs
@@ -52,10 +52,12 @@ pub(crate) enum Token {
     CDataEnd,
     /// `&`
     ReferenceStart,
     /// `;`
     ReferenceEnd,
+    /// `<!` of `ENTITY`
+    MarkupDeclarationStart,
 }
 
 impl fmt::Display for Token {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
@@ -141,10 +143,11 @@ impl Token {
             _ => false,
         }
     }
 }
 
+#[derive(Copy, Clone)]
 enum State {
     /// Default state
     Normal,
     /// Triggered on '<'
     TagStarted,
@@ -152,12 +155,14 @@ enum State {
     CommentOrCDataOrDoctypeStarted,
     /// Triggered on '<!-'
     CommentStarted,
     /// Triggered on '<!D' up to '<!DOCTYPE'
     DoctypeStarted(DoctypeStartedSubstate),
+    /// Other items like `<!ELEMENT` in DTD
+    InsideMarkupDeclaration,
     /// Triggered after DoctypeStarted to handle sub elements
-    DoctypeFinishing(u8),
+    InsideDoctype,
     /// Triggered on '<![' up to '<![CDATA'
     CDataStarted(CDataStartedSubstate),
     /// Triggered on '?'
     ProcessingInstructionClosing,
     /// Triggered on '/'
@@ -172,10 +177,17 @@ enum State {
     InsideComment,
     /// After `<[[`
     InsideCdata,
     /// After `<?`
     InsideProcessingInstruction,
+    /// `<!ENTITY "here">`
+    InsideMarkupDeclarationQuotedString(QuoteStyle),
+}
+
+#[derive(Copy, Clone, Eq, PartialEq)]
+enum QuoteStyle {
+    Single, Double
 }
 
 #[derive(Copy, Clone)]
 enum ClosingSubstate {
     First, Second
@@ -227,10 +239,12 @@ macro_rules! dispatch_on_enum_state(
 pub(crate) struct Lexer {
     pos: TextPosition,
     head_pos: TextPosition,
     char_queue: VecDeque<char>,
     st: State,
+    /// Default state to go back to after a tag end (may be `InsideDoctype`)
+    normal_state: State,
     skip_errors: bool,
     inside_token: bool,
     eof_handled: bool
 }
 
@@ -246,25 +260,20 @@ impl Lexer {
         Lexer {
             pos: TextPosition::new(),
             head_pos: TextPosition::new(),
             char_queue: VecDeque::with_capacity(4),  // TODO: check size
             st: State::Normal,
+            normal_state: State::Normal,
             skip_errors: false,
             inside_token: false,
             eof_handled: false
         }
     }
 
-    /// Enables error handling so `next_token` will return `Some(Err(..))`
-    /// upon invalid lexeme.
-    #[inline]
-    pub fn enable_errors(&mut self) { self.skip_errors = false; }
-
     /// Disables error handling so `next_token` will return `Some(Chunk(..))`
     /// upon invalid lexeme with this lexeme content.
-    #[inline]
-    pub fn disable_errors(&mut self) { self.skip_errors = true; }
+    pub(crate) fn disable_errors(&mut self) { self.skip_errors = true; }
 
     /// Reset the eof handled flag of the lexer.
     #[inline]
     pub fn reset_eof_handled(&mut self) { self.eof_handled = false; }
 
@@ -324,13 +333,13 @@ impl Lexer {
             State::InsideCdata | State::CDataClosing(_) => Err(self.error("Unclosed CDATA")),
 
             State::TagStarted | State::CommentOrCDataOrDoctypeStarted |
             State::CommentStarted | State::CDataStarted(_)| State::DoctypeStarted(_) |
             State::CommentClosing(ClosingSubstate::Second) |
-            State::InsideComment |
+            State::InsideComment | State::InsideMarkupDeclaration |
             State::InsideProcessingInstruction | State::ProcessingInstructionClosing |
-            State::DoctypeFinishing(_) =>
+            State::InsideDoctype | State::InsideMarkupDeclarationQuotedString(_) =>
                 Err(self.error("Unexpected end of stream")),
             State::EmptyTagClosing =>
                 Ok(Some(Token::Character('/'))),
             State::CommentClosing(ClosingSubstate::First) =>
                 Ok(Some(Token::Character('-'))),
@@ -367,19 +376,21 @@ impl Lexer {
             State::TagStarted                     => self.tag_opened(c),
             State::CommentOrCDataOrDoctypeStarted => self.comment_or_cdata_or_doctype_started(c),
             State::CommentStarted                 => self.comment_started(c),
             State::CDataStarted(s)                => self.cdata_started(c, s),
             State::DoctypeStarted(s)              => self.doctype_started(c, s),
-            State::DoctypeFinishing(d)            => self.doctype_finishing(c, d),
+            State::InsideDoctype                  => self.inside_doctype(c),
             State::EmptyTagClosing                => self.empty_element_closing(c),
             State::CommentClosing(s)              => self.comment_closing(c, s),
             State::CDataClosing(s)                => self.cdata_closing(c, s),
             State::InvalidCDataClosing(s)         => self.invalid_cdata_closing(c, s),
             State::InsideComment                  => self.inside_comment_state(c),
             State::InsideCdata                    => self.inside_cdata(c),
             State::InsideProcessingInstruction    => self.inside_processing_instruction(c),
             State::ProcessingInstructionClosing   => self.processing_instruction_closing(c),
+            State::InsideMarkupDeclaration       => self.markup_declaration(c),
+            State::InsideMarkupDeclarationQuotedString(q) => self.markup_declaration_string(c, q),
         }
     }
 
     #[inline]
     fn move_to(&mut self, st: State) -> Result {
@@ -391,10 +402,17 @@ impl Lexer {
     fn move_to_with(&mut self, st: State, token: Token) -> Result {
         self.st = st;
         Ok(Some(token))
     }
 
+    #[inline]
+    fn move_to_and_reset_normal(&mut self, st: State, token: Token) -> Result {
+        self.normal_state = st;
+        self.st = st;
+        Ok(Some(token))
+    }
+
     #[inline]
     fn move_to_with_unread(&mut self, st: State, cs: &[char], token: Token) -> Result {
         self.char_queue.extend(cs.iter().copied());
         self.move_to_with(st, token)
     }
@@ -432,10 +450,11 @@ impl Lexer {
             _                          => Ok(Some(Token::Character(c)))
         }
     }
 
     fn inside_processing_instruction(&mut self, c: char) -> Result {
+        // These tokens are used by `<?xml?>` parser
         match c {
             '?'                        => self.move_to(State::ProcessingInstructionClosing),
             '<'                        => Ok(Some(Token::OpeningTagStart)),
             '>'                        => Ok(Some(Token::TagEnd)),
             '/'                        => Ok(Some(Token::ClosingTagStart)),
@@ -459,24 +478,25 @@ impl Lexer {
 
     /// Encountered '<'
     fn tag_opened(&mut self, c: char) -> Result {
         match c {
             '?'                        => self.move_to_with(State::InsideProcessingInstruction, Token::ProcessingInstructionStart),
-            '/'                        => self.move_to_with(State::Normal, Token::ClosingTagStart),
+            '/'                        => self.move_to_with(self.normal_state, Token::ClosingTagStart),
             '!'                        => self.move_to(State::CommentOrCDataOrDoctypeStarted),
-            _ if is_whitespace_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
-            _ if is_name_char(c)       => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
+            _ if is_whitespace_char(c) => self.move_to_with_unread(self.normal_state, &[c], Token::OpeningTagStart),
+            _ if is_name_char(c)       => self.move_to_with_unread(self.normal_state, &[c], Token::OpeningTagStart),
             _                          => self.handle_error("<", c)
         }
     }
 
     /// Encountered '<!'
     fn comment_or_cdata_or_doctype_started(&mut self, c: char) -> Result {
         match c {
             '-' => self.move_to(State::CommentStarted),
             '[' => self.move_to(State::CDataStarted(CDataStartedSubstate::E)),
             'D' => self.move_to(State::DoctypeStarted(DoctypeStartedSubstate::D)),
+            'E' | 'A' | 'N' if matches!(self.normal_state, State::InsideDoctype) => self.move_to_with(State::InsideMarkupDeclaration, Token::MarkupDeclarationStart),
             _ => self.handle_error("<!", c),
         }
     }
 
     /// Encountered '<!-'
@@ -498,46 +518,68 @@ impl Lexer {
             CDAT  ; 'A' ; CDATA ; "<![CDAT";
             CDATA ; '[' ; "<![CDATA" ; self.move_to_with(State::InsideCdata, Token::CDataStart)
         )
     }
 
+    /// Encountered '<!…' that isn't DOCTYPE or CDATA
+    fn markup_declaration(&mut self, c: char) -> Result {
+        match c {
+            '<'                        => self.handle_error("<!", c),
+            '>'                        => self.move_to_with(self.normal_state, Token::TagEnd),
+            '&'                        => Ok(Some(Token::ReferenceStart)),
+            ';'                        => Ok(Some(Token::ReferenceEnd)),
+            '"'                        => self.move_to_with(State::InsideMarkupDeclarationQuotedString(QuoteStyle::Double), Token::DoubleQuote),
+            '\''                       => self.move_to_with(State::InsideMarkupDeclarationQuotedString(QuoteStyle::Single), Token::SingleQuote),
+            _ => Ok(None),
+        }
+    }
+
+    fn markup_declaration_string(&mut self, c: char, q: QuoteStyle) -> Result {
+        match c {
+            '"' if q == QuoteStyle::Double  => self.move_to_with(State::InsideMarkupDeclaration, Token::DoubleQuote),
+            '\'' if q == QuoteStyle::Single => self.move_to_with(State::InsideMarkupDeclaration, Token::SingleQuote),
+            _ => Ok(None),
+        }
+    }
+
     /// Encountered '<!D'
     fn doctype_started(&mut self, c: char, s: DoctypeStartedSubstate) -> Result {
         use self::DoctypeStartedSubstate::{D, DO, DOC, DOCT, DOCTY, DOCTYP};
         dispatch_on_enum_state!(self, s, c, State::DoctypeStarted,
             D      ; 'O' ; DO     ; "<!D",
             DO     ; 'C' ; DOC    ; "<!DO",
             DOC    ; 'T' ; DOCT   ; "<!DOC",
             DOCT   ; 'Y' ; DOCTY  ; "<!DOCT",
             DOCTY  ; 'P' ; DOCTYP ; "<!DOCTY";
-            DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_with(State::DoctypeFinishing(1), Token::DoctypeStart)
+            DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_and_reset_normal(State::InsideDoctype, Token::DoctypeStart)
         )
     }
 
     /// State used while awaiting the closing bracket for the <!DOCTYPE tag
-    fn doctype_finishing(&mut self, c: char, d: u8) -> Result {
+    fn inside_doctype(&mut self, c: char) -> Result {
         match c {
-            '<' => self.move_to(State::DoctypeFinishing(d + 1)),
-            '>' if d == 1 => self.move_to_with(State::Normal, Token::TagEnd),
-            '>' => self.move_to(State::DoctypeFinishing(d - 1)),
+            '>' => self.move_to_and_reset_normal(State::Normal, Token::TagEnd),
+            '<'                        => self.move_to(State::TagStarted),
+            '&'                        => Ok(Some(Token::ReferenceStart)),
+            ';'                        => Ok(Some(Token::ReferenceEnd)),
             _ => Ok(None),
         }
     }
 
     /// Encountered '?'
     fn processing_instruction_closing(&mut self, c: char) -> Result {
         match c {
-            '>' => self.move_to_with(State::Normal, Token::ProcessingInstructionEnd),
+            '>' => self.move_to_with(self.normal_state, Token::ProcessingInstructionEnd),
             _ => self.move_to_with_unread(State::InsideProcessingInstruction, &[c], Token::Character('?')),
         }
     }
 
     /// Encountered '/'
     fn empty_element_closing(&mut self, c: char) -> Result {
         match c {
-            '>' => self.move_to_with(State::Normal, Token::EmptyTagEnd),
-            _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('/')),
+            '>' => self.move_to_with(self.normal_state, Token::EmptyTagEnd),
+            _ => self.move_to_with_unread(self.normal_state, &[c], Token::Character('/')),
         }
     }
 
     /// Encountered '-'
     fn comment_closing(&mut self, c: char, s: ClosingSubstate) -> Result {
@@ -545,11 +587,11 @@ impl Lexer {
             ClosingSubstate::First => match c {
                 '-' => self.move_to(State::CommentClosing(ClosingSubstate::Second)),
                 _ => self.move_to_with_unread(State::InsideComment, &[c], Token::Character('-')),
             },
             ClosingSubstate::Second => match c {
-                '>' => self.move_to_with(State::Normal, Token::CommentEnd),
+                '>' => self.move_to_with(self.normal_state, Token::CommentEnd),
                 // double dash not followed by a greater-than is a hard error inside comment
                 _ => self.handle_error("--", c),
             },
         }
     }
@@ -574,11 +616,11 @@ impl Lexer {
             ClosingSubstate::First => match c {
                 ']' => self.move_to(State::InvalidCDataClosing(ClosingSubstate::Second)),
                 _ => self.move_to_with_unread(State::Normal, &[c], Token::Character(']')),
             },
             ClosingSubstate::Second => match c {
-                '>' => self.move_to_with(State::Normal, Token::CDataEnd),
+                '>' => self.move_to_with(self.normal_state, Token::CDataEnd),
                 _ => self.move_to_with_unread(State::Normal, &[']', c], Token::Character(']')),
             },
         }
     }
 }
@@ -823,23 +865,58 @@ mod tests {
     }
 
     #[test]
     fn doctype_with_internal_subset_test() {
         let (mut lex, mut buf) = make_lex_and_buf(
-            r#"<a><!DOCTYPE ab[<!ELEMENT ba> ]> "#
+            r#"<a><!DOCTYPE ab[<!ELEMENT ba ">>>>>"> ]> "#
         );
         assert_oks!(for lex and buf ;
             Token::OpeningTagStart
             Token::Character('a')
             Token::TagEnd
             Token::DoctypeStart
+            Token::MarkupDeclarationStart
+            Token::DoubleQuote
+            Token::DoubleQuote
+            Token::TagEnd
             Token::TagEnd
             Token::Whitespace(' ')
         );
         assert_none!(for lex and buf);
     }
 
+    #[test]
+    fn doctype_internal_pi_comment() {
+        let (mut lex, mut buf) = make_lex_and_buf(
+            "<!DOCTYPE a [\n<!ELEMENT leopard ANY> <!-- <?non?>--> <?pi > ?> \n]>"
+        );
+        assert_oks!(for lex and buf ;
+            Token::DoctypeStart
+            Token::MarkupDeclarationStart
+            Token::TagEnd
+            Token::CommentStart
+            Token::Whitespace(' ')
+            Token::Character('<')
+            Token::Character('?')
+            Token::Character('n')
+            Token::Character('o')
+            Token::Character('n')
+            Token::Character('?')
+            Token::Character('>')
+            Token::CommentEnd
+            Token::ProcessingInstructionStart
+            Token::Character('p')
+            Token::Character('i')
+            Token::Whitespace(' ')
+            Token::TagEnd // not really
+            Token::Whitespace(' ')
+            Token::ProcessingInstructionEnd
+            Token::TagEnd // DTD
+        );
+        assert_none!(for lex and buf);
+    }
+
     #[test]
     fn end_of_stream_handling_ok() {
         macro_rules! eof_check(
             ($data:expr ; $token:expr) => ({
                 let (mut lex, mut buf) = make_lex_and_buf($data);
@@ -870,11 +947,10 @@ mod tests {
         eof_check!("<![C"     ; 0, 4);
         eof_check!("<![CD"    ; 0, 5);
         eof_check!("<![CDA"   ; 0, 6);
         eof_check!("<![CDAT"  ; 0, 7);
         eof_check!("<![CDATA" ; 0, 8);
-        // eof_check!("--"       ; 0, 2);
     }
 
     #[test]
     fn error_in_comment_or_cdata_prefix() {
         let (mut lex, mut buf) = make_lex_and_buf("<!x");
diff --git a/src/reader/parser/inside_cdata.rs b/src/reader/parser/inside_cdata.rs
index 4e8c319..559e60b 100644
--- a/src/reader/parser/inside_cdata.rs
+++ b/src/reader/parser/inside_cdata.rs
@@ -5,11 +5,10 @@ use super::{PullParser, Result, State};
 
 impl PullParser {
     pub fn inside_cdata(&mut self, t: Token) -> Option<Result> {
         match t {
             Token::CDataEnd => {
-                self.lexer.enable_errors();
                 let event = if self.config.cdata_to_characters {
                     None
                 } else {
                     let data = self.take_buf();
                     Some(Ok(XmlEvent::CData(data)))
diff --git a/src/reader/parser/inside_doctype.rs b/src/reader/parser/inside_doctype.rs
index fbec90a..ac3eb22 100644
--- a/src/reader/parser/inside_doctype.rs
+++ b/src/reader/parser/inside_doctype.rs
@@ -4,13 +4,26 @@ use super::{PullParser, Result, State};
 
 impl PullParser {
     pub fn inside_doctype(&mut self, t: Token) -> Option<Result> {
         match t {
             Token::TagEnd => {
-                self.lexer.enable_errors();
                 self.into_state_continue(State::OutsideTag)
             }
 
+            Token::MarkupDeclarationStart => {
+                self.into_state_continue(State::InsideDoctypeMarkupDeclaration)
+            },
+
+            _ => None,
+        }
+    }
+
+    pub fn inside_doctype_markup_declaration(&mut self, t: Token) -> Option<Result> {
+        match t {
+            Token::TagEnd => {
+                self.into_state_continue(State::InsideDoctype)
+            }
+
             _ => None,
         }
     }
 }
diff --git a/src/reader/parser/inside_processing_instruction.rs b/src/reader/parser/inside_processing_instruction.rs
index 22a63ff..f032afc 100644
--- a/src/reader/parser/inside_processing_instruction.rs
+++ b/src/reader/parser/inside_processing_instruction.rs
@@ -66,11 +66,10 @@ impl PullParser {
                 _ => Some(self_error!(self; "Unexpected token: <?{}{}", self.buf, t)),
             },
 
             ProcessingInstructionSubstate::PIInsideData => match t {
                 Token::ProcessingInstructionEnd => {
-                    self.lexer.enable_errors();
                     let name = self.data.take_name();
                     let data = self.take_buf();
                     self.into_state_emit(
                         State::OutsideTag,
                         Ok(XmlEvent::ProcessingInstruction {
diff --git a/src/reader/parser/mod.rs b/src/reader/parser/mod.rs
index 402a8fa..62ee255 100644
--- a/src/reader/parser/mod.rs
+++ b/src/reader/parser/mod.rs
@@ -137,10 +137,11 @@ pub enum State {
     InsideProcessingInstruction(ProcessingInstructionSubstate),
     InsideComment,
     InsideCData,
     InsideDeclaration(DeclarationSubstate),
     InsideDoctype,
+    InsideDoctypeMarkupDeclaration,
     InsideReference(Box<State>),
 }
 
 #[derive(Clone, PartialEq)]
 pub enum OpeningTagSubstate {
@@ -335,10 +336,11 @@ impl PullParser {
         match self.st.clone() {
             State::OutsideTag                     => self.outside_tag(t),
             State::InsideProcessingInstruction(s) => self.inside_processing_instruction(t, s),
             State::InsideDeclaration(s)           => self.inside_declaration(t, s),
             State::InsideDoctype                  => self.inside_doctype(t),
+            State::InsideDoctypeMarkupDeclaration => self.inside_doctype_markup_declaration(t),
             State::InsideOpeningTag(s)            => self.inside_opening_tag(t, s),
             State::InsideClosingTag(s)            => self.inside_closing_tag_name(t, s),
             State::InsideComment                  => self.inside_comment(t),
             State::InsideCData                    => self.inside_cdata(t),
             State::InsideReference(s)             => self.inside_reference(t, *s)
diff --git a/src/reader/parser/outside_tag.rs b/src/reader/parser/outside_tag.rs
index 3c2a1d8..d28d6be 100644
--- a/src/reader/parser/outside_tag.rs
+++ b/src/reader/parser/outside_tag.rs
@@ -79,11 +79,10 @@ impl PullParser {
 
                     Token::DoctypeStart if !self.encountered_element => {
                         // We don't have a doctype event so skip this position
                         // FIXME: update when we have a doctype event
                         self.next_pos();
-                        self.lexer.disable_errors();
                         self.into_state(State::InsideDoctype, next_event)
                     }
 
                     Token::OpeningTagStart => {
                         // If declaration was not parsed and we have encountered an element,
diff --git a/tests/xmlconf.rs b/tests/xmlconf.rs
index 965a8c2..6d5c550 100644
--- a/tests/xmlconf.rs
+++ b/tests/xmlconf.rs
@@ -114,11 +114,10 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "rmt-e2e-15f", // Element content can contain entity reference if replacement text is whitespace, even if it came from a character reference in the literal entity value
         "rmt-e2e-15h", // Element content can't contain entity reference if replacement text is character reference to whitespace
         "rmt-e2e-18", // External entity containing start of entity declaration is base URI for system identifier
         "rmt-e2e-19", // Parameter entities and character references are included-in-literal, but general entities are bypassed.
         "rmt-e2e-22", // UTF-8 entities may start with a BOM
-        "rmt-e2e-24", // Either the built-in entity or a character reference can be used to represent greater-than after two close-square-brackets
         "rmt-e2e-34", // A non-deterministic content model is an error even if the element type is not used.
         "rmt-e2e-50", // All line-ends are normalized, even those not passed to the application. NB this can only be tested effectively in XML 1.1, since CR is in the S production; in 1.1 we can use NEL which isn't.
         "rmt-e2e-55", // A reference to an unparsed entity in an entity value is an error rather than forbidden (unless the entity is referenced, of course)
         "rmt-e2e-57", // A value other than preserve or default for xml:space is an error
         "rmt-e2e-61", // (From John Cowan) An encoding declaration in ASCII specifying an encoding that is not compatible with ASCII (so the document is not in its declared encoding).  It should generate a fatal error.
@@ -276,42 +275,38 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
     ]);
 }
 
 #[test] fn oasis() {
     run_suite("oasis/oasis.xml", &[
-        "o-p43pass1", // Valid use of character data, comments, processing instructions and CDATA sections within the start and end tag.
-        "o-p68pass1", // Valid entity references.  Also ensures that a charref to           '&' isn't interpreted as an entity reference open delimiter
-        "o-p04pass1", // names with all valid ASCII characters, and one from each               other class in NameChar
-        "o-p05pass1", // various valid Name constructions
         "o-p01fail1", // S cannot occur before the prolog
         "o-p01fail2", // comments cannot occur before the prolog
         "o-p01fail3", // only one document element
+        "o-p04pass1", // names with all valid ASCII characters, and one from each               other class in NameChar
+        "o-p05pass1", // various valid Name constructions
         "o-p09fail1", // EntityValue excludes '%'
         "o-p09fail2", // EntityValue excludes '&'
         "o-p09fail3", // incomplete character reference
-        "o-p09fail4", // quote types must match
-        "o-p09fail5", // quote types must match
-        "o-p11fail1", // quote types must match
-        "o-p11fail2", // cannot contain delimiting quotes
-        "o-p12fail1", // '"' excluded
+        "o-p11pass1", // p11pass1.xml       system literals may not contain     URI fragments
+        "o-p12fail1", // p12fail1.xml       '"' excluded
         "o-p12fail2", // '\' excluded
         "o-p12fail3", // entity references excluded
+        "o-p12fail4", // p12fail4.xml       '>' excluded
+        "o-p12fail5", // p12fail5.xml       '<' excluded
         "o-p12fail6", // built-in entity refs excluded
         "o-p12fail7", // The public ID has a tab character, which is disallowed
         "o-p14fail3", // "]]>" excluded
         "o-p18fail3", // CDSect's can't nest
         "o-p22fail1", // prolog must start with XML decl
         "o-p22fail2", // prolog must start with XML decl
         "o-p23fail1", // "xml" must be lower-case
         "o-p27fail1", // References aren't allowed in Misc,     even if they would resolve to valid Misc.
-        "o-p29fail1", // A processor must not pass unknown declaration types.
         "o-p30fail1", // An XML declaration is not the same as a TextDecl
         "o-p31fail1", // external subset excludes doctypedecl
         "o-p32fail3", // initial S is required
         "o-p40fail1", // S is required between attributes
+        "o-p43pass1", // Valid use of character data, comments, processing instructions and CDATA sections within the start and end tag.
         "o-p44fail4", // Whitespace required between attributes.
-        "o-p45fail1", // ELEMENT must be upper case.
         "o-p45fail2", // S before contentspec is required.
         "o-p45fail3", // only one content spec
         "o-p45fail4", // no comments in declarations (contrast with SGML)
         "o-p46fail1", // no parens on declared content
         "o-p46fail2", // no inclusions (contrast with SGML)
@@ -369,21 +364,22 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "o-p63fail1", // IGNORE must be upper case
         "o-p63fail2", // delimiters must be balanced
         "o-p64fail1", // section delimiters must balance
         "o-p64fail2", // section delimiters must balance
         "o-p66fail5", // no references to non-characters
+        "o-p68pass1", // Valid entity references.  Also ensures that a charref to           '&' isn't interpreted as an entity reference open delimiter
         "o-p69fail1", // terminating ';' is required
         "o-p69fail2", // no S after '%'
         "o-p69fail3", // no S before ';'
         "o-p70fail1", // This is neither
         "o-p71fail1", // S is required before EntityDef
         "o-p71fail2", // Entity name is a Name, not an NMToken
-        "o-p71fail3", // no S after "<!"
         "o-p71fail4", // S is required after "<!ENTITY"
         "o-p72fail1", // S is required after "<!ENTITY"
         "o-p72fail2", // S is required after '%'
         "o-p72fail3", // S is required after name
+        "o-p76fail4", // p76fail4.xml       notation names are Names
         "o-p72fail4", // Entity name is a name, not an NMToken
         "o-p73fail1", // No typed replacement text
         "o-p73fail2", // Only one replacement value
         "o-p73fail3", // No NDataDecl on replacement text
         "o-p73fail4", // Value is required
@@ -436,12 +432,10 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "cond02", // Must have keyword in conditional sections
         "content01", // No whitespace before "?" in content model
         "content02", // No whitespace before "*" in content model
         "content03", // No whitespace before "+" in content model
         "decl01", // External entities may not have standalone decls.
-        "nwf-dtd00", // Comma mandatory in content model
-        "nwf-dtd01", // Can't mix comma and vertical bar in content models
         "dtd02", // PE name immediately after "%"
         "dtd03", // PE name immediately followed by ";"
         "dtd04", // PUBLIC literal must be quoted
         "dtd05", // SYSTEM identifier must be quoted
         "dtd07", // Text declarations (which optionally begin any external entity)     are required to have "encoding=...".
@@ -449,10 +443,13 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "encoding02", // Illegal character "/" in encoding name
         "encoding03", // Illegal character reference in encoding name
         "encoding04", // Illegal character ":" in encoding name
         "encoding05", // Illegal character "@" in encoding name
         "encoding06", // Illegal character "+" in encoding name
+        "nwf-dtd00", // Comma mandatory in content model
+        "nwf-dtd01", // Can't mix comma and vertical bar in content models
+        "pi", // pi.xml      No space between PI target name and data
         "pubid01", // Illegal entity ref in public ID
         "pubid02", // Illegal characters in public ID
         "pubid03", // Illegal characters in public ID
         "pubid04", // Illegal characters in public ID
         "pubid05", // SGML-ism:  public ID without system ID
@@ -500,11 +497,10 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "not-wf-sa-058", // Invalid character , in ATTLIST enumeration
         "not-wf-sa-059", // String literal must be in quotes.
         "not-wf-sa-060", // Invalid type NAME defined in ATTLIST.
         "not-wf-sa-061", // External entity declarations require whitespace between public     and system IDs.
         "not-wf-sa-062", // Entity declarations need space after the entity name.
-        "not-wf-sa-063", // Conditional sections may only appear in the external     DTD subset.
         "not-wf-sa-064", // Space is required between attribute type and default values     in <!ATTLIST...> declarations.
         "not-wf-sa-065", // Space is required between attribute name and type     in <!ATTLIST...> declarations.
         "not-wf-sa-066", // Required whitespace is missing.
         "not-wf-sa-067", // Space is required between attribute type and default values     in <!ATTLIST...> declarations.
         "not-wf-sa-068", // Space is required between NOTATION keyword and list of     enumerated choices in <!ATTLIST...> declarations.
@@ -521,11 +517,10 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "not-wf-sa-091", // Parameter entities "are" always parsed; NDATA annotations     are not permitted.
         "not-wf-sa-096", // Space is required before the standalone declaration.
         "not-wf-sa-101", // Space is not permitted in an encoding name.
         "not-wf-sa-105", // Invalid placement of CDATA section.
         "not-wf-sa-106", // Invalid placement of entity declaration.
-        "not-wf-sa-107", // Invalid document type declaration.  CDATA alone is invalid.
         "not-wf-sa-113", // Parameter entity values must use valid reference syntax;     this reference is malformed.
         "not-wf-sa-114", // General entity values must use valid reference syntax;     this reference is malformed.
         "not-wf-sa-121", // A name of an ENTITY was started with an invalid character.
         "not-wf-sa-122", // Invalid syntax mixed connectors are used.
         "not-wf-sa-123", // Invalid syntax mismatched parenthesis.
@@ -564,11 +559,10 @@ fn expect_ill_formed(xml_path: &Path, msg: &str, id: &str) -> Result<(), Box<dyn
         "not-wf-sa-172", // Character FFFF is not legal anywhere in an XML document.
         "not-wf-sa-173", // Character FFFF is not legal anywhere in an XML document.
         "not-wf-sa-174", // Character FFFF is not legal anywhere in an XML document.
         "not-wf-sa-175", // Character FFFF is not legal anywhere in an XML document.
         "not-wf-sa-177", // Character FFFF is not legal anywhere in an XML document.
-        "not-wf-sa-179", // Invalid syntax matching double quote is missing.
         "not-wf-sa-180", // The Entity Declared WFC requires entities to be declared     before they are used in an attribute list declaration.
         "not-wf-sa-183", // Mixed content declarations may not include content particles.
         "not-wf-sa-184", // In mixed content models, element names must not be     parenthesized.
         "not-wf-sa-186", // Whitespace is required between attribute/value pairs.
         "not-wf-not-sa-001", // Conditional sections must be properly terminated ("]>" used     instead of "]]>").

Vulnerable code: * DTD validation is not supported, `<!DOCTYPE>` declarations are completely ignored; thus no
    DoctypeFinishing(u8),
    /// Enables error handling so `next_token` will return `Some(Err(..))`
    /// upon invalid lexeme.
    #[inline]
    pub fn enable_errors(&mut self) { self.skip_errors = false; }

    #[inline]
    pub fn disable_errors(&mut self) { self.skip_errors = true; }
            State::InsideComment |
            State::DoctypeFinishing(_) =>
            State::DoctypeFinishing(d)            => self.doctype_finishing(c, d),
            '/'                        => self.move_to_with(State::Normal, Token::ClosingTagStart),
            _ if is_whitespace_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
            _ if is_name_char(c)       => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
            DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_with(State::DoctypeFinishing(1), Token::DoctypeStart)
    fn doctype_finishing(&mut self, c: char, d: u8) -> Result {
            '<' => self.move_to(State::DoctypeFinishing(d + 1)),
            '>' if d == 1 => self.move_to_with(State::Normal, Token::TagEnd),
            '>' => self.move_to(State::DoctypeFinishing(d - 1)),
            '>' => self.move_to_with(State::Normal, Token::ProcessingInstructionEnd),
            '>' => self.move_to_with(State::Normal, Token::EmptyTagEnd),
            _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('/')),
                '>' => self.move_to_with(State::Normal, Token::CommentEnd),
                '>' => self.move_to_with(State::Normal, Token::CDataEnd),
            r#"<a><!DOCTYPE ab[<!ELEMENT ba> ]> "#
        // eof_check!("--"       ; 0, 2);
                self.lexer.enable_errors();
                self.lexer.enable_errors();
                    self.lexer.enable_errors();
                        self.lexer.disable_errors();
        "rmt-e2e-24", // Either the built-in entity or a character reference can be used to represent greater-than after two close-square-brackets
        "o-p43pass1", // Valid use of character data, comments, processing instructions and CDATA sections within the start and end tag.
        "o-p68pass1", // Valid entity references.  Also ensures that a charref to           '&' isn't interpreted as an entity reference open delimiter
        "o-p04pass1", // names with all valid ASCII characters, and one from each               other class in NameChar
        "o-p05pass1", // various valid Name constructions
        "o-p09fail4", // quote types must match
        "o-p09fail5", // quote types must match
        "o-p11fail1", // quote types must match
        "o-p11fail2", // cannot contain delimiting quotes
        "o-p12fail1", // '"' excluded
        "o-p29fail1", // A processor must not pass unknown declaration types.
        "o-p45fail1", // ELEMENT must be upper case.
        "o-p71fail3", // no S after "<!"
        "nwf-dtd00", // Comma mandatory in content model
        "nwf-dtd01", // Can't mix comma and vertical bar in content models
        "not-wf-sa-063", // Conditional sections may only appear in the external     DTD subset.
        "not-wf-sa-107", // Invalid document type declaration.  CDATA alone is invalid.
        "not-wf-sa-179", // Invalid syntax matching double quote is missing.
Fixed code: * There is only very rudimentary parsing of `<!DOCTYPE>` declarations; thus no
* DTD validation is not supported;
    /// `<!` of `ENTITY`
    MarkupDeclarationStart,
#[derive(Copy, Clone)]
    /// Other items like `<!ELEMENT` in DTD
    InsideMarkupDeclaration,
    InsideDoctype,
    /// `<!ENTITY "here">`
    InsideMarkupDeclarationQuotedString(QuoteStyle),
}

#[derive(Copy, Clone, Eq, PartialEq)]
enum QuoteStyle {
    Single, Double
    /// Default state to go back to after a tag end (may be `InsideDoctype`)
    normal_state: State,
            normal_state: State::Normal,
    pub(crate) fn disable_errors(&mut self) { self.skip_errors = true; }
            State::InsideComment | State::InsideMarkupDeclaration |
            State::InsideDoctype | State::InsideMarkupDeclarationQuotedString(_) =>
            State::InsideDoctype                  => self.inside_doctype(c),
            State::InsideMarkupDeclaration       => self.markup_declaration(c),
            State::InsideMarkupDeclarationQuotedString(q) => self.markup_declaration_string(c, q),
    #[inline]
    fn move_to_and_reset_normal(&mut self, st: State, token: Token) -> Result {
        self.normal_state = st;
        self.st = st;
        Ok(Some(token))
    }

        // These tokens are used by `<?xml?>` parser
            '/'                        => self.move_to_with(self.normal_state, Token::ClosingTagStart),
            _ if is_whitespace_char(c) => self.move_to_with_unread(self.normal_state, &[c], Token::OpeningTagStart),
            _ if is_name_char(c)       => self.move_to_with_unread(self.normal_state, &[c], Token::OpeningTagStart),
            'E' | 'A' | 'N' if matches!(self.normal_state, State::InsideDoctype) => self.move_to_with(State::InsideMarkupDeclaration, Token::MarkupDeclarationStart),
    /// Encountered '<!…' that isn't DOCTYPE or CDATA
    fn markup_declaration(&mut self, c: char) -> Result {
        match c {
            '<'                        => self.handle_error("<!", c),
            '>'                        => self.move_to_with(self.normal_state, Token::TagEnd),
            '&'                        => Ok(Some(Token::ReferenceStart)),
            ';'                        => Ok(Some(Token::ReferenceEnd)),
            '"'                        => self.move_to_with(State::InsideMarkupDeclarationQuotedString(QuoteStyle::Double), Token::DoubleQuote),
            '\''                       => self.move_to_with(State::InsideMarkupDeclarationQuotedString(QuoteStyle::Single), Token::SingleQuote),
            _ => Ok(None),
        }
    }

    fn markup_declaration_string(&mut self, c: char, q: QuoteStyle) -> Result {
        match c {
            '"' if q == QuoteStyle::Double  => self.move_to_with(State::InsideMarkupDeclaration, Token::DoubleQuote),
            '\'' if q == QuoteStyle::Single => self.move_to_with(State::InsideMarkupDeclaration, Token::SingleQuote),
            _ => Ok(None),
        }
    }

            DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_and_reset_normal(State::InsideDoctype, Token::DoctypeStart)
    fn inside_doctype(&mut self, c: char) -> Result {
            '>' => self.move_to_and_reset_normal(State::Normal, Token::TagEnd),
            '<'                        => self.move_to(State::TagStarted),
            '&'                        => Ok(Some(Token::ReferenceStart)),
            ';'                        => Ok(Some(Token::ReferenceEnd)),
            '>' => self.move_to_with(self.normal_state, Token::ProcessingInstructionEnd),
            '>' => self.move_to_with(self.normal_state, Token::EmptyTagEnd),
            _ => self.move_to_with_unread(self.normal_state, &[c], Token::Character('/')),
                '>' => self.move_to_with(self.normal_state, Token::CommentEnd),
                '>' => self.move_to_with(self.normal_state, Token::CDataEnd),
            r#"<a><!DOCTYPE ab[<!ELEMENT ba ">>>>>"> ]> "#
            Token::MarkupDeclarationStart
            Token::DoubleQuote
            Token::DoubleQuote
            Token::TagEnd
    #[test]
    fn doctype_internal_pi_comment() {
        let (mut lex, mut buf) = make_lex_and_buf(
            "<!DOCTYPE a [\n<!ELEMENT leopard ANY> <!-- <?non?>--> <?pi > ?> \n]>"
        );
        assert_oks!(for lex and buf ;
            Token::DoctypeStart
            Token::MarkupDeclarationStart
            Token::TagEnd
            Token::CommentStart
            Token::Whitespace(' ')
            Token::Character('<')
            Token::Character('?')
            Token::Character('n')
            Token::Character('o')
            Token::Character('n')
            Token::Character('?')
            Token::Character('>')
            Token::CommentEnd
            Token::ProcessingInstructionStart
            Token::Character('p')
            Token::Character('i')
            Token::Whitespace(' ')
            Token::TagEnd // not really
            Token::Whitespace(' ')
            Token::ProcessingInstructionEnd
            Token::TagEnd // DTD
        );
        assert_none!(for lex and buf);
    }

            Token::MarkupDeclarationStart => {
                self.into_state_continue(State::InsideDoctypeMarkupDeclaration)
            },

            _ => None,
        }
    }

    pub fn inside_doctype_markup_declaration(&mut self, t: Token) -> Option<Result> {
        match t {
            Token::TagEnd => {
                self.into_state_continue(State::InsideDoctype)
            }

    InsideDoctypeMarkupDeclaration,
            State::InsideDoctypeMarkupDeclaration => self.inside_doctype_markup_declaration(t),
        "o-p04pass1", // names with all valid ASCII characters, and one from each               other class in NameChar
        "o-p05pass1", // various valid Name constructions
        "o-p11pass1", // p11pass1.xml       system literals may not contain     URI fragments
        "o-p12fail1", // p12fail1.xml       '"' excluded
        "o-p12fail4", // p12fail4.xml       '>' excluded
        "o-p12fail5", // p12fail5.xml       '<' excluded
        "o-p43pass1", // Valid use of character data, comments, processing instructions and CDATA sections within the start and end tag.
        "o-p68pass1", // Valid entity references.  Also ensures that a charref to           '&' isn't interpreted as an entity reference open delimiter
        "o-p76fail4", // p76fail4.xml       notation names are Names
        "nwf-dtd00", // Comma mandatory in content model
        "nwf-dtd01", // Can't mix comma and vertical bar in content models
        "pi", // pi.xml      No space between PI target name and data