gh-161145: Use _Py_find_first_nonascii in the tokenizer

Include/internal/pycore_unicodeobject.h

@@ -234,6 +234,13 @@ extern PyObject* _PyUnicode_FromASCII(
     const char *buffer,
     Py_ssize_t size);
 
+/* Return the index of the first byte in [start, end) with the high bit set,
+   or end - start if there is none. */
+// Export for test_peg_generator.
+PyAPI_FUNC(Py_ssize_t) _Py_find_first_nonascii(
+    const unsigned char *start,
+    const unsigned char *end);
+
 /* Compute the maximum character of the substring unicode[start:end].
    Return 127 for an empty string. */
 extern Py_UCS4 _PyUnicode_FindMaxChar (

Objects/unicodeobject.c

@@ -5047,8 +5047,8 @@ load_unaligned(const unsigned char *p, size_t size)
  * significant bit set). If all characters in the range are ASCII, it returns
  * `end - start`.
  */
-static Py_ssize_t
-find_first_nonascii(const unsigned char *start, const unsigned char *end)
+Py_ssize_t
+_Py_find_first_nonascii(const unsigned char *start, const unsigned char *end)
 {
     // The search is done in `size_t` chunks.
     // The start and end might not be aligned at `size_t` boundaries,
@@ -5188,8 +5188,8 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest)
         return p - start;
     }
 #endif
-    Py_ssize_t pos = find_first_nonascii((const unsigned char*)start,
-                                         (const unsigned char*)end);
+    Py_ssize_t pos = _Py_find_first_nonascii((const unsigned char*)start,
+                                             (const unsigned char*)end);
     memcpy(dest, start, pos);
     return pos;
 }
@@ -5348,7 +5348,8 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
     const char *starts = s;
     const char *end = s + size;
 
-    Py_ssize_t pos = find_first_nonascii((const unsigned char*)starts, (const unsigned char*)end);
+    Py_ssize_t pos = _Py_find_first_nonascii((const unsigned char*)starts,
+                                             (const unsigned char*)end);
     if (pos == size) {  // fast path: ASCII string.
         PyObject *u = PyUnicode_New(size, 127);
         if (u == NULL) {

Parser/tokenizer/helpers.c

@@ -3,6 +3,7 @@
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_token.h"
 #include "pycore_tuple.h"         // _PyTuple_FromPair
+#include "pycore_unicodeobject.h" // _Py_find_first_nonascii()
 
 #include "../lexer/state.h"
 
@@ -257,14 +258,14 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *
    the sequence if yes, 0 if not.  The special cases match
    those in stringlib/codecs.h:utf8_decode.
 */
-static int
-valid_utf8(const unsigned char* s)
+static Py_ssize_t
+valid_utf8(const unsigned char *s, const unsigned char *end)
 {
     int expected = 0;
     int length;
     if (*s < 0x80) {
-        /* single-byte code */
-        return 1;
+        /* as many ASCII chars as we can find in a row */
+        return _Py_find_first_nonascii(s, end);
     }
     else if (*s < 0xE0) {
         /* \xC2\x80-\xDF\xBF -- 0080-07FF */
@@ -321,19 +322,22 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
 {
     const char *badchar = NULL;
     const char *c;
-    int length;
-    const char *line_start = line;
+    const unsigned char *end = (const unsigned char *)line + strlen(line);
+    Py_ssize_t length;
     for (c = line; *c; c += length) {
-        if (!(length = valid_utf8((const unsigned char *)c))) {
+        if (!(length = valid_utf8((const unsigned char *)c, end))) {
             badchar = c;
             break;
         }
-        if (*c == '\n') {
-            lineno++;
-            line_start = c + 1;
-        }
     }
     if (badchar) {
+        const char *line_start = line;
+        for (c = line; c < badchar; c++) {
+            if (*c == '\n') {
+                lineno++;
+                line_start = c + 1;
+            }
+        }
         _PyTokenizer_syntaxerror_at(
                 tok, line_start, badchar - line_start + 1, lineno,
                 -1, -1,