[preciz]

Assisted-by: Antigravity CLI:Gemini 4 Argon

Faster and uses less memory.

• Track byte length and return the original binary directly (or slice via binary_part/3 when trailing whitespace is present) instead of appending byte-by-byte into a writable binary for unquoted tokens.
• Initialize the binary match context in do_token/2 so unquoted_token/3 and quoted_token/2 resume the match context without per-iteration header checks.
• Fold opening quote handling (len == 0) and empty binary rejection (len != 0) into unquoted_token/3 to avoid upfront empty-binary checks and first-byte backtracking.
• Split multi-byte escape and CRLF patterns (<<?\, rest::binary>> and <<?\r, rest::binary>>) and move <<>> base clauses last so the compiler does not emit per-byte bs_get_position/bs_set_position instructions.

Bench:

Mix.install([:benchee])

defmodule OldToken do
  @space [?\s, ?\t]
  @specials ~c|()<>@,;:\\"/[]?={}|

  def token(""), do: false
  def token(<<?", quoted::binary>>), do: quoted_token(quoted, "")
  def token(token), do: unquoted_token(token, "")

  defp quoted_token(<<>>, _acc), do: false
  defp quoted_token(<<?", t::binary>>, acc), do: strip_spaces(t) == "" and acc
  defp quoted_token(<<?\\, h, t::binary>>, acc), do: quoted_token(t, <<acc::binary, h>>)
  defp quoted_token(<<h, t::binary>>, acc), do: quoted_token(t, <<acc::binary, h>>)

  defp unquoted_token(<<>>, acc), do: acc
  defp unquoted_token("\r\n" <> t, acc), do: strip_spaces(t) == "" and acc
  defp unquoted_token(<<h, t::binary>>, acc) when h in @space, do: strip_spaces(t) == "" and acc

  defp unquoted_token(<<h, _::binary>>, _acc) when h in @specials or h < 32 or h === 127,
    do: false

  defp unquoted_token(<<h, t::binary>>, acc), do: unquoted_token(t, <<acc::binary, h>>)

  defp strip_spaces("\r\n" <> t), do: strip_spaces(t)
  defp strip_spaces(<<h, t::binary>>) when h in [?\s, ?\t], do: strip_spaces(t)
  defp strip_spaces(t), do: trim_trailing(t)

  defp trim_trailing(binary), do: trim_trailing(binary, byte_size(binary))

  defp trim_trailing(binary, pos) do
    if pos > 0 and :binary.at(binary, pos - 1) in [?\s, ?\t] do
      trim_trailing(binary, pos - 1)
    else
      binary_part(binary, 0, pos)
    end
  end
end

defmodule NewToken do
  @space [?\s, ?\t]
  @specials ~c|()<>@,;:\\"/[]?={}|

  def token(token) when is_binary(token), do: do_token(token, token)

  defp do_token(<<token::binary>>, original), do: unquoted_token(token, original, 0)

  defp quoted_token(<<?", t::binary>>, acc), do: strip_spaces(t) == "" and acc

  defp quoted_token(<<?\\, rest::binary>>, acc) do
    case rest do
      <<h, t::binary>> -> quoted_token(t, <<acc::binary, h>>)
      <<>> -> false
    end
  end

  defp quoted_token(<<h, t::binary>>, acc), do: quoted_token(t, <<acc::binary, h>>)
  defp quoted_token(<<>>, _acc), do: false

  defp unquoted_token(<<?", quoted::binary>>, _token, len),
    do: len == 0 and quoted_token(quoted, "")

  defp unquoted_token(<<?\r, rest::binary>>, token, len) do
    case rest do
      <<?\n, t::binary>> -> strip_spaces(t) == "" and binary_part(token, 0, len)
      _ -> false
    end
  end

  defp unquoted_token(<<h, t::binary>>, token, len) when h in @space,
    do: strip_spaces(t) == "" and binary_part(token, 0, len)

  defp unquoted_token(<<h, _::binary>>, _token, _len) when h in @specials or h < 32 or h === 127,
    do: false

  defp unquoted_token(<<_, t::binary>>, token, len),
    do: unquoted_token(t, token, len + 1)

  defp unquoted_token(<<>>, token, len), do: len != 0 and token

  defp strip_spaces("\r\n" <> t), do: strip_spaces(t)
  defp strip_spaces(<<h, t::binary>>) when h in [?\s, ?\t], do: strip_spaces(t)
  defp strip_spaces(t), do: trim_trailing(t)

  defp trim_trailing(binary), do: trim_trailing(binary, byte_size(binary))

  defp trim_trailing(binary, pos) do
    if pos > 0 and :binary.at(binary, pos - 1) in [?\s, ?\t] do
      trim_trailing(binary, pos - 1)
    else
      binary_part(binary, 0, pos)
    end
  end
end

inputs = %{
  "1. empty" => "",
  "2. short unquoted (utf-8)" => "utf-8",
  "3. medium unquoted (multipart boundary, 60B)" =>
    "----WebKitFormBoundary7MA4YWxkTrZu0gW1234567890abcdef123456",
  "4. long unquoted (250B)" => String.duplicate("a1b2c3d4-", 28),
  "5. unquoted with trailing whitespace" =>
    "----WebKitFormBoundary7MA4YWxkTrZu0gW   \t  ",
  "6. unquoted with trailing CRLF + spaces" => "utf-8\r\n  ",
  "7. quoted token" => ~s["----WebKitFormBoundary7MA4YWxkTrZu0gW"],
  "8. quoted token with escapes" => ~S["<f\oo>\"<b\ar>\"<b\az>"],
  "9. invalid unquoted (late special char)" =>
    String.duplicate("abcdef", 10) <> "@invalid"
}

Benchee.run(
  %{
    "old" => fn input -> OldToken.token(input) end,
    "new" => fn input -> NewToken.token(input) end
  },
  inputs: inputs,
  pre_check: :all_same,
  warmup: 1,
  time: 3,
  memory_time: 2
)

Results:

Operating System: Linux
CPU Information: AMD Ryzen 7 8845HS w
Number of Available Cores: 16
Available memory: 54.72 GB
Elixir 1.20.4
Erlang 29.1.1
JIT enabled: true

Benchmark suite executing with the following configuration:
warmup: 1 s
time: 3 s
memory time: 2 s
reduction time: 0 ns
parallel: 1
inputs: 1. empty, 2. short unquoted (utf-8), 3. medium unquoted (multipart boundary, 60B), 4. long unquoted (250B), 5. unquoted with trailing whitespace, 6. unquoted with trailing CRLF + spaces, 7. quoted token, 8. quoted token with escapes, 9. invalid unquoted (late special char)
Estimated total run time: 1 min 48 s
Excluding outliers: false

##### With input 1. empty #####
Name           ips        average  deviation         median         99th %
new        34.90 M       28.65 ns ±10631.95%          20 ns          40 ns
old        27.65 M       36.17 ns ±10328.36%          30 ns          40 ns

Comparison: 
new        34.90 M
old        27.65 M - 1.26x slower +7.52 ns

Memory usage statistics:

Name    Memory usage
new             40 B
old             40 B - 1.00x memory usage +0 B

**All measurements for memory usage were the same**

##### With input 2. short unquoted (utf-8) #####
Name           ips        average  deviation         median         99th %
new        24.01 M       41.64 ns  ±6018.13%          40 ns          60 ns
old         9.14 M      109.44 ns  ±1292.92%         100 ns         141 ns

Comparison: 
new        24.01 M
old         9.14 M - 2.63x slower +67.80 ns

Memory usage statistics:

Name    Memory usage
new             40 B
old            104 B - 2.60x memory usage +64 B

**All measurements for memory usage were the same**

##### With input 3. medium unquoted (multipart boundary, 60B) #####
Name           ips        average  deviation         median         99th %
new         6.57 M      152.28 ns  ±1586.77%         141 ns         241 ns
old         1.89 M      528.59 ns   ±661.34%         511 ns         671 ns

Comparison: 
new         6.57 M
old         1.89 M - 3.47x slower +376.32 ns

Memory usage statistics:

Name    Memory usage
new             40 B
old            104 B - 2.60x memory usage +64 B

**All measurements for memory usage were the same**

##### With input 4. long unquoted (250B) #####
Name           ips        average  deviation         median         99th %
new         1.74 M        0.58 μs    ±53.65%        0.56 μs        0.83 μs
old         0.51 M        1.97 μs    ±48.64%        1.94 μs        2.42 μs

Comparison: 
new         1.74 M
old         0.51 M - 3.43x slower +1.40 μs

Memory usage statistics:

Name    Memory usage
new             40 B
old            104 B - 2.60x memory usage +64 B

**All measurements for memory usage were the same**

##### With input 5. unquoted with trailing whitespace #####
Name           ips        average  deviation         median         99th %
new         5.98 M      167.33 ns  ±2027.56%         150 ns         270 ns
old         2.38 M      420.59 ns   ±253.27%         401 ns         601 ns

Comparison: 
new         5.98 M
old         2.38 M - 2.51x slower +253.26 ns

Memory usage statistics:

Name    Memory usage
new             56 B
old            120 B - 2.14x memory usage +64 B

**All measurements for memory usage were the same**

##### With input 6. unquoted with trailing CRLF + spaces #####
Name           ips        average  deviation         median         99th %
new        11.24 M       89.00 ns  ±4243.02%          71 ns         160 ns
old         6.59 M      151.65 ns   ±810.88%         140 ns         260 ns

Comparison: 
new        11.24 M
old         6.59 M - 1.70x slower +62.65 ns

Memory usage statistics:

Name    Memory usage
new             56 B
old            120 B - 2.14x memory usage +64 B

**All measurements for memory usage were the same**

##### With input 7. quoted token #####
Name           ips        average  deviation         median         99th %
new         3.29 M      303.95 ns   ±299.26%         291 ns         441 ns
old         3.20 M      312.03 ns    ±90.33%         300 ns         450 ns

Comparison: 
new         3.29 M
old         3.20 M - 1.03x slower +8.09 ns

Memory usage statistics:

Name    Memory usage
new            120 B
old            120 B - 1.00x memory usage +0 B

**All measurements for memory usage were the same**

##### With input 8. quoted token with escapes #####
Name           ips        average  deviation         median         99th %
new         5.16 M      193.68 ns   ±128.45%         190 ns         311 ns
old         4.19 M      238.65 ns   ±408.26%         221 ns         361 ns

Comparison: 
new         5.16 M
old         4.19 M - 1.23x slower +44.96 ns

Memory usage statistics:

Name    Memory usage
new            120 B
old            120 B - 1.00x memory usage +0 B

**All measurements for memory usage were the same**

##### With input 9. invalid unquoted (late special char) #####
Name           ips        average  deviation         median         99th %
new         6.26 M      159.62 ns   ±139.40%         160 ns         260 ns
old         1.91 M      523.13 ns   ±109.65%         501 ns         651 ns

Comparison: 
new         6.26 M
old         1.91 M - 3.28x slower +363.51 ns

Memory usage statistics:

Name    Memory usage
new             40 B
old            104 B - 2.60x memory usage +64 B

**All measurements for memory usage were the same**