Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,36 @@ because it turns other people's test suites red.

### Fixed

- **A size with a decimal point is read to the byte.** A fraction finer than
the arithmetic behind it was rounded without a word, so
`--size 255.99999999999999B` made a file of 256 B and `9007199254740993.0B`
planned one a byte smaller than asked. Such a size is now refused with the
two whole numbers around it, wherever a size is written - flags, recipes,
presets and the window. A size that lands on a whole byte gives the same
file as before.
- **`upload-validation` and `size-boundaries` refuse a limit too large for
their set.** The file far over the limit is the limit times `--far-over`,
and nothing asked whether that fitted, so a limit of 1844674407370955264B at
10x made a file of 1024 B the manifest still expected to be refused. A limit
with no room above it got advice to raise it in `upload-validation` and
ended with code 1 in `size-boundaries`. Both presets now refuse such a limit
with code 4 and say how large it can be.
- **An EML message above 1 EiB is refused at once.** With its text in base64,
a message near the largest size there is never finished planning, and in
the other encodings it was planned at 9 EiB, past what the format counts
in. Above 1152921504606846976 B the format now refuses with code 4, a dry
run included.
- **`tabular-import` no longer promises that a spreadsheet wider than 16384
columns is read whole.** A spreadsheet shows 16384 columns and drops the
rest without a word, and the set still expected such a workbook to be
accepted, certain. Past that width `sheet.xlsx` is now `unspecified` for
`count_limit`, like the wide CSV beside it. Up to it, nothing changes.
- **A picture too low for its label no longer says it carries one.** The
label is drawn as large as the width allows, so a low picture cuts it off,
and a BMP 512 px wide and 1 px tall said `label_embedded: true` with no
text in it at all. Every image format and both film formats now say
`false` with a `label_omitted` note when the text is not whole in the
picture. The pixels are the same as before.
- **`tfg cleanup --yes` keeps a file that changed after it was checked.**
Cleanup checks every file before it removes any, and a file changed in that
time was removed without `--force`. It now looks again just before it
Expand Down
104 changes: 80 additions & 24 deletions internal/core/size.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package core

import (
"math"
"math/big"
"strconv"
"strings"
)
Expand Down Expand Up @@ -65,8 +66,9 @@ func ParseSize(s string) (int64, error) {
return n * multiplier, nil
}

// Only the spellings a person writes. ParseFloat also takes exponents, hex
// floats and the words for infinity, and none of those describes a file.
// Only the spellings a person writes. ParseFloat, which read these until
// 2026-10-27, also took exponents, hex floats and the words for infinity,
// and none of those describes a file.
//
// Measured on 2026-08-03: "--size 1e5" quietly meant 100000 bytes, while
// the recipe refuses "1_000" and "0x10" outright, with the reasoning that
Expand All @@ -80,34 +82,82 @@ func ParseSize(s string) (int64, error) {
return 0, RefuseAbout(SettingSize, Says("core.SizeNotPlain",
"{setting} %q is not a plain number: write the digits out, as 10mb, 1.5gib or 1048576. Spellings such as 1e5, 0x10 or 1_000 are refused rather than guessed at, the same as in a recipe", A("Value", s)))
}
return parseDecimal(s, digits, multiplier)
}

f, err := strconv.ParseFloat(digits, 64)
if err != nil {
return 0, RefuseAbout(SettingSize, Says("core.SizeNotANumber", "{setting} %q is not a number: write something like 10mb, 1.5gib or 1048576", A("Value", s)))
}
if f < 0 {
// parseDecimal is ParseSize for a number with a decimal point, s being what
// was written and digits its number, already known to be plain.
func parseDecimal(s, digits string, multiplier int64) (int64, error) {
negative, whole, fraction := splitDecimal(digits)
if negative && (whole != "" || fraction != "") {
return 0, sizeNegative(s)
}

exact := f * float64(multiplier)
// The boundary itself has to go, not just what is above it. A float64
// cannot hold MaxInt64 exactly - the nearest value it has is 2^63, one past
// the largest int64 - so "greater than MaxInt64" compares 2^63 against 2^63
// and is false, and the conversion below then wraps to a negative size.
//
// Found by fuzzing on 2026-08-02, not by reading: "9223372036854775808" was
// accepted and came back as -9223372036854775808. ParseInt refuses it for
// being out of range, so it reaches the float path, which does not.
if exact >= float64(math.MaxInt64) {
// Twenty digits before the point is ten to the nineteenth at the least,
// already past the largest int64 before any unit multiplies it.
if len(whole) > 19 {
return 0, sizeTooLarge(s)
}
n := int64(exact)
if exact != math.Trunc(exact) {
n, exact := decimalBytes(whole, fraction, multiplier)
if !n.IsInt64() || (!exact && n.Int64() == math.MaxInt64) {
return 0, sizeTooLarge(s)
}
if !exact {
return 0, RefuseAbout(SettingSize, Says("core.SizeNotWhole",
"{setting} %q is not a whole number of bytes: it works out to %.4f bytes. Ask for %d or %d instead",
A("Value", s), A("Exact", exact), A("Below", n), A("Above", n+1)))
"{setting} %q is not a whole number of bytes: it falls between %d and %d. Ask for one of the two instead",
A("Value", s), A("Below", n.Int64()), A("Above", n.Int64()+1)))
}
return n, nil
return n.Int64(), nil
}

// splitDecimal takes a plain decimal apart into its sign, the digits before the
// point without the zeros that lead them and the digits after it without the
// zeros that end them - so a part that is nought is empty.
func splitDecimal(digits string) (negative bool, whole, fraction string) {
negative = strings.HasPrefix(digits, "-")
digits = strings.TrimPrefix(strings.TrimPrefix(digits, "+"), "-")
whole, fraction, _ = strings.Cut(digits, ".")
return negative, strings.TrimLeft(whole, "0"), strings.TrimRight(fraction, "0")
}

// fractionDigits is how many digits after the point decimalBytes reads.
//
// A fraction lands on a whole byte only when the unit cancels its
// denominator, and the largest unit is 2^40, which cancels 2^40 of the 10^k
// and none of the 5^k - so the fraction has to be a multiple of 5^k with k at
// most 40 once its trailing zeros are gone. A longer one is never whole, and
// only the byte below it is still asked for. That byte is decided by the
// first sixty digits: what they leave over is a multiple of the unit short of
// the next byte, and every digit after them adds less than the unit times
// 10^-60. Reading no further keeps the cost the same for a fraction of sixty
// digits and for one of a million, which is what an argument or a recipe
// written to be hostile can send (untouchable rule 23).
const fractionDigits = 60

// decimalBytes is whole.fraction times the unit, to the byte below when it is
// not a whole number of bytes, and whether it is.
//
// Until 2026-10-27 this went through float64, which carries 53 bits, and a
// fraction too fine for it was rounded before anyone asked whether it was
// whole: 255.99999999999999B became 256 B, 1.0000000000000001kb 1024 B and
// 9007199254740993.0B a byte smaller than asked, each without a word (D6).
// Exact arithmetic on the digits is the only way the question is answered
// about the number somebody wrote rather than about its nearest float.
func decimalBytes(whole, fraction string, unit int64) (*big.Int, bool) {
long := len(fraction) > fractionDigits
if long {
fraction = fraction[:fractionDigits]
}
scale := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(len(fraction))), nil)
// Both parts are digits only - plainDecimal let nothing else through - and
// both are empty when the number is nought.
n := new(big.Int)
if both := whole + fraction; both != "" {
n.SetString(both, 10)
}
n.Mul(n, big.NewInt(unit))
rest := new(big.Int)
n.QuoRem(n, scale, rest)
return n, !long && rest.Sign() == 0
}

// sizeNegative and sizeTooLarge are two refusals ParseSize gives from two
Expand All @@ -125,8 +175,14 @@ func sizeTooLarge(s string) error {
//
// Leading zeros are allowed and mean nothing, which matches how the recipe
// reads a number: 010 is ten because that is what it looks like.
//
// One sign, not one of each. "+-5.5" passed here until 2026-10-27 and was
// refused only because the float parser behind it refused it - and the digits
// are read by splitDecimal now, which would have taken it for 5.5.
func plainDecimal(s string) bool {
s = strings.TrimPrefix(strings.TrimPrefix(s, "+"), "-")
if s != "" && (s[0] == '+' || s[0] == '-') {
s = s[1:]
}
if s == "" {
return false
}
Expand Down
2 changes: 2 additions & 0 deletions internal/format/avif/avif.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import (
"github.com/donislawdev/TestingFilesGenerator/internal/core"
"github.com/donislawdev/TestingFilesGenerator/internal/format"
"github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim"
"github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel"
"github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff"
)

Expand Down Expand Up @@ -204,6 +205,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", m.width)),
})
}
carries, p.Notes = imagelabel.Claim(carries, m.width, m.height, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = carries
p.Memo = m
return p, nil
Expand Down
7 changes: 4 additions & 3 deletions internal/format/avif/picture.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ func picture(m memo) *image.RGBA {
return img
}

// labelled says whether this picture carries a readable label, and is asked
// both while planning, to report a file that will not have one, and while
// drawing. One answer, so the manifest cannot claim a label the picture lacks.
// labelled says whether this picture draws its label, and is asked both while
// planning and while drawing, so the two cannot disagree about it. Whether the
// label drawn can be read is imagelabel.Claim, asked while planning: a low
// picture draws it and cuts it off, and the manifest says it carries none.
func labelled(width int, label string) bool {
return label != "" && imagelabel.Fits(width, len(label))
}
1 change: 1 addition & 0 deletions internal/format/bmp/bmp.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)),
})
}
labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = labelled
p.Memo = m
return p, nil
Expand Down
25 changes: 25 additions & 0 deletions internal/format/eml/eml.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ func (g generator) Plan(r format.Request) (format.Plan, error) {
// PlanContext is Plan, stoppable between one file the message carries and the
// next.
func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) {
if !r.SizeFromContents && r.Bytes > maxMessage {
return format.Plan{}, aboveLargest(r.Bytes)
}
s, err := parse(r.Properties)
if err != nil {
return format.Plan{}, err
Expand Down Expand Up @@ -102,6 +105,28 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan
return p, nil
}

// aboveLargest refuses a message asked for above the largest this format
// counts in - before the files it carries are planned, so the answer is about
// the size asked and comes at once whatever is attached.
//
// Until 2026-10-27 only the attachments were held to the ceiling, and a text
// in base64 of 9223372036854775807B or one byte less never came back: near the
// top of int64 the length of the encoded words wraps round, the search in
// shareBase64 is handed a size that does not grow with them, and what it
// leaves goes to empty lines, billions of them, one at a time
// (docs/STABILITY-REVIEW-2026-10-26.md, R15). The search itself is left as it
// is, because below this ceiling nothing in it can wrap. A message sized from
// what it carries is not asked: its attachments have the ceiling already.
func aboveLargest(asked int64) error {
return &format.AboveMaximumError{
Format: "EML",
Requested: asked,
Maximum: maxMessage,
Reason: core.Says("eml.AMessageThisLargeCannotBe", "a message this large cannot be counted to the byte in this build"),
Hint: core.Says("eml.AskForBOrLess", "Ask for %d B or less.", core.A("Most", maxMessage)),
}
}

// fit makes room for the label in a message of the size asked for, or says
// why the size is too small. A label that does not fit is left out with a
// note, as txt leaves it out - the minimum is the message without it.
Expand Down
1 change: 1 addition & 0 deletions internal/format/gif/gif.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)),
})
}
labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = labelled
p.Memo = m
return p, nil
Expand Down
1 change: 1 addition & 0 deletions internal/format/ico/ico.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.TheIconIsPxWideAnd", "The icon is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("W", w)),
})
}
labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = labelled
p.Memo = m
return p, nil
Expand Down
48 changes: 46 additions & 2 deletions internal/format/imagelabel/imagelabel.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
// Package imagelabel draws the self describing label into pixels.
//
// Seven image formats burn the same label, so they share one rasteriser
// rather than carrying seven copies of it.
// The image formats and the films burn the same label, so they share one
// rasteriser rather than carrying a copy each.
package imagelabel

import (
"image"
"image/color"
"image/draw"

"github.com/donislawdev/TestingFilesGenerator/internal/core"
"github.com/donislawdev/TestingFilesGenerator/internal/format"
)

// Padding around the text inside its band, in scaled pixels.
Expand Down Expand Up @@ -88,10 +91,51 @@ func drawGlyph(img draw.Image, g string, x, y, scale int, ink color.Color) {
// Fits says whether a label would be drawn at all at this width. The caller
// needs to know before planning, so that a picture too small to carry one is
// reported rather than quietly unlabelled.
//
// It decides whether Draw is called, and only that. Whether the label can be
// READ is Readable, because a label is drawn by its width and can still be cut
// off at the foot of a low picture.
func Fits(width, chars int) bool {
return scaleFor(width, chars) > 0
}

// Readable says whether a label drawn across width is whole in a picture of
// this height - every row of every glyph inside it. The padding below the text
// may be cut, the text itself may not.
//
// What the manifest says of a picture is this, not Fits. Until 2026-10-27 it
// was Fits, and a picture 512 px wide and 1 px tall said it carried its label
// with not one pixel of text in it (docs/STABILITY-REVIEW-2026-10-26.md, R14).
// Draw is unchanged, so the pixels of such a picture are what they were - the
// owner's decision, because they are bytes somebody's suite has a hash of
// (D11) - and only what is said about them moved.
func Readable(width, height, chars int) bool {
return Fits(width, chars) && height >= textHeight(width, chars)
}

// Claim is what a plan says about a label: drawn is whether it is drawn at
// all, which the caller decides with Fits and reports in its own words when it
// is not. A label drawn and cut off at the foot of the picture is not carried
// either, and the note saying so goes after the notes given.
//
// One place for the eleven formats that draw the label, so the sentence and
// the answer cannot come apart between them.
func Claim(drawn bool, width, height, chars int, notes []format.Note) (bool, []format.Note) {
if !drawn || Readable(width, height, chars) {
return drawn, notes
}
return false, append(notes, format.Note{
Code: "label_omitted",
Detail: core.Says("imagelabel.ThePictureIsPxTallAnd", "The picture is %d px tall and the label needs %d px, so its text is cut off and this file carries no readable label. Its name and the manifest still identify it.",
core.A("Height", height), core.A("Needs", textHeight(width, chars))),
})
}

// textHeight is how many rows from the top the text of a label reaches.
func textHeight(width, chars int) int {
return pad + glyphHeight*scaleFor(width, chars)
}

// Cells is where Draw puts the characters of a text: character k covers the
// columns from First + k*Step, Width of them, and every row of the band. A
// film asks it to know which characters of its clock fall in which part of
Expand Down
4 changes: 3 additions & 1 deletion internal/format/jpg/jpg.go
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)),
})
}
p.Properties[format.PropertyLabelEmbedded] = r.Label && imagelabel.Fits(w, len(label))
labelled := r.Label && imagelabel.Fits(w, len(label))
labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = labelled
p.Memo = m
return p, nil
}
Expand Down
2 changes: 2 additions & 0 deletions internal/format/jxl/jxl.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import (
"github.com/donislawdev/TestingFilesGenerator/internal/core"
"github.com/donislawdev/TestingFilesGenerator/internal/format"
"github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim"
"github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel"
"github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff"
)

Expand Down Expand Up @@ -250,6 +251,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", m.width)),
})
}
carries, p.Notes = imagelabel.Claim(carries, m.width, m.height, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = carries
p.Memo = m
return p, nil
Expand Down
7 changes: 4 additions & 3 deletions internal/format/jxl/picture.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ func picture(m memo) *image.RGBA {
return img
}

// labelled says whether this picture carries a readable label, and is asked
// both while planning, to report a file that will not have one, and while
// drawing. One answer, so the manifest cannot claim a label the picture lacks.
// labelled says whether this picture draws its label, and is asked both while
// planning and while drawing, so the two cannot disagree about it. Whether the
// label drawn can be read is imagelabel.Claim, asked while planning: a low
// picture draws it and cuts it off, and the manifest says it carries none.
func labelled(width int, label string) bool {
return label != "" && imagelabel.Fits(width, len(label))
}
4 changes: 3 additions & 1 deletion internal/format/png/png.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)),
})
}
p.Properties[format.PropertyLabelEmbedded] = r.Label && imagelabel.Fits(w, len(label))
labelled := r.Label && imagelabel.Fits(w, len(label))
labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)
p.Properties[format.PropertyLabelEmbedded] = labelled
p.Memo = m
return p, nil
}
Expand Down
Loading
Loading