diff --git a/CHANGELOG.md b/CHANGELOG.md index e95c3a8c..7e13bd1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,36 @@ because it turns other people's test suites red. ### Fixed +- **A size with a decimal point is read to the byte.** A fraction finer than + the arithmetic behind it was rounded without a word, so + `--size 255.99999999999999B` made a file of 256 B and `9007199254740993.0B` + planned one a byte smaller than asked. Such a size is now refused with the + two whole numbers around it, wherever a size is written - flags, recipes, + presets and the window. A size that lands on a whole byte gives the same + file as before. +- **`upload-validation` and `size-boundaries` refuse a limit too large for + their set.** The file far over the limit is the limit times `--far-over`, + and nothing asked whether that fitted, so a limit of 1844674407370955264B at + 10x made a file of 1024 B the manifest still expected to be refused. A limit + with no room above it got advice to raise it in `upload-validation` and + ended with code 1 in `size-boundaries`. Both presets now refuse such a limit + with code 4 and say how large it can be. +- **An EML message above 1 EiB is refused at once.** With its text in base64, + a message near the largest size there is never finished planning, and in + the other encodings it was planned at 9 EiB, past what the format counts + in. Above 1152921504606846976 B the format now refuses with code 4, a dry + run included. +- **`tabular-import` no longer promises that a spreadsheet wider than 16384 + columns is read whole.** A spreadsheet shows 16384 columns and drops the + rest without a word, and the set still expected such a workbook to be + accepted, certain. Past that width `sheet.xlsx` is now `unspecified` for + `count_limit`, like the wide CSV beside it. Up to it, nothing changes. +- **A picture too low for its label no longer says it carries one.** The + label is drawn as large as the width allows, so a low picture cuts it off, + and a BMP 512 px wide and 1 px tall said `label_embedded: true` with no + text in it at all. Every image format and both film formats now say + `false` with a `label_omitted` note when the text is not whole in the + picture. The pixels are the same as before. - **`tfg cleanup --yes` keeps a file that changed after it was checked.** Cleanup checks every file before it removes any, and a file changed in that time was removed without `--force`. It now looks again just before it diff --git a/internal/core/size.go b/internal/core/size.go index ca6680d8..45c4cfd7 100644 --- a/internal/core/size.go +++ b/internal/core/size.go @@ -2,6 +2,7 @@ package core import ( "math" + "math/big" "strconv" "strings" ) @@ -65,8 +66,9 @@ func ParseSize(s string) (int64, error) { return n * multiplier, nil } - // Only the spellings a person writes. ParseFloat also takes exponents, hex - // floats and the words for infinity, and none of those describes a file. + // Only the spellings a person writes. ParseFloat, which read these until + // 2026-10-27, also took exponents, hex floats and the words for infinity, + // and none of those describes a file. // // Measured on 2026-08-03: "--size 1e5" quietly meant 100000 bytes, while // the recipe refuses "1_000" and "0x10" outright, with the reasoning that @@ -80,34 +82,82 @@ func ParseSize(s string) (int64, error) { return 0, RefuseAbout(SettingSize, Says("core.SizeNotPlain", "{setting} %q is not a plain number: write the digits out, as 10mb, 1.5gib or 1048576. Spellings such as 1e5, 0x10 or 1_000 are refused rather than guessed at, the same as in a recipe", A("Value", s))) } + return parseDecimal(s, digits, multiplier) +} - f, err := strconv.ParseFloat(digits, 64) - if err != nil { - return 0, RefuseAbout(SettingSize, Says("core.SizeNotANumber", "{setting} %q is not a number: write something like 10mb, 1.5gib or 1048576", A("Value", s))) - } - if f < 0 { +// parseDecimal is ParseSize for a number with a decimal point, s being what +// was written and digits its number, already known to be plain. +func parseDecimal(s, digits string, multiplier int64) (int64, error) { + negative, whole, fraction := splitDecimal(digits) + if negative && (whole != "" || fraction != "") { return 0, sizeNegative(s) } - - exact := f * float64(multiplier) - // The boundary itself has to go, not just what is above it. A float64 - // cannot hold MaxInt64 exactly - the nearest value it has is 2^63, one past - // the largest int64 - so "greater than MaxInt64" compares 2^63 against 2^63 - // and is false, and the conversion below then wraps to a negative size. - // - // Found by fuzzing on 2026-08-02, not by reading: "9223372036854775808" was - // accepted and came back as -9223372036854775808. ParseInt refuses it for - // being out of range, so it reaches the float path, which does not. - if exact >= float64(math.MaxInt64) { + // Twenty digits before the point is ten to the nineteenth at the least, + // already past the largest int64 before any unit multiplies it. + if len(whole) > 19 { return 0, sizeTooLarge(s) } - n := int64(exact) - if exact != math.Trunc(exact) { + n, exact := decimalBytes(whole, fraction, multiplier) + if !n.IsInt64() || (!exact && n.Int64() == math.MaxInt64) { + return 0, sizeTooLarge(s) + } + if !exact { return 0, RefuseAbout(SettingSize, Says("core.SizeNotWhole", - "{setting} %q is not a whole number of bytes: it works out to %.4f bytes. Ask for %d or %d instead", - A("Value", s), A("Exact", exact), A("Below", n), A("Above", n+1))) + "{setting} %q is not a whole number of bytes: it falls between %d and %d. Ask for one of the two instead", + A("Value", s), A("Below", n.Int64()), A("Above", n.Int64()+1))) } - return n, nil + return n.Int64(), nil +} + +// splitDecimal takes a plain decimal apart into its sign, the digits before the +// point without the zeros that lead them and the digits after it without the +// zeros that end them - so a part that is nought is empty. +func splitDecimal(digits string) (negative bool, whole, fraction string) { + negative = strings.HasPrefix(digits, "-") + digits = strings.TrimPrefix(strings.TrimPrefix(digits, "+"), "-") + whole, fraction, _ = strings.Cut(digits, ".") + return negative, strings.TrimLeft(whole, "0"), strings.TrimRight(fraction, "0") +} + +// fractionDigits is how many digits after the point decimalBytes reads. +// +// A fraction lands on a whole byte only when the unit cancels its +// denominator, and the largest unit is 2^40, which cancels 2^40 of the 10^k +// and none of the 5^k - so the fraction has to be a multiple of 5^k with k at +// most 40 once its trailing zeros are gone. A longer one is never whole, and +// only the byte below it is still asked for. That byte is decided by the +// first sixty digits: what they leave over is a multiple of the unit short of +// the next byte, and every digit after them adds less than the unit times +// 10^-60. Reading no further keeps the cost the same for a fraction of sixty +// digits and for one of a million, which is what an argument or a recipe +// written to be hostile can send (untouchable rule 23). +const fractionDigits = 60 + +// decimalBytes is whole.fraction times the unit, to the byte below when it is +// not a whole number of bytes, and whether it is. +// +// Until 2026-10-27 this went through float64, which carries 53 bits, and a +// fraction too fine for it was rounded before anyone asked whether it was +// whole: 255.99999999999999B became 256 B, 1.0000000000000001kb 1024 B and +// 9007199254740993.0B a byte smaller than asked, each without a word (D6). +// Exact arithmetic on the digits is the only way the question is answered +// about the number somebody wrote rather than about its nearest float. +func decimalBytes(whole, fraction string, unit int64) (*big.Int, bool) { + long := len(fraction) > fractionDigits + if long { + fraction = fraction[:fractionDigits] + } + scale := new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(len(fraction))), nil) + // Both parts are digits only - plainDecimal let nothing else through - and + // both are empty when the number is nought. + n := new(big.Int) + if both := whole + fraction; both != "" { + n.SetString(both, 10) + } + n.Mul(n, big.NewInt(unit)) + rest := new(big.Int) + n.QuoRem(n, scale, rest) + return n, !long && rest.Sign() == 0 } // sizeNegative and sizeTooLarge are two refusals ParseSize gives from two @@ -125,8 +175,14 @@ func sizeTooLarge(s string) error { // // Leading zeros are allowed and mean nothing, which matches how the recipe // reads a number: 010 is ten because that is what it looks like. +// +// One sign, not one of each. "+-5.5" passed here until 2026-10-27 and was +// refused only because the float parser behind it refused it - and the digits +// are read by splitDecimal now, which would have taken it for 5.5. func plainDecimal(s string) bool { - s = strings.TrimPrefix(strings.TrimPrefix(s, "+"), "-") + if s != "" && (s[0] == '+' || s[0] == '-') { + s = s[1:] + } if s == "" { return false } diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index dc1230e1..cb71e5c3 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -47,6 +47,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff" ) @@ -204,6 +205,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", m.width)), }) } + carries, p.Notes = imagelabel.Claim(carries, m.width, m.height, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = carries p.Memo = m return p, nil diff --git a/internal/format/avif/picture.go b/internal/format/avif/picture.go index 56d98e12..6856c379 100644 --- a/internal/format/avif/picture.go +++ b/internal/format/avif/picture.go @@ -37,9 +37,10 @@ func picture(m memo) *image.RGBA { return img } -// labelled says whether this picture carries a readable label, and is asked -// both while planning, to report a file that will not have one, and while -// drawing. One answer, so the manifest cannot claim a label the picture lacks. +// labelled says whether this picture draws its label, and is asked both while +// planning and while drawing, so the two cannot disagree about it. Whether the +// label drawn can be read is imagelabel.Claim, asked while planning: a low +// picture draws it and cuts it off, and the manifest says it carries none. func labelled(width int, label string) bool { return label != "" && imagelabel.Fits(width, len(label)) } diff --git a/internal/format/bmp/bmp.go b/internal/format/bmp/bmp.go index 69451a8d..ae27a640 100644 --- a/internal/format/bmp/bmp.go +++ b/internal/format/bmp/bmp.go @@ -171,6 +171,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil diff --git a/internal/format/eml/eml.go b/internal/format/eml/eml.go index 8e825ccd..b6ba2c66 100644 --- a/internal/format/eml/eml.go +++ b/internal/format/eml/eml.go @@ -72,6 +72,9 @@ func (g generator) Plan(r format.Request) (format.Plan, error) { // PlanContext is Plan, stoppable between one file the message carries and the // next. func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan, error) { + if !r.SizeFromContents && r.Bytes > maxMessage { + return format.Plan{}, aboveLargest(r.Bytes) + } s, err := parse(r.Properties) if err != nil { return format.Plan{}, err @@ -102,6 +105,28 @@ func (generator) PlanContext(ctx context.Context, r format.Request) (format.Plan return p, nil } +// aboveLargest refuses a message asked for above the largest this format +// counts in - before the files it carries are planned, so the answer is about +// the size asked and comes at once whatever is attached. +// +// Until 2026-10-27 only the attachments were held to the ceiling, and a text +// in base64 of 9223372036854775807B or one byte less never came back: near the +// top of int64 the length of the encoded words wraps round, the search in +// shareBase64 is handed a size that does not grow with them, and what it +// leaves goes to empty lines, billions of them, one at a time +// (docs/STABILITY-REVIEW-2026-10-26.md, R15). The search itself is left as it +// is, because below this ceiling nothing in it can wrap. A message sized from +// what it carries is not asked: its attachments have the ceiling already. +func aboveLargest(asked int64) error { + return &format.AboveMaximumError{ + Format: "EML", + Requested: asked, + Maximum: maxMessage, + Reason: core.Says("eml.AMessageThisLargeCannotBe", "a message this large cannot be counted to the byte in this build"), + Hint: core.Says("eml.AskForBOrLess", "Ask for %d B or less.", core.A("Most", maxMessage)), + } +} + // fit makes room for the label in a message of the size asked for, or says // why the size is too small. A label that does not fit is left out with a // note, as txt leaves it out - the minimum is the message without it. diff --git a/internal/format/gif/gif.go b/internal/format/gif/gif.go index 36404673..5b22f101 100644 --- a/internal/format/gif/gif.go +++ b/internal/format/gif/gif.go @@ -229,6 +229,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil diff --git a/internal/format/ico/ico.go b/internal/format/ico/ico.go index d61c5e63..8edf2c16 100644 --- a/internal/format/ico/ico.go +++ b/internal/format/ico/ico.go @@ -160,6 +160,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.TheIconIsPxWideAnd", "The icon is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("W", w)), }) } + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil diff --git a/internal/format/imagelabel/imagelabel.go b/internal/format/imagelabel/imagelabel.go index 159aa6c8..67fd17cc 100644 --- a/internal/format/imagelabel/imagelabel.go +++ b/internal/format/imagelabel/imagelabel.go @@ -1,13 +1,16 @@ // Package imagelabel draws the self describing label into pixels. // -// Seven image formats burn the same label, so they share one rasteriser -// rather than carrying seven copies of it. +// The image formats and the films burn the same label, so they share one +// rasteriser rather than carrying a copy each. package imagelabel import ( "image" "image/color" "image/draw" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" ) // Padding around the text inside its band, in scaled pixels. @@ -88,10 +91,51 @@ func drawGlyph(img draw.Image, g string, x, y, scale int, ink color.Color) { // Fits says whether a label would be drawn at all at this width. The caller // needs to know before planning, so that a picture too small to carry one is // reported rather than quietly unlabelled. +// +// It decides whether Draw is called, and only that. Whether the label can be +// READ is Readable, because a label is drawn by its width and can still be cut +// off at the foot of a low picture. func Fits(width, chars int) bool { return scaleFor(width, chars) > 0 } +// Readable says whether a label drawn across width is whole in a picture of +// this height - every row of every glyph inside it. The padding below the text +// may be cut, the text itself may not. +// +// What the manifest says of a picture is this, not Fits. Until 2026-10-27 it +// was Fits, and a picture 512 px wide and 1 px tall said it carried its label +// with not one pixel of text in it (docs/STABILITY-REVIEW-2026-10-26.md, R14). +// Draw is unchanged, so the pixels of such a picture are what they were - the +// owner's decision, because they are bytes somebody's suite has a hash of +// (D11) - and only what is said about them moved. +func Readable(width, height, chars int) bool { + return Fits(width, chars) && height >= textHeight(width, chars) +} + +// Claim is what a plan says about a label: drawn is whether it is drawn at +// all, which the caller decides with Fits and reports in its own words when it +// is not. A label drawn and cut off at the foot of the picture is not carried +// either, and the note saying so goes after the notes given. +// +// One place for the eleven formats that draw the label, so the sentence and +// the answer cannot come apart between them. +func Claim(drawn bool, width, height, chars int, notes []format.Note) (bool, []format.Note) { + if !drawn || Readable(width, height, chars) { + return drawn, notes + } + return false, append(notes, format.Note{ + Code: "label_omitted", + Detail: core.Says("imagelabel.ThePictureIsPxTallAnd", "The picture is %d px tall and the label needs %d px, so its text is cut off and this file carries no readable label. Its name and the manifest still identify it.", + core.A("Height", height), core.A("Needs", textHeight(width, chars))), + }) +} + +// textHeight is how many rows from the top the text of a label reaches. +func textHeight(width, chars int) int { + return pad + glyphHeight*scaleFor(width, chars) +} + // Cells is where Draw puts the characters of a text: character k covers the // columns from First + k*Step, Width of them, and every row of the band. A // film asks it to know which characters of its clock fall in which part of diff --git a/internal/format/jpg/jpg.go b/internal/format/jpg/jpg.go index 49c5e78f..edadddff 100644 --- a/internal/format/jpg/jpg.go +++ b/internal/format/jpg/jpg.go @@ -252,7 +252,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } - p.Properties[format.PropertyLabelEmbedded] = r.Label && imagelabel.Fits(w, len(label)) + labelled := r.Label && imagelabel.Fits(w, len(label)) + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) + p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil } diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index 0a84d63b..e2e963b1 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -61,6 +61,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" "github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff" ) @@ -250,6 +251,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", m.width)), }) } + carries, p.Notes = imagelabel.Claim(carries, m.width, m.height, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = carries p.Memo = m return p, nil diff --git a/internal/format/jxl/picture.go b/internal/format/jxl/picture.go index 1d9faf00..5ead09b4 100644 --- a/internal/format/jxl/picture.go +++ b/internal/format/jxl/picture.go @@ -37,9 +37,10 @@ func picture(m memo) *image.RGBA { return img } -// labelled says whether this picture carries a readable label, and is asked -// both while planning, to report a file that will not have one, and while -// drawing. One answer, so the manifest cannot claim a label the picture lacks. +// labelled says whether this picture draws its label, and is asked both while +// planning and while drawing, so the two cannot disagree about it. Whether the +// label drawn can be read is imagelabel.Claim, asked while planning: a low +// picture draws it and cuts it off, and the manifest says it carries none. func labelled(width int, label string) bool { return label != "" && imagelabel.Fits(width, len(label)) } diff --git a/internal/format/png/png.go b/internal/format/png/png.go index bf2ac5cc..68d439a4 100644 --- a/internal/format/png/png.go +++ b/internal/format/png/png.go @@ -244,7 +244,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } - p.Properties[format.PropertyLabelEmbedded] = r.Label && imagelabel.Fits(w, len(label)) + labelled := r.Label && imagelabel.Fits(w, len(label)) + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) + p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil } diff --git a/internal/format/tiff/tiff.go b/internal/format/tiff/tiff.go index 30a36cdf..0c0f0f94 100644 --- a/internal/format/tiff/tiff.go +++ b/internal/format/tiff/tiff.go @@ -212,6 +212,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index 8dad90b8..42642a84 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -6,6 +6,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagedim" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" ) // Rung is a picture size planning can choose without coding the picture, with @@ -72,8 +73,14 @@ type Choice struct { Named bool } -// Labelled is whether this picture carries its label. -func (c Choice) Labelled() bool { return Labelled(c.Width, c.Label) } +// Labelled is whether this picture carries its label where it can be read: +// drawn across its width, and whole in its height. The manifest says this, +// and the drawing asks the package's Labelled, which is the width alone - a +// low film draws its label and cuts it off (R14 of +// docs/STABILITY-REVIEW-2026-10-26.md, where the pixels stayed as they were). +func (c Choice) Labelled() bool { + return Labelled(c.Width, c.Label) && imagelabel.Readable(c.Width, c.Height, len(c.Label)) +} // ClockShown is whether this film's pictures show the clock. func (c Choice) ClockShown(t Timeline) bool { return ClockShown(c.Width, c.Height, c.Label, t) } diff --git a/internal/format/video/picture.go b/internal/format/video/picture.go index e2eded53..4073a421 100644 --- a/internal/format/video/picture.go +++ b/internal/format/video/picture.go @@ -60,9 +60,9 @@ func WholePicture(width, height int, seed uint64, label string, t Timeline, c in return f.whole(f.lookOf(c)) } -// Labelled says whether a picture this wide carries a readable label - asked -// while planning, to report a film without one, and while drawing, so the -// manifest cannot claim a label the picture lacks. +// Labelled says whether a picture this wide draws its label - asked while +// planning and while drawing, so the two cannot disagree about where the clock +// goes. Whether the label drawn can be read is Choice.Labelled. func Labelled(width int, label string) bool { return label != "" && imagelabel.Fits(width, len(label)) } diff --git a/internal/format/video/plan.go b/internal/format/video/plan.go index 96294978..ed6b44b6 100644 --- a/internal/format/video/plan.go +++ b/internal/format/video/plan.go @@ -3,6 +3,7 @@ package video import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" ) // Film is what planning settles for a film and writing makes it from: the @@ -52,12 +53,15 @@ func Plan(formatID string, r format.Request, need func(Stream) int64, refuse fun Memo: Film{Choice: c, Settings: s}, Work: st.Work(), } - if r.Label && !c.Labelled() { + if r.Label && !Labelled(c.Width, c.Label) { p.Notes = append(p.Notes, format.Note{ Code: "label_omitted", Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", c.Width)), }) } + // The answer is c.Labelled, which Facts has already put in the plan. This + // is the note a label drawn and cut off at the foot of the picture owes. + _, p.Notes = imagelabel.Claim(r.Label && Labelled(c.Width, c.Label), c.Width, c.Height, len(c.Label), p.Notes) if !c.ClockShown(s.Timeline) { p.Notes = append(p.Notes, format.Note{ Code: "clock_omitted", diff --git a/internal/format/webp/webp.go b/internal/format/webp/webp.go index 0732e8f9..0ed89756 100644 --- a/internal/format/webp/webp.go +++ b/internal/format/webp/webp.go @@ -189,6 +189,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Detail: core.Says("format.ThePictureIsPxWideAnd", "The picture is %d px wide and the label needs more room, so this file carries no visible label. Its name and the manifest still identify it.", core.A("Width", w)), }) } + labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes) p.Properties[format.PropertyLabelEmbedded] = labelled p.Memo = m return p, nil diff --git a/internal/guard/eml_test.go b/internal/guard/eml_test.go index cbf99149..e852deab 100644 --- a/internal/guard/eml_test.go +++ b/internal/guard/eml_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "math" "os" "path/filepath" "sort" @@ -11,6 +12,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/eml" @@ -209,6 +211,58 @@ func TestEMLRefusesASizeBelowItsAttachmentsInBase64(t *testing.T) { } } +// A message has a largest size, the one the format counts in, and above it is +// refused at once with the four parts (D6) - in every encoding of the text. +// +// Until 2026-10-27 only the attachments were held to it. A text in base64 of +// 9223372036854775807B or one byte less never came back: the length of the +// encoded words wraps round near the top of int64, the search for how many +// words fit was handed a size that does not grow with them, and what was left +// went to empty lines, billions of them, one at a time +// (docs/STABILITY-REVIEW-2026-10-26.md, R15). The other encodings planned a +// message of 9 EiB that the format says it does not plan. +func TestAnEMLAboveTheLargestMessageIsRefusedAtOnce(t *testing.T) { + d, err := format.Get("eml") + if err != nil { + t.Fatal(err) + } + const largest = int64(1) << 60 + // The last carries files, three of 2^59 B that add up past the ceiling + // on their own: the refusal is still about the size asked, so it is the + // size asked that comes first (found by review on 2026-10-27). + for _, props := range []map[string]string{ + {"text_encoding": "base64"}, + {"text_encoding": "quoted-printable"}, + {"text_encoding": "7bit"}, + {"text_encoding": "base64", "attachments": "3", "attachment_format": "txt", "attachment_size": "576460752303423488b"}, + } { + encoding := props["text_encoding"] + " with " + props["attachments"] + " attachment(s)" + for _, size := range []int64{largest + 1, math.MaxInt64 - 1, math.MaxInt64} { + done := make(chan error, 1) + go func() { + _, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 1, Label: true, Properties: props}) + done <- err + }() + select { + case err := <-done: + var above *format.AboveMaximumError + if !errors.As(err, &above) || above.Maximum != largest || above.Requested != size || above.Hint.IsZero() { + t.Errorf("%s at %d B gave %v, expected a refusal of that size naming %d B as the largest", encoding, size, err, largest) + } + case <-time.After(5 * time.Second): + t.Errorf("%s at %d B had no answer after five seconds", encoding, size) + } + } + // The largest itself is a message the format plans. + if props["attachments"] != "" { + continue + } + if _, err := d.Generator.Plan(format.Request{Bytes: largest, Seed: 1, Label: true, Properties: props}); err != nil { + t.Errorf("%s at the largest message, %d B, was refused: %v", encoding, largest, err) + } + } +} + // alternative carries the same words in both parts, so its text grows two // bytes at a time, and an odd byte left over becomes one space in the HTML, // where it changes nothing a person sees. Two neighbouring sizes have to land diff --git a/internal/guard/generatorbytes_test.go b/internal/guard/generatorbytes_test.go index 7082a2ed..ce1734ce 100644 --- a/internal/guard/generatorbytes_test.go +++ b/internal/guard/generatorbytes_test.go @@ -395,7 +395,7 @@ func goldenCases() map[string]engine.Target { // have to agree to the byte. "targz_past_the_comment_limit": {ID: "g", Format: "targz", Sizes: engine.Uniform(1, 262144), Label: true}, } - for _, more := range []map[string]engine.Target{mapGoldenCases(), mailGoldenCases(), pageGoldenCases(), rtfGoldenCases()} { + for _, more := range []map[string]engine.Target{mapGoldenCases(), mailGoldenCases(), pageGoldenCases(), rtfGoldenCases(), cutLabelGoldenCases()} { for name, target := range more { cases[name] = target } diff --git a/internal/guard/generatorbyteslabel_test.go b/internal/guard/generatorbyteslabel_test.go new file mode 100644 index 00000000..3e9a3b50 --- /dev/null +++ b/internal/guard/generatorbyteslabel_test.go @@ -0,0 +1,41 @@ +package guard + +import "github.com/donislawdev/TestingFilesGenerator/internal/engine" + +// cutLabelGoldenCases are pictures too low for the whole of their label, one +// per format that draws it into pixels. +// +// Wide enough that the label is drawn, so the band and the top rows of the +// text are in the picture, and too low for the rest - four rows, where the +// text starts on the third. Until 2026-10-27 the manifest said such a picture +// carried its label. It now says it does not, and the owner's decision was +// that the pixels stay as they were (docs/STABILITY-REVIEW-2026-10-26.md +// section 4a.2). These values were measured on the code from before that +// change, so they are the proof that the manifest moved and the bytes did not. +// +// A film is cut at the same place by a different arithmetic - it is wide, so +// its label is drawn four times larger and needs 22 rows of its 16. +func cutLabelGoldenCases() map[string]engine.Target { + low := func(format, width, height string, size int64) engine.Target { + return engine.Target{ID: "g", Format: format, Sizes: engine.Uniform(1, size), Label: true, + Properties: map[string]string{"width": width, "height": height}} + } + film := func(format string) engine.Target { + t := low(format, "1920", "16", 65536) + t.Properties["duration"] = "1s" + return t + } + return map[string]engine.Target{ + "avif_label_cut": low("avif", "512", "4", 16384), + "bmp_label_cut": low("bmp", "512", "4", 16384), + "gif_label_cut": low("gif", "512", "4", 16384), + "ico_label_cut": low("ico", "256", "4", 16384), + "jpg_label_cut": low("jpg", "512", "4", 16384), + "jxl_label_cut": low("jxl", "512", "4", 16384), + "png_label_cut": low("png", "512", "4", 16384), + "tiff_label_cut": low("tiff", "512", "4", 16384), + "webp_label_cut": low("webp", "512", "4", 16384), + "mp4_label_cut": film("mp4"), + "webm_label_cut": film("webm"), + } +} diff --git a/internal/guard/labelclaim_test.go b/internal/guard/labelclaim_test.go new file mode 100644 index 00000000..a5322186 --- /dev/null +++ b/internal/guard/labelclaim_test.go @@ -0,0 +1,112 @@ +package guard + +import ( + "image" + "strconv" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" +) + +// A picture says it carries its label only when the text of the label is whole +// in it. +// +// The label is drawn at a scale chosen from the width, so a wide picture +// draws it large, and a low one cuts it off at its foot. Until 2026-10-27 the +// manifest asked only the width: a BMP 512 px wide and 1 px tall said +// label_embedded true with not one pixel of text in it, and so did a film +// 1920 px wide and 16 px tall whose label needs 22 - eleven formats in all +// (docs/STABILITY-REVIEW-2026-10-26.md, R14). The formats are read from the +// registry, so one added with a width and a height is asked too, and every +// one is asked at a height with room as well, so "never" cannot pass for +// "only when it is true". +func TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole(t *testing.T) { + asked := 0 + for _, id := range format.IDs() { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + width, wok := settingNamed(d, "width") + height, hok := settingNamed(d, "height") + if !wok || !hok { + continue + } + asked++ + // As wide as 1920 px where the format goes that far, which draws the + // label four times larger in the films and the pictures alike. + wide := min(width.Max, 1920) + claims := func(tall int64) (bool, []format.Note) { + t.Helper() + props := map[string]string{"width": strconv.FormatInt(wide, 10), "height": strconv.FormatInt(tall, 10)} + r := format.Request{Seed: 1, Label: true, Properties: props} + // Room in the bytes as well, so that a label left out is about + // the picture and not about the size - an SVG at its smallest has + // no bytes for one. + r.Bytes = format.SmallestRemembered(d, r) + 64<<10 + p, err := d.Generator.Plan(r) + if err != nil { + t.Fatalf("%s at %dx%d: %v", id, wide, tall, err) + } + embedded, ok := p.Properties[format.PropertyLabelEmbedded].(bool) + if !ok { + t.Fatalf("%s at %dx%d has no label_embedded, or not a yes or no: %v", id, wide, tall, p.Properties[format.PropertyLabelEmbedded]) + } + return embedded, p.Notes + } + + if embedded, notes := claims(height.Min); embedded || !hasNote(notes, "label_omitted") { + t.Errorf("%s at %dx%d says label_embedded %v with notes %v - its label cannot be whole in %d px", + id, wide, height.Min, embedded, notes, height.Min) + } + if embedded, notes := claims(max(height.Min, 64)); !embedded { + t.Errorf("%s at %dx%d says it carries no label (%v) - there is room for one", id, wide, max(height.Min, 64), notes) + } + } + if asked < 11 { + t.Fatalf("only %d format(s) declare a width and a height - the registry was not read", asked) + } +} + +// A label is readable from the first height that holds every row its text is +// drawn in, and not one row before - asked of the pixels Draw puts down rather +// than of the arithmetic beside it, at each of the four scales a label is +// drawn at. The guard above asks the lowest height and one with room, and an +// answer one row out at the boundary passed it (found by review, 2026-10-27). +func TestALabelIsReadableFromTheFirstHeightThatHoldsItsText(t *testing.T) { + label := core.Label("bmp", 2048, 1) + across := len(label)*4 + 4 // a glyph and its gap is four columns at scale 1, with the padding either side + for scale := 1; scale <= 4; scale++ { + width := across * scale + img := image.NewRGBA(image.Rect(0, 0, width, 64)) + imagelabel.Draw(img, label) + band := img.RGBAAt(0, 0) + last := -1 + for y := 0; y < 64; y++ { + for x := 0; x < width; x++ { + if c := img.RGBAAt(x, y); c != band && c.A != 0 { + last = y + } + } + } + if last < 0 { + t.Fatalf("no text was drawn at %d px across - the guard did not reach a drawn label", width) + } + holds := last + 1 + if imagelabel.Readable(width, holds-1, len(label)) || !imagelabel.Readable(width, holds, len(label)) { + t.Errorf("at %d px across the text is drawn in the top %d rows, and Readable says %v at %d rows and %v at %d", + width, holds, imagelabel.Readable(width, holds-1, len(label)), holds-1, imagelabel.Readable(width, holds, len(label)), holds) + } + } +} + +func hasNote(notes []format.Note, code string) bool { + for _, n := range notes { + if n.Code == code { + return true + } + } + return false +} diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 16adc19d..5c8988dc 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -3068,10 +3068,15 @@ # turning one of them into txt left md pinned by the other two and # the guard green - a mutation of one of two layers moves nothing. # pdf has one case, measured by counting Format: fields in the file. + # + # The same happened to png on 2026-10-27: a second png case pinning a + # picture whose label is cut off left png pinned after the mutation, and + # CI reported this entry as a HOLE. svg has one case, counted the same + # way across goldenCases and the maps it merges in. "a format quietly stops having its bytes pinned anywhere", "internal/guard/generatorbytes_test.go", - "\"png_64kib\": {ID: \"g\", Format: \"png\",", - "\"png_64kib\": {ID: \"g\", Format: \"txt\",", + "\"svg_8kib\": {ID: \"g\", Format: \"svg\",", + "\"svg_8kib\": {ID: \"g\", Format: \"txt\",", "TestEveryFormatHasAGoldenValue", ), ( @@ -3728,8 +3733,8 @@ # undone when the generate screen's refusals were moved. O73. "a preset refusal stops saying which parameter it is about", "internal/preset/limitset.go", - 'Setting: s.setting,', - 'Setting: "",', + "\t\t\tSetting: s.setting,\n\t\t\tDetail: what,", + "\t\t\tSetting: \"\",\n\t\t\tDetail: what,", "TestARefusalAboutAPresetSettingAppearsUnderIt", ), ( @@ -6150,9 +6155,9 @@ ), ( "the size-boundaries preset stops refusing a limit with no number above it", - "internal/preset/sizeboundaries.go", - "\t\tif limit+o.bytes < limit {", - "\t\tif false && limit+o.bytes < limit {", + "internal/preset/limitset.go", + "if s.limit > math.MaxInt64-far.bytes {", + "if false && s.limit > math.MaxInt64-far.bytes {", "TestThePresetAnswersALimitTooLargeTheSameWayTheFlagDoes", ), ( @@ -15640,4 +15645,192 @@ "return 0, nil", "TestTheRecordARunIsToldBeforehandIsTheRecordItWrites", ), + # R17 (docs/STABILITY-REVIEW-2026-10-26.md): a size with a decimal point is + # read to the byte, not through a float, and a hostile one is answered + # without reading all of it. + ( + "a fraction longer than sixty digits is taken as whole when its first sixty are", + "internal/core/size.go", + "return n, !long && rest.Sign() == 0", + "return n, rest.Sign() == 0", + "TestASizeWithADecimalPointIsReadToTheByte", + ), + ( + "a fraction just under the largest size asks for a byte past it", + "internal/core/size.go", + "if !n.IsInt64() || (!exact && n.Int64() == math.MaxInt64) {", + "if !n.IsInt64() {", + "TestASizeWithADecimalPointIsReadToTheByte", + ), + ( + "four million digits before the point are read one by one", + "internal/core/size.go", + "if len(whole) > 19 {", + "if len(whole) > 19 && false {", + "TestASizeWithADecimalPointIsReadToTheByte", + ), + ( + "a fraction of four million digits is read to its end", + "internal/core/size.go", + "long := len(fraction) > fractionDigits", + "long := false", + "TestASizeWithADecimalPointIsReadToTheByte", + ), + ( + "a decimal size forgets its unit", + "internal/core/size.go", + "n.Mul(n, big.NewInt(unit))", + "n.Mul(n, big.NewInt(1))", + "TestEveryExactDecimalOfAUnitReadsBackAsItself", + ), + ( + "a negative size with a decimal point is taken as its size without the sign", + "internal/core/size.go", + "if negative && (whole != \"\" || fraction != \"\") {", + "if negative && false {", + "TestNonsenseSizesAreRefusedWithAUsefulMessage", + ), + ( + "a size with two signs is read as the number after them", + "internal/core/size.go", + "if s != \"\" && (s[0] == '+' || s[0] == '-') {\n\t\ts = s[1:]\n\t}", + "s = strings.TrimPrefix(strings.TrimPrefix(s, \"+\"), \"-\")", + "TestNonsenseSizesAreRefusedWithAUsefulMessage", + ), + # R13 (docs/STABILITY-REVIEW-2026-10-26.md): a set laid out around a limit + # is refused, not wrapped, when the limit has no room above it or the file + # far over it is past the largest size. + ( + "a set around a limit with no room above it wraps its step over the limit", + "internal/preset/limitset.go", + "if s.limit > math.MaxInt64-far.bytes {", + "if s.limit > math.MaxInt64-far.bytes && false {", + "TestEveryPresetAnswersTheLargestSizeAboutTheTop", + ), + ( + "a preset's refusal of a limit with no room above it stops being the one --boundary gives", + "internal/preset/limitset.go", + "Cause: core.ErrBoundaryTooLarge,", + "Cause: nil,", + "TestEveryPresetAnswersTheLargestSizeAboutTheTop", + ), + ( + "a preset's refusal hides the refusal it is", + "internal/preset/preset.go", + "func (e *ImpossibleError) Unwrap() error { return e.Cause }", + "func (e *ImpossibleError) Unwrap() error { return nil }", + "TestThePresetAnswersALimitTooLargeTheSameWayTheFlagDoes", + ), + ( + "the file far over the limit wraps round instead of being refused", + "internal/preset/uploadset.go", + "if s.farOver == 0 || s.limit <= math.MaxInt64/s.farOver {", + "if true {", + "TestTheFileFarOverTheLimitIsThatManyTimesTheLimitOrNothing", + ), + ( + "the largest limit the file far over it fits is refused as well", + "internal/preset/uploadset.go", + "if s.farOver == 0 || s.limit <= math.MaxInt64/s.farOver {", + "if s.farOver == 0 || s.limit < math.MaxInt64/s.farOver {", + "TestTheFileFarOverTheLimitIsThatManyTimesTheLimitOrNothing", + ), + # R15 (docs/STABILITY-REVIEW-2026-10-26.md): a message above the largest + # the format counts in is refused before anything is counted at its size. + ( + "a message above the largest the format counts in is planned, and in base64 never comes back", + "internal/format/eml/eml.go", + "if !r.SizeFromContents && r.Bytes > maxMessage {", + "if !r.SizeFromContents && r.Bytes > maxMessage && false {", + "TestAnEMLAboveTheLargestMessageIsRefusedAtOnce", + ), + ( + "the largest message itself is refused", + "internal/format/eml/eml.go", + "if !r.SizeFromContents && r.Bytes > maxMessage {", + "if !r.SizeFromContents && r.Bytes >= maxMessage {", + "TestAnEMLAboveTheLargestMessageIsRefusedAtOnce", + ), + # R16 (docs/STABILITY-REVIEW-2026-10-26.md): the spreadsheet is expected + # whole only as wide as a spreadsheet is. + ( + "a spreadsheet wider than a spreadsheet shows is still expected to be taken whole", + "internal/preset/tabularimport.go", + "if wide, _ := strconv.Atoi(columns); wide > spreadsheetColumns {", + "if wide, _ := strconv.Atoi(columns); wide > spreadsheetColumns && false {", + "TestTheSpreadsheetIsExpectedWholeOnlyAsWideAsASpreadsheet", + ), + ( + "a spreadsheet exactly as wide as a spreadsheet shows stops being expected whole", + "internal/preset/tabularimport.go", + "if wide, _ := strconv.Atoi(columns); wide > spreadsheetColumns {", + "if wide, _ := strconv.Atoi(columns); wide >= spreadsheetColumns {", + "TestTheSpreadsheetIsExpectedWholeOnlyAsWideAsASpreadsheet", + ), + # R14 (docs/STABILITY-REVIEW-2026-10-26.md): a picture says it carries its + # label only when the text of the label is whole in it. + ( + "a label cut off at the foot of a low picture is said to be carried", + "internal/format/imagelabel/imagelabel.go", + "if !drawn || Readable(width, height, chars) {", + "if true {", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + ( + "a label is called readable from its width alone", + "internal/format/imagelabel/imagelabel.go", + "return Fits(width, chars) && height >= textHeight(width, chars)", + "return Fits(width, chars)", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + ( + "a label whole in its picture is called cut off", + "internal/format/imagelabel/imagelabel.go", + "return Fits(width, chars) && height >= textHeight(width, chars)", + "return Fits(width, chars) && height > 1<<20", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + ( + "a film says it carries a label cut off at the foot of its picture", + "internal/format/video/choose.go", + "return Labelled(c.Width, c.Label) && imagelabel.Readable(c.Width, c.Height, len(c.Label))", + "return Labelled(c.Width, c.Label) && imagelabel.Readable(c.Width, 1<<20, len(c.Label))", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + ( + "a BMP says it carries a label cut off at the foot of its picture", + "internal/format/bmp/bmp.go", + "labelled, p.Notes = imagelabel.Claim(labelled, w, h, len(label), p.Notes)", + "labelled, p.Notes = imagelabel.Claim(labelled, w, 1<<20, len(label), p.Notes)", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + ( + "an AVIF says it carries a label cut off at the foot of its picture", + "internal/format/avif/avif.go", + "carries, p.Notes = imagelabel.Claim(carries, m.width, m.height, len(label), p.Notes)", + "carries, p.Notes = imagelabel.Claim(carries, m.width, 1<<20, len(label), p.Notes)", + "TestAPictureSaysItCarriesItsLabelOnlyWhenItsTextIsWhole", + ), + # The review of 2026-10-27 on the R13 and R14 guards. + ( + "the farthest step of a set is not the one asked whether it fits above the limit", + "internal/preset/limitset.go", + "if o.bytes > far.bytes {", + "if false {", + "TestEveryPresetAnswersTheLargestSizeAboutTheTop", + ), + ( + "a limit too large for its spread is told the wrong limit to use instead", + "internal/preset/limitset.go", + "core.A(\"Most\", math.MaxInt64-far.bytes)", + "core.A(\"Most\", math.MaxInt64-1)", + "TestEveryPresetAnswersTheLargestSizeAboutTheTop", + ), + ( + "a label whose text exactly fills the picture is called cut off", + "internal/format/imagelabel/imagelabel.go", + "return Fits(width, chars) && height >= textHeight(width, chars)", + "return Fits(width, chars) && height > textHeight(width, chars)", + "TestALabelIsReadableFromTheFirstHeightThatHoldsItsText", + ), ] diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index bf66796a..89273602 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -103,7 +103,6 @@ var notProvenByMutation = map[string]bool{ "TestLayeringHoldsForEveryPackage": true, "TestNoNetworkImports": true, "TestNoProcessExecutionInLowerLayers": true, - "TestNonsenseSizesAreRefusedWithAUsefulMessage": true, "TestNothingIsEverWrittenOverInSilence": true, "TestOracleStaysOutOfProductionCode": true, "TestPlanningTenThousandFilesStaysCheap": true, diff --git a/internal/guard/presetlimits_test.go b/internal/guard/presetlimits_test.go new file mode 100644 index 00000000..243dd6da --- /dev/null +++ b/internal/guard/presetlimits_test.go @@ -0,0 +1,133 @@ +package guard + +import ( + "errors" + "strconv" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/preset" + "github.com/donislawdev/TestingFilesGenerator/internal/recipe" +) + +// A size parameter of any preset, set to the largest size there is, gives a +// set or a refusal about that size - never an answer about the bottom of the +// range, and never a refusal the command line reads as the tool failing. +// +// On 2026-10-27 the two presets that lay out a set around a limit answered +// that limit two different ways. size-boundaries gave the sentence --boundary +// gives, but as a bare error, so the command line ended with code 1, which the +// frozen table keeps for faults of the tool. upload-validation let the step +// above the limit wrap and said "over_1b would be -9223372036854775808 B ... +// Raise the limit above 1 B", the advice the size-boundaries fix of 2026-08-26 +// had removed from that preset alone (docs/STABILITY-REVIEW-2026-10-26.md, R13). +// The presets are asked from the registry, so one added with a size parameter +// is asked too. +func TestEveryPresetAnswersTheLargestSizeAboutTheTop(t *testing.T) { + asked := 0 + for _, p := range preset.All() { + for _, param := range p.Parameters { + if param.Kind != format.PropertySize { + continue + } + asked++ + got, err := preset.Expand(p.ID, preset.Args{param.Name: "9223372036854775807b"}) + if err == nil { + for _, target := range sourceTargets(t, p.ID, got.Source) { + for _, size := range target.Sizes { + if size < 0 { + t.Errorf("%s with %s at the largest size built %s of %d B", p.ID, param.Name, target.ID, size) + } + } + } + continue + } + var impossible *preset.ImpossibleError + if !errors.As(err, &impossible) { + t.Errorf("%s with %s at the largest size gave %v, which is not a refusal of the set and ends as a fault of the tool", p.ID, param.Name, err) + continue + } + if impossible.Setting != param.Name { + t.Errorf("%s with %s at the largest size marks %q, not the setting that was given", p.ID, param.Name, impossible.Setting) + } + if id := impossible.Detail.ID(); id == "preset.StepBelowZero" || id == "preset.StepBelowFloor" { + t.Errorf("%s with %s at the largest size answered about the bottom of the range: %v", p.ID, param.Name, err) + } + } + } + // Two presets lay a set out around a limit and one more takes a sample + // size, so fewer than three asked means the registry was not read. + if asked < 3 { + t.Fatalf("only %d size parameter(s) were asked - the presets were not read", asked) + } + + for _, id := range []string{"size-boundaries", "upload-validation"} { + _, err := preset.Expand(id, preset.Args{"limit": "9223372036854775807b"}) + if !errors.Is(err, core.ErrBoundaryTooLarge) { + t.Errorf("%s gave %v for a limit with no room above it, expected the refusal --boundary gives", id, err) + } + } + + // A spread wider than a byte leaves numbers above the limit and no room + // for its farthest step. The refusal names the largest limit that would + // fit, ten bytes under the top here, and that limit builds - so the number + // the hint gives is one that works (found by review on 2026-10-27: the + // sentence said there was no number above a limit two bytes below it). + spread := preset.Args{"limit": "9223372036854775805b", "spread": "10b", "format": "txt"} + _, err := preset.Expand("size-boundaries", spread) + var impossible *preset.ImpossibleError + if !errors.As(err, &impossible) || !errors.Is(err, core.ErrBoundaryTooLarge) || + !strings.Contains(impossible.Hint.String(), "9223372036854775797 B or less") { + t.Errorf("a limit two bytes under the top with a spread of 10b gave %v, expected the largest limit that fits", err) + } + spread["limit"] = "9223372036854775797b" + if _, err := preset.Expand("size-boundaries", spread); err != nil { + t.Errorf("the limit the refusal names was refused as well: %v", err) + } +} + +// The file far over the limit is the limit times the multiplier to the byte, or +// the set is refused - it is never a product that wrapped round. +// +// Measured on 2026-10-27: a limit of 1844674407370955264B at 10x gave +// far_over_10x a size of 1024 B, under the limit, while the manifest still +// expected it to be refused for size_limit (R13). The largest limit that fits +// builds, so this cannot be satisfied by refusing every large limit. +func TestTheFileFarOverTheLimitIsThatManyTimesTheLimitOrNothing(t *testing.T) { + const most = int64(922337203685477580) // the largest int64 over ten + args := func(limit int64) preset.Args { + return preset.Args{"limit": strconv.FormatInt(limit, 10) + "b", "far-over": "10x", "bulk": "0", "allow": "txt", "deny": "exe"} + } + + found := false + for _, target := range expandedTargets(t, "upload-validation", args(most)) { + if target.ID != "far_over_10x" { + continue + } + found = true + if len(target.Sizes) != 1 || target.Sizes[0] != 10*most { + t.Errorf("far_over_10x at a limit of %d B is %v, expected %d B", most, target.Sizes, 10*most) + } + } + if !found { + t.Fatal("the set has no far_over_10x, so this guard read the wrong thing") + } + + _, err := preset.Expand("upload-validation", args(most+1)) + var impossible *preset.ImpossibleError + if !errors.As(err, &impossible) || impossible.Setting != "far-over" { + t.Fatalf("a limit of %d B at 10x gave %v, expected the set refused about far-over", most+1, err) + } +} + +// sourceTargets reads the targets of a recipe a preset expanded to. +func sourceTargets(t *testing.T, id string, src []byte) []recipe.Target { + t.Helper() + rec, err := recipe.Parse(src, id+".yaml") + if err != nil { + t.Fatalf("the recipe %s expanded to does not read back: %v", id, err) + } + return rec.Targets +} diff --git a/internal/guard/size_test.go b/internal/guard/size_test.go index 34a80b58..22f0d3a1 100644 --- a/internal/guard/size_test.go +++ b/internal/guard/size_test.go @@ -1,8 +1,12 @@ package guard import ( + "math/big" + "math/rand/v2" + "strconv" "strings" "testing" + "time" "github.com/donislawdev/TestingFilesGenerator/internal/core" ) @@ -94,8 +98,127 @@ func TestASizeThatIsNotAWholeByteIsRefused(t *testing.T) { } } +// A size with a decimal point is read to the byte, not through a float. +// +// Until 2026-10-27 the decimal went through float64, which carries 53 bits, so +// a fraction too fine for it was rounded before anyone asked whether it was +// whole: 255.99999999999999B planned a file of 256 B, 1.0000000000000001kb one +// of 1024 B, and 9007199254740993.0B one a byte smaller than asked - three +// sizes the tool did not ask about, contrary to D6 and to RECIPE.md, which +// already said such a fraction is an error. The refusal it did give read +// "works out to 0.0000 bytes" for a value that is not nought. +// docs/STABILITY-REVIEW-2026-10-26.md, R17. +func TestASizeWithADecimalPointIsReadToTheByte(t *testing.T) { + const largest = int64(9223372036854775807) + for _, c := range []struct { + in string + want int64 + }{ + {"9007199254740993.0B", 9007199254740993}, + {"9223372036854775807.0B", largest}, + {".5kb", 512}, + {"5.kb", 5120}, + {"+1.5kb", 1536}, + {"007.50kb", 7680}, + {"-0.0", 0}, + {"0.0kb", 0}, + {"8191.9990234375tb", (8191<<10 + 1023) << 30}, + } { + got, err := core.ParseSize(c.in) + if err != nil || got != c.want { + t.Errorf("%q gave %d B and %v, expected %d B", c.in, got, err, c.want) + } + } + + for _, c := range []struct{ in, between string }{ + {"255.99999999999999B", "between 255 and 256"}, + {"1.0000000000000001kb", "between 1024 and 1025"}, + {"0.0000000000000000001tb", "between 0 and 1"}, + // Whole in its first sixty digits and not in its sixty first. + {"0.5" + strings.Repeat("0", 59) + "1kb", "between 512 and 513"}, + } { + got, err := core.ParseSize(c.in) + if err == nil { + t.Errorf("%.40q was accepted as %d B instead of being refused", c.in, got) + continue + } + if !strings.Contains(err.Error(), c.between) { + t.Errorf("%.40q was refused without naming the whole numbers around it (%s): %.300v", c.in, c.between, err) + } + } + + for _, in := range []string{"9223372036854775808.0B", "8388608.0tb", "8388607.9999999999999999999tb"} { + if got, err := core.ParseSize(in); err == nil || !strings.Contains(err.Error(), "too large") { + t.Errorf("%q gave %d B and %v, expected too large", in, got, err) + } + } + + // Four million digits on either side of the point, which a recipe can + // carry. Exact arithmetic reads digits in quadratic time - 23 s for this + // many on the machine of 2026-10-27 - so the answer is owed without + // reading them all (untouchable rule 23). + many := strings.Repeat("7", 4_000_000) + answersAtOnce(t, many+".5b", "too large") + answersAtOnce(t, "0."+many+"b", "between 0 and 1") +} + +// answersAtOnce asks ParseSize and wants its refusal within five seconds. +func answersAtOnce(t *testing.T, in, want string) { + t.Helper() + done := make(chan error, 1) + go func() { + _, err := core.ParseSize(in) + done <- err + }() + select { + case err := <-done: + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("a size of %d digits gave %.200v, expected %q", len(in), err, want) + } + case <-time.After(5 * time.Second): + t.Errorf("a size of %d digits had no answer after five seconds", len(in)) + } +} + +// Every whole number of bytes, written as the exact decimal of a unit, reads +// back as itself - below 2^53, where a float held it and the answer must not +// have moved, and above, where it did not and the answer was wrong. +func TestEveryExactDecimalOfAUnitReadsBackAsItself(t *testing.T) { + units := []struct { + name string + shift uint + }{{"b", 0}, {"kb", 10}, {"mb", 20}, {"gb", 30}, {"tb", 40}} + rng := rand.New(rand.NewPCG(27, 10)) + for _, u := range units { + for i := 0; i < 2000; i++ { + w := rng.Int64N(1 << 53) + if i%2 == 1 { + w = rng.Int64() + } + text := exactDecimal(w, u.shift) + u.name + got, err := core.ParseSize(text) + if err != nil || got != w { + t.Fatalf("%q gave %d B and %v, expected %d B", text, got, err, w) + } + } + } +} + +// exactDecimal writes w / 2^shift with every digit it has, and always with a +// decimal point, so the value goes the way a fraction goes rather than the way +// a whole number does. +func exactDecimal(w int64, shift uint) string { + whole := w >> shift + rest := w - whole< far.bytes { + far = o + } + } + if s.limit > math.MaxInt64-far.bytes { + return &ImpossibleError{ + Preset: s.preset, Setting: s.setting, + Detail: core.Says("preset.StepPastTheLargest", "%s would be %d B over the limit of %d B, more than the largest size there is, %d B", + core.A("Step", "over_"+far.text), core.A("Over", far.bytes), core.A("Limit", s.limit), core.A("Largest", int64(math.MaxInt64))), + Hint: core.Says("preset.UseASettingOfAtMost", "Use a {setting} of %d B or less.", core.A("Most", math.MaxInt64-far.bytes)), + Cause: core.ErrBoundaryTooLarge, + } + } floor := format.SmallestRemembered(s.desc, format.Request{Seed: 1, Label: true}) for _, one := range set { if one.size >= floor { diff --git a/internal/preset/preset.go b/internal/preset/preset.go index 534ace25..c8dfd26f 100644 --- a/internal/preset/preset.go +++ b/internal/preset/preset.go @@ -309,8 +309,15 @@ type ImpossibleError struct { // both surfaces call it - the flag without its dashes and the label on the // field. See AboutSetting. Setting string + + // Cause is the refusal this one is, where another door gives the same one + // - a limit with no room above it is core.ErrBoundaryTooLarge whether + // --boundary or a preset was asked - so errors.Is finds it either way. + Cause error } +func (e *ImpossibleError) Unwrap() error { return e.Cause } + // What happened, why, and what to do instead - the three parts a report keeps // apart, on the refusal a preset can produce. Error still assembles the one // sentence the command line has always printed. diff --git a/internal/preset/sizeboundaries.go b/internal/preset/sizeboundaries.go index 5afddd0d..8a132659 100644 --- a/internal/preset/sizeboundaries.go +++ b/internal/preset/sizeboundaries.go @@ -180,22 +180,8 @@ func expandSizeBoundaries(args Args) ([]byte, error) { return nil, err } - // The top of the range before the bottom of it, because the wrap happens - // on the way up and the refusal that followed was about the way down. - // - // Measured on 2026-08-26: "--preset size-boundaries --limit - // 9223372036854775807b" said "over_1b would be -9223372036854775808 B, and - // a file cannot be smaller than nothing. Raise the limit above 1051991 B" - - // an answer about the bottom of the range to a question about the top, with - // advice pointing the wrong way. That is the same defect core already fixed - // for --boundary on 2026-08-03, so this reaches for the same sentence - // rather than writing a second one that could drift from it. - for _, o := range spread { - if limit+o.bytes < limit { - return nil, core.ErrBoundaryTooLarge - } - } - + // A limit with no room above it is refused by reachable, which asks it of + // every set laid out around a limit rather than of this preset alone. around := limitSet{ preset: boundariesID, setting: "limit", group: boundariesID, desc: desc, limit: limit, limitText: limitText, spread: spread, diff --git a/internal/preset/tabularimport.go b/internal/preset/tabularimport.go index 32767331..cbfb468c 100644 --- a/internal/preset/tabularimport.go +++ b/internal/preset/tabularimport.go @@ -27,6 +27,13 @@ const ( sheetRowsMax = 200000 sheetColumnsMax = 32768 + // spreadsheetColumns is how many columns a spreadsheet shows. LibreOffice + // Calc read a workbook of 16384 columns back whole and one of 16385 as + // 16384, the last dropped without a word (measured 2026-09-08 - see + // format/xlsx), and Excel stops at the same column. Past it, whether an + // import refuses a table, cuts it or takes it whole is its own policy. + spreadsheetColumns = 16384 + dialectGroup = "csv-dialects" wideGroup = "csv-wide" sheetGroup = "xlsx-rows" @@ -221,14 +228,19 @@ func wideFile(csv format.Descriptor) setFile { // or take it whole is its owner's decision, so this is a position // rather than a promise - MF5. expected: "unspecified", reason: "count_limit", - purpose: fmt.Sprintf("A CSV table with %d columns, the most this build writes. A spreadsheet may show only the first 16384 and drop the rest without a word - "+ - "whether your import refuses such a table, cuts it or takes it whole is its own policy.", columns.Max), + purpose: fmt.Sprintf("A CSV table with %d columns, the most this build writes. A spreadsheet may show only the first %d and drop the rest without a word - "+ + "whether your import refuses such a table, cuts it or takes it whole is its own policy.", columns.Max, spreadsheetColumns), } } // sheetFile is the spreadsheet at the row and column counts asked for. +// +// Taken whole, certain, only as wide as a spreadsheet is. Until 2026-10-27 it +// said so at every width, and past spreadsheetColumns that is a promise the +// reader it is written for breaks without a word - MF5 and +// docs/STABILITY-REVIEW-2026-10-26.md, R16. func sheetFile(xlsx format.Descriptor, rows, columns string) setFile { - return setFile{ + f := setFile{ id: "sheet", name: "sheet.xlsx", group: sheetGroup, desc: xlsx, props: map[string]string{rowsParam: rows, columnsParam: columns}, atFloor: true, @@ -236,6 +248,14 @@ func sheetFile(xlsx format.Descriptor, rows, columns string) setFile { purpose: fmt.Sprintf("An Excel workbook of %s rows and %s columns, as many as were asked for. "+ "Your import should take it whole - it checks that a real spreadsheet of this size is read to its last row.", rows, columns), } + // Settled already as a whole number inside the declared range. + if wide, _ := strconv.Atoi(columns); wide > spreadsheetColumns { + f.expected, f.reason = "unspecified", "count_limit" + f.purpose = fmt.Sprintf("An Excel workbook of %s rows and %s columns, wider than the %d a spreadsheet shows. "+ + "A spreadsheet may show only the first %d and drop the rest without a word - "+ + "whether your import refuses such a workbook, cuts it or takes it whole is its own policy.", rows, columns, spreadsheetColumns, spreadsheetColumns) + } + return f } // layoutFiles is the same records written the three ways a document can be diff --git a/internal/preset/uploadset.go b/internal/preset/uploadset.go index 2690bf91..7768f2fc 100644 --- a/internal/preset/uploadset.go +++ b/internal/preset/uploadset.go @@ -3,6 +3,7 @@ package preset import ( "errors" "fmt" + "math" "sort" "strconv" "strings" @@ -255,6 +256,30 @@ func farOverTimes(raw string) (int64, error) { return times, nil } +// farOverFits refuses a file far over the limit that would be past the largest +// size there is. +// +// That file is the limit times the multiplier, and until 2026-10-27 nothing +// asked whether the product fitted: a limit of 1844674407370955264B at 10x +// wrapped round to a file of 1024 B, under the limit, still named far_over_10x +// and still expected to be refused for size_limit (R13 of +// docs/STABILITY-REVIEW-2026-10-26.md). +// +// Asked after the limit itself is, so that a limit with no room above it at +// all hears that, rather than advice to turn this file off which would not +// help. +func (s uploadSet) farOverFits() error { + if s.farOver == 0 || s.limit <= math.MaxInt64/s.farOver { + return nil + } + return &ImpossibleError{ + Preset: uploadID, Setting: farOverParam, + Detail: core.Says("preset.FarOverPastTheLargest", "%s would be %d times the limit of %d B, more than the largest size there is, %d B", + core.A("Step", "far_over_"+strconv.FormatInt(s.farOver, 10)+"x"), core.A("Times", s.farOver), core.A("Limit", s.limit), core.A("Largest", int64(math.MaxInt64))), + Hint: core.Says("preset.LowerTheLimitOrTurnOff", "Lower the limit to %d B or less, or turn {setting} off.", core.A("Most", math.MaxInt64/s.farOver)), + } +} + func expandUploadValidation(args Args) ([]byte, error) { s, err := settleUpload(args) if err != nil { @@ -273,6 +298,9 @@ func expandUploadValidation(args Args) ([]byte, error) { if err := around.reachable(steps); err != nil { return nil, err } + if err := s.farOverFits(); err != nil { + return nil, err + } files := s.files() if err := s.reachable(files); err != nil {