diff --git a/docs/src/commands/generate.md b/docs/src/commands/generate.md index 5fcf4ca..f0fea07 100644 --- a/docs/src/commands/generate.md +++ b/docs/src/commands/generate.md @@ -160,9 +160,10 @@ What to know before using it: usual cause, because in a fixed-length tag it looks like substitutions. A cut at the designed position in a row whose last 8 bases and primer bases are as designed was wrong in 0.2% to 0.3% of rows, and any other cut in 3% to 5%. -- **A row keeps its right primer** when the cut would remove the target base or leave fewer than - 10 bases. The QC JSON counts these rows. Its count of rows that lost right-primer bases - (`rows_right_primer_removed`) includes the Ref and Alt rows of markers without reads. +- **A row keeps its right primer** when the cut would remove the base its category was read from, + or leave fewer than 10 bases. A row whose target is not read has no such base and is cut where + its primer is found. The QC JSON counts these rows. Its count of rows that lost right-primer + bases (`rows_right_primer_removed`) includes the Ref and Alt rows of markers without reads. - **Merged rows of different categories.** When rows of two categories become the same sequence, the row takes the category of the one with the most reads, or Ref or Alt if one of them is the canonical row. The QC JSON counts these merges (`rows_merged_across_allele_groups`). They are the diff --git a/docs/src/development/changelog.md b/docs/src/development/changelog.md index 46d7634..7e2c6c3 100644 --- a/docs/src/development/changelog.md +++ b/docs/src/development/changelog.md @@ -38,10 +38,8 @@ Notable changes by version. Measurements behind each change are in the rebuilt once. On three sets of simulated haplotypes the rows counted for the right allele rise from 91.7% to 94.8% (v0.13: 92.2%) and unread rows fall from 3.2% to 0.06%; rows counted for the wrong allele stay at 0.14% (v0.13: 1.15%). On pecan 149 rows are settled and 124 of them gain an - allele; on sweetpotato 45 and 31. No other row of those tables changes, and no read count. With - `--remove-primers on`, 7 of the settled pecan rows are cut at their right primer before the - target, like the rows named below. A run takes 4% to 7% longer on these cohorts and the cache - grows by 2% to 3%. + allele; on sweetpotato 45 and 31. No other row of those tables changes, and no read count. A + run takes 4% to 7% longer on these cohorts and the cache grows by 2% to 3%. - `generate` reads a row from the Alt tag when the Ref tag and the Alt tag explain it through gaps in the same number of edits and the Alt tag needs fewer separate gaps. Before, the Ref tag's alignment was used on every such tie. An Alt haplotype with one insertion or deletion beside its @@ -58,8 +56,19 @@ Notable changes by version. Measurements behind each change are in the substitution uses that base; `call` reads this form. `--remove-primers on` writes the rows from the same reading. On pecan, sweetpotato, alfalfa and blackberry 0.07% to 0.25% of rows get other `Variants` (28 to 75 rows), and with primers kept no row is left whose `Variants` and category - put different bases on the target. Without primers 16 to 31 rows per panel still do: rows cut at - their right primer to another length than the Ref row. Categories and counts do not change. + put different bases on the target. Without primers up to 27 rows per panel still do, nearly all + of them rows of another length than the Ref row, which are read again as printed. Categories and + counts do not change. +- `generate --remove-primers on` no longer cuts off the base a row's category was read from. The cut + placed the target with its own alignment, which could differ from the row's reading. The row then + lost that base, and its `Variants` showed no base or another one on the target. Such a row now + keeps its right primer: 14 rows on pecan and 5 on alfalfa, none on sweetpotato or blackberry. A + row whose target is not read has no such base and is cut where its primer is found: 10 rows on + pecan, 17 on alfalfa, 3 on sweetpotato and 1 on blackberry, all `Other`. No row merges or splits, + and no read moves between alleles. Rows whose category and `Variants` disagree fall from 37 to 27 + on pecan and from 22 to 18 on alfalfa. Against pecan's full reads, 7 of the 14 rows are now cut + right or closer. In the other 7 the old cut was right or one base off: their category had been + read from a primer base. Tables written with primers are unchanged. - `Variants` writes an insertion or deletion on the very first base of a tag without an anchor base (`1:.>GG`, `1:TT>.`). Before, it borrowed the base after it, which another event could use as well, and the row then could not be rebuilt from its `Variants`. A tag that starts with a diff --git a/docs/src/reference/output-formats.md b/docs/src/reference/output-formats.md index 90005c1..30a4971 100644 --- a/docs/src/reference/output-formats.md +++ b/docs/src/reference/output-formats.md @@ -51,9 +51,9 @@ To keep SNP-only microhaplotypes, drop `indel`, `both` and `unresolved` rows by **`Variants`** lists the row's events against the Ref tag, `;`-separated, in 1-based coordinates on the tag as printed. They come from the alignment the row's category was read from, so the two show -the same base on the target. One exception: in a table without primers, a row cut at its right -primer to another length than the Ref row is aligned end to end, which can differ from the reading -its category came from. +the same base on the target. One exception: in a table without primers, a row of another length +than the Ref row is read again as printed, which can differ from the reading its category came +from. The row still holds the base its category was read from. | Token | Meaning | |-------|---------| diff --git a/madcats.py b/madcats.py index 69a5299..3c755c6 100644 --- a/madcats.py +++ b/madcats.py @@ -3970,6 +3970,26 @@ def parse_variants(text: str, ref: str, sequence: str): return events +def category_base( + locus: Locus, sequence: str, gap_band: int, continuations: Optional[Mapping[str, str]] = None +) -> Optional[int]: + """Which base of a row its category is read from, as an index into the row; None when it shows none. + + The fixed offset, or where the row's accepted gapped reading puts the + target (row_decision). None for a target inside a deletion, past the + row's end, or in an unresolved end. classify_sequence names the + category from this base and right_primer_cut never removes it. + """ + decision = row_decision(locus, sequence, gap_band, continuations) + if _target_unread(locus, decision): + return None + if decision.reading is None: + index = locus.variant_pos + else: + index = decision.reading.ref_to_read.get(locus.variant_pos) + return index if index is not None and index < len(sequence) else None + + def classify_sequence( locus: Locus, sequence: str, gap_band: int = GAP_BAND_DEFAULT, continuations: Optional[Mapping[str, str]] = None ) -> str: @@ -3999,19 +4019,9 @@ def classify_sequence( return "Ref" if sequence == locus.alt_tag: return "Alt" - index = locus.variant_pos - if gap_band > 0: - # The cheapest substitution-only reading over both canonicals is the - # bound a gapped reading must beat; the reading itself is the one - # accepted_gapped_reading gives, shared with the row's Variants. - best, unresolved = row_decision(locus, sequence, gap_band, continuations) - if unresolved is not None and locus.variant_pos >= unresolved[0]: - return "Other" - if best is not None: - mapped = best.ref_to_read.get(locus.variant_pos) - if mapped is None or mapped >= len(sequence): - return "Other" - index = mapped + index = category_base(locus, sequence, gap_band, continuations) + if index is None: + return "Other" base = sequence[index] if len(IUPAC_BASES.get(base, frozenset(base))) != 1: return "Other" @@ -4150,8 +4160,8 @@ def right_primer_cut( The row without its primers is ``sequence[len(left primer):][:cut]``. ``kept`` is None, or why a right primer found in the row stays in it: - "target" (the cut would remove the target base) or "short" (fewer than - TRIM_MIN_BASES bases would remain). + "target" (the cut would remove the base the row's category was read + from) or "short" (fewer than TRIM_MIN_BASES bases would remain). A row is a fixed-length window of its amplicon, so an insertion moves the right primer toward the row's end and a deletion brings more of it @@ -4164,8 +4174,7 @@ def right_primer_cut( and the reading leaves two or more bases from there on unexplained. An insertion: the bases at the cut are not the primer's (two or more differ), and the row reads exactly as primer from up to ``gap_band`` - bases later. Neither rule moves the cut onto the base the row's - category was read from. + bases later. Last, the primer is looked for directly, in a row whose cut does not read as primer and in every row of a marker without a site: whole, or @@ -4175,17 +4184,26 @@ def right_primer_cut( ``gap_band`` allows for. The whole primer is not looked for on a marker whose own Ref tag holds a copy of it before the site: there a copy in a row is the insert's, not the end of a shorter product. + + No cut removes the base the row's category was read from + (category_base). The alignment made here to find the primer can put + the target elsewhere, so it is not asked. A row whose target is not + read has no such base, and is cut wherever its primer is found. """ after_left = len(locus.left_primer) body = sequence[after_left:] size = len(body) cut = size + read_from = category_base(locus, sequence, gap_band, continuations) def primer_from(index: int, primer: str) -> bool: """body[index:] is exactly the start of ``primer``.""" return size - index <= len(primer) and tag_mismatches(body[index:], primer[: size - index]) == 0 - ops = None + def removes_category_base(index: int) -> bool: + """A cut at ``index`` takes the base the row's category was read from.""" + return read_from is not None and index <= read_from - after_left + primers: Tuple[str, ...] = () if site is not None: start, primer = site @@ -4204,11 +4222,8 @@ def primer_from(index: int, primer: str) -> bool: if primer_from(earlier, primer) and 2 <= sum( 1 for i in range(earlier, size) if i not in aligned or not _design_matches(body[i], template[aligned[i]]) ): - reading = row_reading(locus, sequence, gap_band, continuations) # where classification read the category - category_base = locus.variant_pos if reading is None else reading.ref_to_read.get(locus.variant_pos) - if category_base is None or earlier > category_base - after_left: + if not removes_category_base(earlier): cut, moved = earlier, True - ops = tuple(op for op in ops if op[1] < cut) break at_cut = body[cut : cut + len(primer)] if not moved and size - cut >= 2 and tag_mismatches(at_cut, primer[: len(at_cut)]) >= 2: @@ -4217,7 +4232,6 @@ def primer_from(index: int, primer: str) -> bool: if later > size - 2: break if primer_from(later, primer): - ops = tuple(op for op in ops if op[1] < cut) cut = later break elif locus.right_primer: @@ -4234,10 +4248,7 @@ def primer_from(index: int, primer: str) -> bool: cut = hit break if cut < size: - if ops is None: - ops = _cheapest_ops(body, locus.ref_tag[after_left:], gap_band) - target = next((i for kind, i, j in ops if kind == "M" and j == locus.variant_pos - after_left), None) - if target is not None and cut <= target: + if removes_category_base(cut): return size, "target" if cut < TRIM_MIN_BASES: return size, "short" diff --git a/tests/test_remove_primers.py b/tests/test_remove_primers.py index 3288f7a..fac82ad 100644 --- a/tests/test_remove_primers.py +++ b/tests/test_remove_primers.py @@ -258,6 +258,41 @@ def test_row_keeps_its_primer_rather_than_lose_the_target_or_most_of_itself(self few_bases = tag_of(parts, parts[1][: mp.TRIM_MIN_BASES - 1]) self.assertEqual(self.trimmed(early_target, few_bases), (whole(few_bases), "short")) + def test_row_keeps_the_base_its_category_was_read_from(self): + # A diverged haplotype: seven substitutions, and two bases missing shortly before the + # target, which is the last insert base but one. Seven substitutions are too many for a + # gapped reading, so the row is read at the fixed offsets. There its first primer base + # stands on the target, and that base is the Alt base. The alignment that finds the + # primer reads the deletion and places the target two bases earlier, so it would cut + # the primer off together with the base the category was read from. + rng = random.Random(2) + parts = left, insert, right = dna(rng, LEFT), dna(rng, INSERT), dna(rng, RIGHT) + target = INSERT - 2 + alt_insert = insert[:target] + right[0] + insert[target + 1 :] + locus = mp.Locus(0, "m", left, right, insert[target], right[0], LEFT + target, tag_of(parts, insert), tag_of(parts, alt_insert)) + diverged = insert + for index in range(1, 20, 3): + diverged = substituted(diverged, index) + row = tag_of(parts, diverged[:26] + diverged[28:]) + site = (LEFT + INSERT, right) + self.assertIsNone(mp.row_reading(locus, row, GAP_BAND)) + self.assertEqual(mp.classify_sequence(locus, row, GAP_BAND), "AltMatch") + self.assertTrue(right.startswith(row[LEFT + target :])) # the primer begins on the base that was read + self.assertEqual(self.trimmed(locus, row, site), (row[LEFT:], "target")) + # On a marker with its target elsewhere, the same row is cut where its primer begins. + early = dataclasses.replace(locus, variant_pos=LEFT + TARGET, alt_tag=tag_of(parts, substituted(insert, TARGET))) + self.assertEqual(self.trimmed(early, row, site), (row[LEFT : LEFT + INSERT - 2], None)) + + def test_row_that_ends_before_its_target_has_no_category_base(self): + # generate never writes such a row: a row is a whole tag. The functions still answer for + # one. It shows no base on the target, so it is Other and no primer is kept for it. + locus = self.loci[0] + short = locus.ref_tag[: locus.variant_pos] + for gap_band in (GAP_BAND, 0): + self.assertIsNone(mp.category_base(locus, short, gap_band)) + self.assertEqual(mp.classify_sequence(locus, short, gap_band), "Other") + self.assertEqual(self.trimmed(locus, short), (short[LEFT:], None)) + class DescriptionTests(unittest.TestCase): """Events of rows of another length than their Ref row, as a table without primers has them."""