From 27dfc826481bab201feaee2b11ec95a31472e780 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Mon, 16 Nov 2026 17:04:48 +0000 Subject: [PATCH 1/7] remove sort by fit, and fit def --- datafaker/interactive/generators.py | 28 +++++++++++++++------------- datafaker/proposers/base.py | 16 ++++++++-------- datafaker/proposers/choice.py | 12 ++++++------ datafaker/proposers/continuous.py | 26 +++++++++++++------------- datafaker/proposers/extract.py | 9 +++++---- datafaker/proposers/intervals.py | 13 +++++++------ datafaker/proposers/mimesis.py | 6 +++--- 7 files changed, 57 insertions(+), 53 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index cde6d532..32d38e28 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1,4 +1,5 @@ """Generator configuration shell.""" # pylint: disable=too-many-lines + import copy import functools import re @@ -387,7 +388,7 @@ def _get_table_and_proposer(self) -> tuple[str | None, ProposerInfo | None]: def _get_column_names(self) -> list[str]: """Get the (unqualified) names for all the current columns.""" - (_, proposer_info) = self._get_table_and_proposer() + _, proposer_info = self._get_table_and_proposer() return proposer_info.columns if proposer_info else [] def _column_metadata(self) -> list[Column]: @@ -399,7 +400,7 @@ def _column_metadata(self) -> list[Column]: def set_prompt(self) -> None: """Set the prompt according to the current table, column and generator.""" - (table_name, prop_info) = self._get_table_and_proposer() + table_name, prop_info = self._get_table_and_proposer() if table_name is None: self.prompt = "(generators) " return @@ -666,7 +667,7 @@ def _get_table_and_column(self, target: str) -> tuple[int | None, int | None]: if prop_index is not None: return (None, prop_index) # The whole thing isn't a column either, so let's split it: - (first_part, last_part) = split_column_full_name(target) + first_part, last_part = split_column_full_name(target) if not first_part: # It doesn't split, so that's the end self.print(self.ERROR_NO_SUCH_TABLE_OR_COLUMN, last_part) @@ -687,7 +688,7 @@ def go_to(self, target: str) -> bool: :return: True on success. """ - (table_index, prop_index) = self._get_table_and_column(target) + table_index, prop_index = self._get_table_and_column(target) if prop_index is None: return False if table_index is not None: @@ -734,7 +735,7 @@ def complete_next( self, text: str, _line: str, _begidx: int, _endidx: int ) -> list[str]: """Completions for the arguments of the ``next`` command.""" - (first_part, last_part) = split_column_full_name(text) + first_part, last_part = split_column_full_name(text) if first_part: # first_part is table, last_part is column table_index = self.get_table_index(first_part) @@ -789,7 +790,7 @@ def _proposers_valid(self) -> bool: def _get_proposer_proposals(self) -> list[Proposer]: """ - Get a list of acceptable proposers, sorted by decreasing fit to the actual data. + Get a list of acceptable proposers, sorted by name. The cached list from the most recent 'propose'/'propose all' is reused if still valid for the current table/columns, so that 'set @@ -800,7 +801,8 @@ def _get_proposer_proposals(self) -> list[Proposer]: props = everything_factory(self.config, self.metadata).get_proposers( columns, self.sync_engine ) - sorted_props = sorted(props, key=lambda g: g.fit(9999)) + # sorted_props = sorted(props, key=lambda g: g.fit(9999)) + sorted_props = sorted(props, key=lambda g: g.name()) self.proposers = sorted_props self.proposers_valid_columns = ( self.table_index, @@ -888,9 +890,9 @@ def do_compare(self, arg: str) -> None: n = int(argument) if 0 < n <= len(props): prop = props[n - 1] - comparison[ - f"{n}. {theme.function}{prop.name()}" - ] = prop.generate_data(limit) + comparison[f"{n}. {theme.function}{prop.name()}"] = ( + prop.generate_data(limit) + ) self._print_values_queried(table_name, n, prop) self.print_table_by_columns(comparison) @@ -1334,7 +1336,7 @@ def do_set(self, arg: str) -> None: def set_proposer(self, prop: Proposer | None) -> None: """Set the current column's generator.""" - (table, gen_info) = self._get_table_and_proposer() + table, gen_info = self._get_table_and_proposer() if table is None: self.print("Error: no table") return @@ -1578,7 +1580,7 @@ def do_role(self, arg: str) -> None: parse = self.parse_role(arg) if parse is None: return - (column, command) = parse + column, command = parse if not column: cols = self.get_current_columns() if len(cols) != 1: @@ -1587,7 +1589,7 @@ def do_role(self, arg: str) -> None: ) return column = list(cols)[0] - (table, col) = split_column_full_name(column) + table, col = split_column_full_name(column) if not table: entry = self.get_table() if entry is None: diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 6b456ca0..351c76d1 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -139,14 +139,14 @@ def actual_kwargs(self) -> dict[str, Any]: def generate_data(self, count: int) -> list[Any]: """Generate ``count`` random data points for this column.""" - def fit(self, default: float = -1) -> float: - """ - Return a value representing how well the distribution fits the real source data. - - 0.0 means "perfectly". - Returns default if no fitness has been defined. - """ - return default + # def fit(self, default: float = -1) -> float: + # """ + # Return a value representing how well the distribution fits the real source data. + + # 0.0 means "perfectly". + # Returns default if no fitness has been defined. + # """ + # return default def guarantees_fresh_uniqueness(self) -> bool: """ diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index 38e475f2..f080342b 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -190,9 +190,9 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: }, } - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default if self._fit is None else self._fit + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # return default if self._fit is None else self._fit class ZipfChoiceProposer(ChoiceProposer): @@ -288,9 +288,9 @@ def __init__(self, results: CursorResult, suppress_count: int = 0) -> None: cvs: list[dict[str, Any]] = [] # list of dicts with keys "v" and "count" values_not_suppressed = [] # All values found more than SUPPRESS_COUNT times counts_not_suppressed = [] # The number for each value not suppressed - cvs_not_suppressed: list[ - dict[str, Any] - ] = [] # list of dicts with keys "v" and "count" + cvs_not_suppressed: list[dict[str, Any]] = ( + [] + ) # list of dicts with keys "v" and "count" for result in results: c = result.f if c != 0: diff --git a/datafaker/proposers/continuous.py b/datafaker/proposers/continuous.py index 33574a36..b79c7395 100644 --- a/datafaker/proposers/continuous.py +++ b/datafaker/proposers/continuous.py @@ -100,11 +100,11 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: }, } - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - if self.buckets is None: - return default - return self.buckets.fit_from_counts(self.expected_buckets) + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # if self.buckets is None: + # return default + # return self.buckets.fit_from_counts(self.expected_buckets) class GaussianProposer(ContinuousDistributionProposer): @@ -290,11 +290,11 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: }, } - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - if self.buckets is None: - return default - return self.buckets.fit_from_counts(self.expected_buckets) + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # if self.buckets is None: + # return default + # return self.buckets.fit_from_counts(self.expected_buckets) class ContinuousLogDistributionProposerFactory(ContinuousDistributionProposerFactory): @@ -395,9 +395,9 @@ def generate_data(self, count: int) -> list[Any]: for _ in range(count) ] - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # return default class MultivariateNormalGeneratorFactoryBase(ProposerFactory): diff --git a/datafaker/proposers/extract.py b/datafaker/proposers/extract.py index efd960d0..8e8b59c2 100644 --- a/datafaker/proposers/extract.py +++ b/datafaker/proposers/extract.py @@ -1,4 +1,5 @@ """Proposers for date member extraction.""" + import datetime from abc import abstractmethod from collections.abc import Mapping, Sequence @@ -59,10 +60,10 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: """Return that no data is required from the database.""" return {} - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - # What should the fit really be? - return 0.0 + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # # What should the fit really be? + # return 0.0 def get_based_on_data(self, count: int) -> list[datetime.datetime]: """Get the data from the based-on column in the source data.""" diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index 8ff0118e..6a3a1122 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -1,4 +1,5 @@ """Proposers for date intervals.""" + import datetime from collections.abc import Mapping, Sequence from typing import Any @@ -360,9 +361,9 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: } } - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default if self._fit is None else self._fit + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # return default if self._fit is None else self._fit def generate_data(self, count: int) -> list[datetime.datetime]: """Generate ``count`` random data points for this column.""" @@ -536,9 +537,9 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: """No source data required.""" return {} - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # return default def generate_data(self, count: int) -> list[Any]: """Generate ``count`` random data points for this column.""" diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py index dbb9282f..a4e7fb12 100644 --- a/datafaker/proposers/mimesis.py +++ b/datafaker/proposers/mimesis.py @@ -104,9 +104,9 @@ def actual_kwargs(self) -> dict[str, Any]: """Get the kwargs (summary statistics) this generator was instantiated with.""" return {} - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default if self._fit is None else self._fit + # def fit(self, default: float = -1) -> float: + # """Get this generator's fit against the real data.""" + # return default if self._fit is None else self._fit class MimesisGeneratorTruncated(MimesisProposer): From 0cb06bc00f8317db6a519eab03891ada96462402 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Tue, 17 Nov 2026 10:57:47 +0000 Subject: [PATCH 2/7] cleaned up unused functions --- datafaker/interactive/generators.py | 6 +-- datafaker/proposers/choice.py | 6 +-- datafaker/proposers/mimesis.py | 60 ++--------------------------- 3 files changed, 9 insertions(+), 63 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 32d38e28..cb7b090b 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -890,9 +890,9 @@ def do_compare(self, arg: str) -> None: n = int(argument) if 0 < n <= len(props): prop = props[n - 1] - comparison[f"{n}. {theme.function}{prop.name()}"] = ( - prop.generate_data(limit) - ) + comparison[ + f"{n}. {theme.function}{prop.name()}" + ] = prop.generate_data(limit) self._print_values_queried(table_name, n, prop) self.print_table_by_columns(comparison) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index f080342b..6b531336 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -288,9 +288,9 @@ def __init__(self, results: CursorResult, suppress_count: int = 0) -> None: cvs: list[dict[str, Any]] = [] # list of dicts with keys "v" and "count" values_not_suppressed = [] # All values found more than SUPPRESS_COUNT times counts_not_suppressed = [] # The number for each value not suppressed - cvs_not_suppressed: list[dict[str, Any]] = ( - [] - ) # list of dicts with keys "v" and "count" + cvs_not_suppressed: list[ + dict[str, Any] + ] = [] # list of dicts with keys "v" and "count" for result in results: c = result.f if c != 0: diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py index a4e7fb12..ca6e2035 100644 --- a/datafaker/proposers/mimesis.py +++ b/datafaker/proposers/mimesis.py @@ -6,11 +6,9 @@ import mimesis import mimesis.locales from sqlalchemy import Column, Engine, cast, extract, func, select -from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.types import Date, DateTime, Integer, Numeric, String, Time from datafaker.proposers.base import ( - Buckets, Proposer, ProposerError, ProposerFactory, @@ -68,30 +66,6 @@ def generate_data(self, count: int) -> list[Any]: class MimesisProposer(MimesisProposerBase): """A generator using Mimesis.""" - def __init__( - self, - function_name: str, - value_fn: Callable[[Any], float] | None = None, - buckets: Buckets | None = None, - ): - """ - Initialise a generator using Mimesis. - - :param function_name: is relative to 'generic', for example 'person.name'. - :param value_fn: Function to convert generator output to floats, if needed. The values - thus produced are compared against the buckets to estimate the fit. - :param buckets: The distribution of string lengths in the real data. If this is None - then the fit method will return None. - """ - super().__init__(function_name) - if buckets is None: - self._fit = None - return - samples = self.generate_data(400) - if value_fn: - samples = [value_fn(s) for s in samples] - self._fit = buckets.fit_from_values(samples) - def function_name(self) -> str: """Get the name of the generator function to call.""" return self._name @@ -104,10 +78,6 @@ def actual_kwargs(self) -> dict[str, Any]: """Get the kwargs (summary statistics) this generator was instantiated with.""" return {} - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # return default if self._fit is None else self._fit - class MimesisGeneratorTruncated(MimesisProposer): """A string generator using Mimesis that must fit within a certain number of characters.""" @@ -116,12 +86,10 @@ def __init__( self, function_name: str, length: int, - value_fn: Callable[[Any], float] | None = None, - buckets: Buckets | None = None, ): """Initialise a MimesisGeneratorTruncated.""" self._length = length - super().__init__(function_name, value_fn, buckets) + super().__init__(function_name) def function_name(self) -> str: """Get the name of the generator function to call.""" @@ -320,32 +288,10 @@ def get_proposers( column_type = get_column_type(column) if not isinstance(column_type, String): return [] - try: - buckets = Buckets.make_buckets( - engine, - column.table, - func.length(column), - ) - fitness_fn = len - except SQLAlchemyError: - # Some column types that appear to be strings (such as enums) - # cannot have their lengths measured. In this case we cannot - # detect fitness using lengths. - buckets = None - fitness_fn = None length = column_type.length if length: - return self._get_generators_with( - MimesisGeneratorTruncated, - length=length, - value_fn=fitness_fn, - buckets=buckets, - ) - return self._get_generators_with( - MimesisProposer, - value_fn=fitness_fn, - buckets=buckets, - ) + return self._get_generators_with(MimesisGeneratorTruncated, length=length) + return self._get_generators_with(MimesisProposer) class MimesisFloatProposerFactory(ProposerFactory): From 0d1e75c6cfb0f3fe9df55db70892277784b8ce51 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Tue, 17 Nov 2026 15:01:11 +0000 Subject: [PATCH 3/7] removed unused bucket and count --- datafaker/proposers/choice.py | 74 +---------------------- datafaker/proposers/continuous.py | 69 +-------------------- datafaker/proposers/intervals.py | 25 -------- tests/test_evaluators_proposal_ranking.py | 5 +- tests/test_functional_mssql.py | 4 +- tests/test_generators_dialect.py | 16 ++--- 6 files changed, 13 insertions(+), 180 deletions(-) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index 6b531336..579c6fc3 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -1,19 +1,15 @@ """Generator factories for making generators for choices of values.""" import decimal -import math -import typing -from abc import abstractmethod from typing import Any, Sequence, Union from sqlalchemy import Column, CursorResult, Engine, desc, func, select from datafaker.dialects import Random -from datafaker.proposers.base import ( +from datafaker.proposers.base import ( # fit_from_buckets, Proposer, ProposerFactory, dist_gen, - fit_from_buckets, ) NumericType = Union[int, float] @@ -23,28 +19,6 @@ MAXIMUM_CHOICES = 500 -def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None]: - """ - Get a zipf distribution for a certain number of items. - - :param total: The total number of items to be distributed. - :param bins: The total number of bins to distribute the items into. - :return: A generator of the number of items in each bin, from the - largest to the smallest. - """ - basic_dist = list(map(lambda n: 1 / n, range(1, bins + 1))) - bd_remaining = sum(basic_dist) - for b in basic_dist: - # yield b/bd_remaining of the `total` remaining - if bd_remaining == 0: - yield 0 - else: - x = math.floor(0.5 + total * b / bd_remaining) - bd_remaining -= x * bd_remaining / total - total -= x - yield x - - def _choice_stmt( # pylint: disable=R0913,R0917 column: Column, store_counts: bool, @@ -104,7 +78,6 @@ def __init__( self, column: Column, values: list[Any], - counts: list[int], sample_count: int | None = None, suppress_count: int = 0, dialect: Any = None, @@ -113,8 +86,6 @@ def __init__( super().__init__() self.column = column self.values = values - estimated_counts = self.get_estimated_counts(counts) - self._fit = fit_from_buckets(counts, estimated_counts) extra_comment = " and their counts" if self.STORE_COUNTS else "" stmt = _choice_stmt( @@ -156,10 +127,6 @@ def __init__( ) self._annotation = "sampled and suppressed" - @abstractmethod - def get_estimated_counts(self, counts: list[int]) -> list[int]: - """Get the counts that we would expect if this distribution was the correct one.""" - def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" return { @@ -190,18 +157,10 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: }, } - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # return default if self._fit is None else self._fit - class ZipfChoiceProposer(ChoiceProposer): """Generator producing items in a Zipf distribution.""" - def get_estimated_counts(self, counts: list[int]) -> list[int]: - """Get the counts that we would expect if this distribution was the correct one.""" - return list(zipf_distribution(sum(counts), len(counts))) - def function_name(self) -> str: """Get the name of the generator function to call.""" return "dist_gen.zipf_choice" @@ -214,28 +173,9 @@ def generate_data(self, count: int) -> list[float]: ] -def uniform_distribution(total: int, bins: int) -> typing.Generator[int, None, None]: - """ - Construct a distribution putting ``total`` items uniformly into ``bins`` bins. - - If they don't fit exactly evenly, the earlier bins will have one more - item than the later bins so the total is as required. - """ - p = total // bins - n = total % bins - for _ in range(0, n): - yield p + 1 - for _ in range(n, bins): - yield p - - class UniformChoiceProposer(ChoiceProposer): """A proposer producing values, each roughly as frequently as each other.""" - def get_estimated_counts(self, counts: list[int]) -> list[int]: - """Get the counts that we would expect if this distribution was the correct one.""" - return list(uniform_distribution(sum(counts), len(counts))) - def function_name(self) -> str: """Get the name of the generator function to call.""" return "dist_gen.choice" @@ -351,19 +291,16 @@ def get_proposers( # pylint: disable=too-many-locals ZipfChoiceProposer( column, vg.values, - vg.counts, dialect=dialect, ), UniformChoiceProposer( column, vg.values, - vg.counts, dialect=dialect, ), WeightedChoiceProposer( column, vg.cvs, - vg.counts, dialect=dialect, ), ] @@ -372,21 +309,18 @@ def get_proposers( # pylint: disable=too-many-locals ZipfChoiceProposer( column, vg.values_not_suppressed, - vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, ), UniformChoiceProposer( column, vg.values_not_suppressed, - vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, ), WeightedChoiceProposer( column=column, values=vg.cvs_not_suppressed, - counts=vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, ), @@ -414,21 +348,18 @@ def get_proposers( # pylint: disable=too-many-locals ZipfChoiceProposer( column, vg.values, - vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, ), UniformChoiceProposer( column, vg.values, - vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, ), WeightedChoiceProposer( column, vg.cvs, - vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, ), @@ -438,7 +369,6 @@ def get_proposers( # pylint: disable=too-many-locals ZipfChoiceProposer( column, vg.values_not_suppressed, - vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, @@ -446,7 +376,6 @@ def get_proposers( # pylint: disable=too-many-locals UniformChoiceProposer( column, vg.values_not_suppressed, - vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, @@ -454,7 +383,6 @@ def get_proposers( # pylint: disable=too-many-locals WeightedChoiceProposer( column=column, values=vg.cvs_not_suppressed, - counts=vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, diff --git a/datafaker/proposers/continuous.py b/datafaker/proposers/continuous.py index b79c7395..72bd47fd 100644 --- a/datafaker/proposers/continuous.py +++ b/datafaker/proposers/continuous.py @@ -29,7 +29,6 @@ from datafaker.dialects import IsNotNull, LogNatural, NullIf, Random, StdDev from datafaker.proposers.base import ( Buckets, - NumericType, Proposer, ProposerFactory, dist_gen, @@ -41,8 +40,6 @@ class ContinuousDistributionProposer(Proposer): """Base class for generators producing continuous distributions.""" - expected_buckets: Sequence[NumericType] = [] - def __init__( self, table: Table, @@ -100,29 +97,10 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: }, } - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # if self.buckets is None: - # return default - # return self.buckets.fit_from_counts(self.expected_buckets) - class GaussianProposer(ContinuousDistributionProposer): """Generator producing numbers in a Gaussian (normal) distribution.""" - expected_buckets = [ - 0.0227, - 0.0441, - 0.0918, - 0.1499, - 0.1915, - 0.1915, - 0.1499, - 0.0918, - 0.0441, - 0.0227, - ] - def function_name(self) -> str: """Get the name of the generator function to call.""" return "dist_gen.normal" @@ -138,19 +116,6 @@ def generate_data(self, count: int) -> list[Any]: class UniformProposer(ContinuousDistributionProposer): """Generator producing numbers in a uniform distribution.""" - expected_buckets = [ - 0, - 0.06698, - 0.14434, - 0.14434, - 0.14434, - 0.14434, - 0.14434, - 0.14434, - 0.06698, - 0, - ] - def function_name(self) -> str: """Get the name of the generator function to call.""" return "dist_gen.uniform_ms" @@ -199,31 +164,11 @@ def get_proposers( class LogNormalProposer(Proposer): """Generator producing numbers in a log-normal distribution.""" - # R: - # > xs<-seq(-2,2,0.5)*sqrt((exp(1)-1)*exp(1))+exp(0.5) - # > ys <- plnorm(xs) - # > c(ys, 1) - c(0,ys) - # [1] 0.00000000 0.00000000 0.00000000 0.28589471 0.40556775 0.15086088 - # [7] 0.06716451 0.03428958 0.01924848 0.03697409 - expected_buckets = [ - 0, - 0, - 0, - 0.28589471, - 0.40556775, - 0.15086088, - 0.06716451, - 0.03428958, - 0.01924848, - 0.03697409, - ] - # pylint: disable=too-many-arguments too-many-positional-arguments def __init__( self, table: Table, column: Column, - buckets: Buckets, logmean: float, logstddev: float, dialect: Dialect, @@ -232,7 +177,6 @@ def __init__( super().__init__() self.table = table self.column = column - self.buckets = buckets self.logmean = logmean self.logstddev = logstddev self._dialect = dialect @@ -290,17 +234,11 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: }, } - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # if self.buckets is None: - # return default - # return self.buckets.fit_from_counts(self.expected_buckets) - class ContinuousLogDistributionProposerFactory(ContinuousDistributionProposerFactory): """All generators that want an average and standard deviation of log data.""" - def _get_generators_from_buckets( + def _get_generators_from_buckets( # pylint: disable=unused-argument self, engine: Engine, src_table: Table, @@ -323,7 +261,6 @@ def _get_generators_from_buckets( LogNormalProposer( src_table, column, - buckets, float(result.logmean), float(result.logstddev), dialect=engine.dialect, @@ -395,10 +332,6 @@ def generate_data(self, count: int) -> list[Any]: for _ in range(count) ] - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # return default - class MultivariateNormalGeneratorFactoryBase(ProposerFactory): """Generator factory that makes distributions and maybe partitions.""" diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index 6a3a1122..2cb76b81 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -9,7 +9,6 @@ from datafaker.dialects import Random, SecondsDifference, StdDev from datafaker.proposers.base import ( - Buckets, ForeignKeyRelationship, Proposer, ProposerFactory, @@ -184,7 +183,6 @@ def __init__( column: Column, anchor_related: RelatedColumn, engine: Engine, - buckets: Buckets | None = None, ): """ Initialise a date after proposer. @@ -196,7 +194,6 @@ def __init__( :param anchor: The anchor column, which must be in the same table as column or a related table. :param engine: The source database. - :param buckets: The buckets for the intervals in the source database. """ super().__init__() self._sd = sd @@ -205,13 +202,6 @@ def __init__( self._column = column self._engine = engine self._provider = AnchoredProvider(metadata=metadata) - if buckets is None: - self._fit = None - return - intervals = self.generate_intervals(400) - self._fit = buckets.fit_from_values( - [(b - coerce_to_datetime(a, b)).total_seconds() for (a, b) in intervals] - ) @classmethod def get_query( @@ -361,10 +351,6 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: } } - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # return default if self._fit is None else self._fit - def generate_data(self, count: int) -> list[datetime.datetime]: """Generate ``count`` random data points for this column.""" return [b for (_, b) in self.generate_intervals(count)] @@ -416,12 +402,6 @@ def _make_date_after_proposers( result = connection.execute(query).first() if result is None or result.sd is None: return [] - buckets = Buckets.make_buckets( - engine, - column.table, - SecondsDifference(column, anchor_related.target), - anchor_related, - ) return [ DateAfterProposer( self._metadata, @@ -430,7 +410,6 @@ def _make_date_after_proposers( column, anchor_related, engine=engine, - buckets=buckets, ) ] @@ -537,10 +516,6 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: """No source data required.""" return {} - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # return default - def generate_data(self, count: int) -> list[Any]: """Generate ``count`` random data points for this column.""" anchor = self._anchor_related.target diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index 64c033f5..fb93a275 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -1,4 +1,5 @@ """Unit tests for datafaker.evaluators.proposal_ranking.""" + from typing import Any from sqlalchemy import Column, MetaData, Table, Text @@ -230,7 +231,7 @@ def test_keyword_boost_can_flip_the_recommendation(self) -> None: def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" table = Table("t", MetaData(), Column("c", Text())) - choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"]) results = [ make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), make_result(overall_score=0.5, copy_fraction=0.0), @@ -311,7 +312,7 @@ def test_combined_score_is_clamped_to_one(self) -> None: def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: """A heavily-discounted best-fit candidate triggers the fallback warning.""" table = Table("t", MetaData(), Column("c", Text())) - choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"]) results = [ # Fits almost perfectly but is a resampler on a near-unique column: # gets discounted to (near) zero. diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index 89191868..b22a765c 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -12,6 +12,7 @@ docker compose up -d mssql # wait ~30 s for SQL Server to start """ + import asyncio import os from tempfile import mkstemp @@ -50,7 +51,7 @@ def setUp(self) -> None: super().setUp() # Write orm.yaml so generate_data() has the file it expects. - (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) + self.orm_fd, self.orm_file_path = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as fh: fh.write( make_tables_file( @@ -126,7 +127,6 @@ def test_dialect_rand(self) -> None: proposer = ZipfChoiceProposer( column=table.c.name, values=["Blender", "Gibbs"], - counts=[5, 5], sample_count=2, dialect=dialect, ) diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index 99261c12..b9460492 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -1,4 +1,5 @@ """Tests for dialect-correct SQL in generator classes.""" + # pylint: disable=protected-access import unittest import unittest.mock @@ -121,9 +122,9 @@ def capture_execute(stmt, *args, **kwargs): self.assertEqual(len(executed_stmts), 1) compiled = str( executed_stmts[0].compile( - dialect=mssql.dialect() - if dialect_name == "mssql" - else postgresql.dialect(), + dialect=( + mssql.dialect() if dialect_name == "mssql" else postgresql.dialect() + ), compile_kwargs={"literal_binds": True}, ) ) @@ -154,7 +155,6 @@ def _make_gen(self, dialect, sample_count=None, suppress_count=0): return ZipfChoiceProposer( column=table.c.gender, values=["M", "F"], - counts=[70, 30], sample_count=sample_count, suppress_count=suppress_count, dialect=dialect, @@ -544,9 +544,7 @@ def test_lognormal_postgresql_uses_stddev(self) -> None: """LogNormalProposer.select_aggregate_clauses uses STDDEV on PostgreSQL.""" tbl, col = self._make_table() - proposer = LogNormalProposer( - tbl, col, MagicMock(), 1.0, 0.5, dialect=postgresql.dialect() - ) + proposer = LogNormalProposer(tbl, col, 1.0, 0.5, dialect=postgresql.dialect()) clause = proposer.select_aggregate_clauses()["logstddev__age"]["clause"] self.assert_str_in("STDDEV", clause.upper()) @@ -554,9 +552,7 @@ def test_lognormal_mssql_uses_stdev(self) -> None: """LogNormalProposer.select_aggregate_clauses uses STDEV on MSSQL.""" tbl, col = self._make_table() - proposer = LogNormalProposer( - tbl, col, MagicMock(), 1.0, 0.5, dialect=mssql.dialect() - ) + proposer = LogNormalProposer(tbl, col, 1.0, 0.5, dialect=mssql.dialect()) clause = proposer.select_aggregate_clauses()["logstddev__age"]["clause"] self.assert_str_in("STDEV", clause.upper()) self.assert_str_not_in("STDDEV", clause.upper()) From e0d0618c820f3950a70b059a532a173657952a6a Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Thu, 19 Nov 2026 11:17:06 +0000 Subject: [PATCH 4/7] cleaned up --- datafaker/interactive/generators.py | 1 - datafaker/proposers/base.py | 74 ----------------------------- datafaker/proposers/extract.py | 5 -- 3 files changed, 80 deletions(-) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index cb7b090b..2dc3a66e 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -801,7 +801,6 @@ def _get_proposer_proposals(self) -> list[Proposer]: props = everything_factory(self.config, self.metadata).get_proposers( columns, self.sync_engine ) - # sorted_props = sorted(props, key=lambda g: g.fit(9999)) sorted_props = sorted(props, key=lambda g: g.name()) self.proposers = sorted_props self.proposers_valid_columns = ( diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 351c76d1..b778c9bc 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -139,15 +139,6 @@ def actual_kwargs(self) -> dict[str, Any]: def generate_data(self, count: int) -> list[Any]: """Generate ``count`` random data points for this column.""" - # def fit(self, default: float = -1) -> float: - # """ - # Return a value representing how well the distribution fits the real source data. - - # 0.0 means "perfectly". - # Returns default if no fitness has been defined. - # """ - # return default - def guarantees_fresh_uniqueness(self) -> bool: """ Whether this proposer is structurally guaranteed never to collide. @@ -301,13 +292,6 @@ def get_proposers( """Get the proposers appropriate to these columns.""" -def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> float: - """Calculate the fit by comparing a pair of lists of buckets.""" - sum_diff_squared = sum(map(lambda t, a: (t - a) * (t - a), xs, ys)) - count = len(ys) - return sum_diff_squared / (count * count) - - class ForeignKeyRelationship: """How a table is related.""" @@ -406,51 +390,12 @@ class Buckets: # pylint: disable=too-many-arguments too-many-positional-arguments def __init__( self, - engine: Engine, - table: Table | Join, - column: Any, mean: float, stddev: float, - count: int, - anchor_relation: RelatedColumn | None = None, ): """Initialise a Buckets object.""" self.mean = mean self.stddev = stddev - query = self._get_bucket_query(table, column, anchor_relation) - with engine.connect() as connection: - raw_buckets = connection.execute(query.group_by("b")) - self.buckets: Sequence[int] = [0] * 10 - for rb in raw_buckets: - try: - x = float(rb.b) - if x.is_integer(): - bucket = min(9, max(0, int(x) + 1)) - self.buckets[bucket] += rb.f / count - except TypeError: - # We get a type error if there are no rows returned at all - # because rb.b is None in this case. - # We could just test for None explicitly, but this way - # catches errors if SQLAlchemy returns something that - # isn't a number for some other unknown reason. - pass - - def _get_bucket_query( - self, - table: Table | Join, - column: Any, - anchor_relation: RelatedColumn | None = None, - ) -> Any: - """Make the query for half-sd buckets.""" - bottom = self.mean - 2 * self.stddev - width = self.stddev / 2 - query = select( - func.count(column).label("f"), # pylint: disable=not-callable - ((func.floor(column) - bottom) / width).label("b"), - ).select_from(table) - if anchor_relation is not None: - query = anchor_relation.add_join_to_query(query) - return query @classmethod def make_buckets( @@ -488,33 +433,14 @@ def make_buckets( return None try: buckets = cls( - engine, - table, - column, result.mean, result.stddev, - getattr(result, "count"), - anchor_relation, ) except DatabaseError as exc: logger.debug("Failed to instantiate Buckets object: %s", exc) return None return buckets - def fit_from_counts(self, bucket_counts: Sequence[float]) -> float: - """Figure out the fit from bucket counts from the generator distribution.""" - return fit_from_buckets(self.buckets, bucket_counts) - - def fit_from_values(self, values: list[float]) -> float: - """Figure out the fit from samples from the generator distribution.""" - buckets = [0] * 10 - x = self.mean - 2 * self.stddev - w = self.stddev / 2 - for v in values: - b = min(9, max(0, int((v - x) / w))) - buckets[b] += 1 - return self.fit_from_counts(buckets) - class MultiProposerFactory(ProposerFactory): """A composite factory.""" diff --git a/datafaker/proposers/extract.py b/datafaker/proposers/extract.py index 8e8b59c2..659de32a 100644 --- a/datafaker/proposers/extract.py +++ b/datafaker/proposers/extract.py @@ -60,11 +60,6 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: """Return that no data is required from the database.""" return {} - # def fit(self, default: float = -1) -> float: - # """Get this generator's fit against the real data.""" - # # What should the fit really be? - # return 0.0 - def get_based_on_data(self, count: int) -> list[datetime.datetime]: """Get the data from the based-on column in the source data.""" with self._engine.connect() as conn: From f0b57779d0f7d1fad82c4831d5747f87f4e972a2 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Thu, 19 Nov 2026 14:12:10 +0000 Subject: [PATCH 5/7] a bit more cleaned up --- datafaker/proposers/partitioned.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datafaker/proposers/partitioned.py b/datafaker/proposers/partitioned.py index 6f8bc669..53feb84e 100644 --- a/datafaker/proposers/partitioned.py +++ b/datafaker/proposers/partitioned.py @@ -340,10 +340,6 @@ def generate_data(self, count: int) -> list[Any]: kwargs = self.actual_kwargs() return [dist_gen.alternatives(**kwargs) for _ in range(count)] - def fit(self, default: float = -1) -> float: - """Get this generator's fit against the real data.""" - return default - def is_numeric(col: Column) -> bool: """Test if this column stores a numeric value.""" From df65708741dbec8813264feadb6f3f52ea4c5c44 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Fri, 20 Nov 2026 16:23:52 +0000 Subject: [PATCH 6/7] precommit from merged --- datafaker/proposers/choice.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index b1f5f054..007cfe6b 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -6,11 +6,7 @@ from sqlalchemy import Column, CursorResult, Engine, desc, func, select from datafaker.dialects import random_sample -from datafaker.proposers.base import ( - Proposer, - ProposerFactory, - dist_gen, -) +from datafaker.proposers.base import Proposer, ProposerFactory, dist_gen NumericType = Union[int, float] From 523b4903d51660793d0f54fbf19e6ba8fe454b59 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Fri, 20 Nov 2026 19:19:38 +0000 Subject: [PATCH 7/7] fix bugs --- datafaker/proposers/continuous.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafaker/proposers/continuous.py b/datafaker/proposers/continuous.py index 54e94b44..20ebe91a 100644 --- a/datafaker/proposers/continuous.py +++ b/datafaker/proposers/continuous.py @@ -246,12 +246,12 @@ def _get_generators_from_buckets( # pylint: disable=unused-argument buckets: Buckets, ) -> Sequence[Proposer]: col = case( - (column > 0, func.log(column)), + (column > 0, LogNatural(column)), else_=null(), ) stmt = select( func.avg(col).label("logmean"), - func.stddev_samp(col).label("logstddev"), + StdDev(col).label("logstddev"), ).select_from(src_table) with engine.connect() as connection: result = connection.execute(stmt).first()