Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions datafaker/interactive/generators.py
Original file line number Diff line number Diff line change
Expand Up @@ -790,7 +790,7 @@ def _proposers_valid(self) -> bool:

def _get_proposer_proposals(self) -> list[Proposer]:
"""
Get a list of acceptable proposers, sorted by decreasing fit to the actual data.
Get a list of acceptable proposers, sorted by name.

The cached list from the most recent 'propose'/'propose all' is
reused if still valid for the current table/columns, so that 'set
Expand All @@ -801,7 +801,7 @@ def _get_proposer_proposals(self) -> list[Proposer]:
props = everything_factory(self.config, self.metadata).get_proposers(
columns, self.sync_engine
)
sorted_props = sorted(props, key=lambda g: g.fit(9999))
sorted_props = sorted(props, key=lambda g: g.name())
self.proposers = sorted_props
self.proposers_valid_columns = (
self.table_index,
Expand Down
74 changes: 0 additions & 74 deletions datafaker/proposers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,15 +139,6 @@ def actual_kwargs(self) -> dict[str, Any]:
def generate_data(self, count: int) -> list[Any]:
"""Generate ``count`` random data points for this column."""

def fit(self, default: float = -1) -> float:
"""
Return a value representing how well the distribution fits the real source data.

0.0 means "perfectly".
Returns default if no fitness has been defined.
"""
return default

def guarantees_fresh_uniqueness(self) -> bool:
"""
Whether this proposer is structurally guaranteed never to collide.
Expand Down Expand Up @@ -301,13 +292,6 @@ def get_proposers(
"""Get the proposers appropriate to these columns."""


def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> float:
"""Calculate the fit by comparing a pair of lists of buckets."""
sum_diff_squared = sum(map(lambda t, a: (t - a) * (t - a), xs, ys))
count = len(ys)
return sum_diff_squared / (count * count)


class ForeignKeyRelationship:
"""How a table is related."""

Expand Down Expand Up @@ -406,51 +390,12 @@ class Buckets:
# pylint: disable=too-many-arguments too-many-positional-arguments
def __init__(
self,
engine: Engine,
table: Table | Join,
column: Any,
mean: float,
stddev: float,
count: int,
anchor_relation: RelatedColumn | None = None,
):
"""Initialise a Buckets object."""
self.mean = mean
self.stddev = stddev
query = self._get_bucket_query(table, column, anchor_relation)
with engine.connect() as connection:
raw_buckets = connection.execute(query.group_by("b"))
self.buckets: Sequence[int] = [0] * 10
for rb in raw_buckets:
try:
x = float(rb.b)
if x.is_integer():
bucket = min(9, max(0, int(x) + 1))
self.buckets[bucket] += rb.f / count
except TypeError:
# We get a type error if there are no rows returned at all
# because rb.b is None in this case.
# We could just test for None explicitly, but this way
# catches errors if SQLAlchemy returns something that
# isn't a number for some other unknown reason.
pass

def _get_bucket_query(
self,
table: Table | Join,
column: Any,
anchor_relation: RelatedColumn | None = None,
) -> Any:
"""Make the query for half-sd buckets."""
bottom = self.mean - 2 * self.stddev
width = self.stddev / 2
query = select(
func.count(column).label("f"), # pylint: disable=not-callable
((func.floor(column) - bottom) / width).label("b"),
).select_from(table)
if anchor_relation is not None:
query = anchor_relation.add_join_to_query(query)
return query

@classmethod
def make_buckets(
Expand Down Expand Up @@ -488,33 +433,14 @@ def make_buckets(
return None
try:
buckets = cls(
engine,
table,
column,
result.mean,
result.stddev,
getattr(result, "count"),
anchor_relation,
)
except DatabaseError as exc:
logger.debug("Failed to instantiate Buckets object: %s", exc)
return None
return buckets

def fit_from_counts(self, bucket_counts: Sequence[float]) -> float:
"""Figure out the fit from bucket counts from the generator distribution."""
return fit_from_buckets(self.buckets, bucket_counts)

def fit_from_values(self, values: list[float]) -> float:
"""Figure out the fit from samples from the generator distribution."""
buckets = [0] * 10
x = self.mean - 2 * self.stddev
w = self.stddev / 2
for v in values:
b = min(9, max(0, int((v - x) / w)))
buckets[b] += 1
return self.fit_from_counts(buckets)


class MultiProposerFactory(ProposerFactory):
"""A composite factory."""
Expand Down
78 changes: 1 addition & 77 deletions datafaker/proposers/choice.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,12 @@
"""Generator factories for making generators for choices of values."""

import decimal
import math
import typing
from abc import abstractmethod
from typing import Any, Sequence, Union

from sqlalchemy import Column, CursorResult, Engine, desc, func, select

from datafaker.dialects import random_sample
from datafaker.proposers.base import (
Proposer,
ProposerFactory,
dist_gen,
fit_from_buckets,
)
from datafaker.proposers.base import Proposer, ProposerFactory, dist_gen

NumericType = Union[int, float]

Expand All @@ -23,28 +15,6 @@
MAXIMUM_CHOICES = 500


def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None]:
"""
Get a zipf distribution for a certain number of items.

:param total: The total number of items to be distributed.
:param bins: The total number of bins to distribute the items into.
:return: A generator of the number of items in each bin, from the
largest to the smallest.
"""
basic_dist = list(map(lambda n: 1 / n, range(1, bins + 1)))
bd_remaining = sum(basic_dist)
for b in basic_dist:
# yield b/bd_remaining of the `total` remaining
if bd_remaining == 0:
yield 0
else:
x = math.floor(0.5 + total * b / bd_remaining)
bd_remaining -= x * bd_remaining / total
total -= x
yield x


def _choice_stmt( # pylint: disable=R0913,R0917
column: Column,
store_counts: bool,
Expand Down Expand Up @@ -100,7 +70,6 @@ def __init__(
self,
column: Column,
values: list[Any],
counts: list[int],
sample_count: int | None = None,
suppress_count: int = 0,
dialect: Any = None,
Expand All @@ -109,8 +78,6 @@ def __init__(
super().__init__()
self.column = column
self.values = values
estimated_counts = self.get_estimated_counts(counts)
self._fit = fit_from_buckets(counts, estimated_counts)

extra_comment = " and their counts" if self.STORE_COUNTS else ""
stmt = _choice_stmt(
Expand Down Expand Up @@ -152,10 +119,6 @@ def __init__(
)
self._annotation = "sampled and suppressed"

@abstractmethod
def get_estimated_counts(self, counts: list[int]) -> list[int]:
"""Get the counts that we would expect if this distribution was the correct one."""

def nominal_kwargs(self) -> dict[str, Any]:
"""Get the arguments to be entered into ``config.yaml``."""
return {
Expand Down Expand Up @@ -186,18 +149,10 @@ def custom_queries(self) -> dict[str, dict[str, Any]]:
},
}

def fit(self, default: float = -1) -> float:
"""Get this generator's fit against the real data."""
return default if self._fit is None else self._fit


class ZipfChoiceProposer(ChoiceProposer):
"""Generator producing items in a Zipf distribution."""

def get_estimated_counts(self, counts: list[int]) -> list[int]:
"""Get the counts that we would expect if this distribution was the correct one."""
return list(zipf_distribution(sum(counts), len(counts)))

def function_name(self) -> str:
"""Get the name of the generator function to call."""
return "dist_gen.zipf_choice"
Expand All @@ -210,28 +165,9 @@ def generate_data(self, count: int) -> list[float]:
]


def uniform_distribution(total: int, bins: int) -> typing.Generator[int, None, None]:
"""
Construct a distribution putting ``total`` items uniformly into ``bins`` bins.

If they don't fit exactly evenly, the earlier bins will have one more
item than the later bins so the total is as required.
"""
p = total // bins
n = total % bins
for _ in range(0, n):
yield p + 1
for _ in range(n, bins):
yield p


class UniformChoiceProposer(ChoiceProposer):
"""A proposer producing values, each roughly as frequently as each other."""

def get_estimated_counts(self, counts: list[int]) -> list[int]:
"""Get the counts that we would expect if this distribution was the correct one."""
return list(uniform_distribution(sum(counts), len(counts)))

def function_name(self) -> str:
"""Get the name of the generator function to call."""
return "dist_gen.choice"
Expand Down Expand Up @@ -347,19 +283,16 @@ def get_proposers( # pylint: disable=too-many-locals
ZipfChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
WeightedChoiceProposer(
column,
vg.cvs,
vg.counts,
dialect=dialect,
),
]
Expand All @@ -368,21 +301,18 @@ def get_proposers( # pylint: disable=too-many-locals
ZipfChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
WeightedChoiceProposer(
column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
Expand All @@ -406,21 +336,18 @@ def get_proposers( # pylint: disable=too-many-locals
ZipfChoiceProposer(
column,
vg.values,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
),
WeightedChoiceProposer(
column,
vg.cvs,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
),
Expand All @@ -430,23 +357,20 @@ def get_proposers( # pylint: disable=too-many-locals
ZipfChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
WeightedChoiceProposer(
column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
Expand Down
Loading