Skip to content

fix: skip integer cleaning for unhashable columns #954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lntutor wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: skip integer cleaning for unhashable columns #954

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions buckaroo/customizations/pd_stats_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,8 @@ def histogram(value_counts: pd.Series, nan_per: float, is_numeric: bool, length:
def pd_cleaning_stats(value_counts: pd.Series, length: int) -> PdCleaningResult:
"""Compute int parsing stats for cleaning."""
vc = value_counts
if vc.empty:
return {'int_parse_fail': 1.0, 'int_parse': 0.0}
coerced_ser = pd.to_numeric(vc.index.values, errors='coerce', downcast='integer', dtype_backend='pyarrow')
nan_sum = (pd.Series(coerced_ser).isna() * 1 * vc.values).sum()
return {'int_parse_fail': nan_sum / length, 'int_parse': (length - nan_sum) / length}
Expand Down
15 changes: 13 additions & 2 deletions tests/unit/test_pd_stats_v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline
from buckaroo.pluggable_analysis_framework.utils import PERVERSE_DF

from buckaroo.customizations.pd_stats_v2 import (typing_stats, _type, base_summary_stats, numeric_stats, computed_default_summary_stats, histogram_series, histogram, pd_cleaning_stats, heuristic_fracs, orig_col_name, PD_ANALYSIS_V2)
from buckaroo.customizations.pd_stats_v2 import (typing_stats, _type, base_summary_stats, numeric_stats, computed_default_summary_stats, histogram_series, histogram, pd_cleaning_stats, heuristic_fracs, orig_col_name, PD_ANALYSIS_V2, PD_AUTOCLEAN_DEFAULT_V2)


# ============================================================================
Expand Down Expand Up @@ -335,6 +335,18 @@ def test_string_column(self):
assert result['int_parse'] > 0
assert result['int_parse_fail'] > 0

def test_unhashable_column_is_not_integer_parseable(self):
result = pd_cleaning_stats(pd.Series(dtype=object), length=4)
assert result['int_parse'] == 0.0
assert result['int_parse_fail'] == 1.0

def test_default_autoclean_skips_safe_int_for_unhashable_column(self):
pipeline = StatPipeline(PD_AUTOCLEAN_DEFAULT_V2, unit_test=False)
summary, _ = pipeline.process_df(pd.DataFrame({'lists': [['a'], ['b']]}))
column_summary = next(iter(summary.values()))
assert column_summary['orig_col_name'] == 'lists'
assert column_summary['cleaning_ops'] == []


# ============================================================================
# Tests: heuristic_fracs
Expand Down Expand Up @@ -436,4 +448,3 @@ def test_unit_test_runs(self):
pipeline = StatPipeline(PD_ANALYSIS_V2, unit_test=True)
passed, errors = pipeline._unit_test_result
# Some errors may occur on edge cases, but shouldn't crash