Adding Fuzzy Match Rule for DuckDB

[eden-chan]

I want to understand how to add custom rules for blocking rules that use fuzzy matching. Would this implementation be possible?

# TODO: create a fuzzy match rule using 
# https://duckdb.org/docs/sql/functions/char.html
# damerau_levenshtein('duckdb', 'udckbd')
def fuzzy_match_rule(
    col_name: str,
    _sql_dialect: str,
    salting_partitions: int = None,
    threshold: int = 10
) -> BlockingRule:
    """Represents an fuzzy match blocking rule.
    Args:
        col_name (str): Input column name, or a str represent a sql
            statement you'd like to match on. For example, `surname` or
            `"substr(surname,1,2)"` are both valid.
        salting_partitions (optional, int): Whether to add salting
            to the blocking rule. More information on salting can
            be found within the docs. Salting is currently only valid
            for Spark.
        threshold (optional, int): edit distance tolerance when comparing columns
    """
    warnings.warn(
        "`exact_match_rule` is deprecated; use `block_on`",
        DeprecationWarning,
        stacklevel=2,
    )

    syntax_tree = sqlglot.parse_one(col_name, read=_sql_dialect)

    l_col = add_quotes_and_table_prefix(syntax_tree, "l").sql(_sql_dialect)
    r_col = add_quotes_and_table_prefix(syntax_tree, "r").sql(_sql_dialect)

    blocking_rule = f"damerau_levenshtein({l_col}, {r_col}) <= {threshold}"

    return blocking_rule_to_obj(
        {
            "blocking_rule": blocking_rule,
            "salting_partitions": salting_partitions,
            "sql_dialect": _sql_dialect,
        }
    )
    
 fuzzy_match_rule = partial(fuzzy_match_rule, _sql_dialect="duckdb")

# TODO: create a fuzzy match rule using 
# https://duckdb.org/docs/sql/functions/char.html
# damerau_levenshtein('duckdb', 'udckbd')
def fuzzy_block_on(
    col_names: list[str],
    salting_partitions: int = 1,
    threshold: int = 10 
) -> BlockingRule:
    fuzzy_match_rule.keywords['threshold'] = threshold
    return _block_on_(
        fuzzy_match_rule,
        col_names,
        salting_partitions,
    )

[RobinL]

A blocking rule can be an aribtrary sql expression like:

settings = {
    "link_type": "link_only",
    "blocking_rules_to_generate_predictions": [
        "l.first_name = r.first_name and jaro_winkler_similarity(l.surname, r.surname)"
        block_on("surname"),
    ],
    "comparisons": [
        ctl.name_comparison("first_name",),
        ctl.name_comparison("surname"),
        ctl.date_comparison("dob", cast_strings_to_date=True),
        cl.exact_match("city", term_frequency_adjustments=True),
        ctl.email_comparison("email", include_username_fuzzy_level=False),
    ],       
}

However, it's not generally recommended because it impacts performance:
https://moj-analytical-services.github.io/splink/topic_guides/blocking/performance.html?h=equi#efficient-blocking-rules