Repository: ipums/hlink (language: python)

Issue:
Allow 1 or 3+ columns as input in the array feature selection
Right now the `array` feature selection only allows combining exactly two input columns into an output column. To make this more flexible, we could support passing any number of columns, with a minimum of 1. This should be a small change in hlink/linking/core/transforms.py, where we unpack `feature_selection["input_columns"]` with

```
col1, col2 = feature_selection["input_columns"]
```

The `pyspark.sql.functions.array()` function which we're using accepts a variable number of arguments.


Discussion / hints:

Fix patch:
diff --git a/docs/_sources/feature_selection_transforms.md.txt b/docs/_sources/feature_selection_transforms.md.txt
index 0e7e332..c87440a 100644
--- a/docs/_sources/feature_selection_transforms.md.txt
+++ b/docs/_sources/feature_selection_transforms.md.txt
@@ -46,10 +46,10 @@ transform = "sql_condition"
 
 ## array
 
-Combine two input columns into an array output column.
+Combine any number of input columns into a single array output column.
 
 * Attributes:
-  * `input_columns` - Type: list of strings. Required. The two input columns.
+  * `input_columns` - Type: list of strings. Required. The list of input columns.
   * `output_column` - Type: `string`. Required.
 
 ```
diff --git a/docs/feature_selection_transforms.html b/docs/feature_selection_transforms.html
index bef511e..075d3f2 100644
--- a/docs/feature_selection_transforms.html
+++ b/docs/feature_selection_transforms.html
@@ -83,11 +83,11 @@ <h2>sql_condition<a class="headerlink" href="#sql-condition" title="Link to this
 </section>
 <section id="array">
 <h2>array<a class="headerlink" href="#array" title="Link to this heading">¶</a></h2>
-<p>Combine two input columns into an array output column.</p>
+<p>Combine any number of input columns into a single array output column.</p>
 <ul class="simple">
 <li><p>Attributes:</p>
 <ul>
-<li><p><code class="docutils literal notranslate"><span class="pre">input_columns</span></code> - Type: list of strings. Required. The two input columns.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">input_columns</span></code> - Type: list of strings. Required. The list of input columns.</p></li>
 <li><p><code class="docutils literal notranslate"><span class="pre">output_column</span></code> - Type: <code class="docutils literal notranslate"><span class="pre">string</span></code>. Required.</p></li>
 </ul>
 </li>
diff --git a/docs/searchindex.js b/docs/searchindex.js
index c64d96a..f74a1bf 100644
--- a/docs/searchindex.js
+++ b/docs/searchindex.js
@@ -1,1 +1,1 @@
-Search.setIndex({"docnames": ["column_mappings", "comparison_types", "config", "feature_selection_transforms", "index", "installation", "introduction", "link_tasks", "models", "pipeline_features", "running_the_program", "substitutions", "use_examples"], "filenames": ["column_mappings.md", "comparison_types.md", "config.md", "feature_selection_transforms.md", "index.rst", "installation.md", "introduction.md", "link_tasks.md", "models.md", "pipeline_features.md", "running_the_program.md", "substitutions.md", "use_examples.md"], "titles": ["Column Mappings", "Comparison types, transform add-ons, aggregate features, and household aggregate features", "Configuration", "Feature Selection transforms", "Welcome to hlink\u2019s documentation!", "Installation", "Introduction", "Link Tasks", "Models", "Pipeline generated features", "Running hlink", "Substitutions", "Advanced Workflow Examples"], "terms": {"each": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10], "read": [0, 1, 2, 7, 10], "from": [0, 1, 2, 4, 6, 7, 8, 9, 10, 12], "input": [0, 1, 2, 3, 6, 7, 10, 11], "dataset": [0, 1, 2, 6, 7, 10, 12], "hlink": [0, 1, 2, 5, 6, 7, 12], "It": [0, 1, 2, 6, 10, 12], "ha": [0, 1, 2, 6, 10, 12], "column_nam": [0, 1, 2, 11], "attribut": [0, 1, 2, 3, 7, 8, 9, 10, 11], "which": [0, 1, 2, 3, 6, 7, 9, 10, 12], "specifi": [0, 1, 2, 6, 7, 9, 10, 11], "name": [0, 1, 2, 10, 11], "both": [0, 1, 2, 7, 12], "option": [0, 1, 2, 3, 6, 7, 8, 10, 12], "mai": [0, 2, 6, 7, 10], "have": [0, 1, 2, 5, 6, 7, 8, 10, 12], "an": [0, 1, 2, 3, 6, 8, 10], "alia": [0, 2, 7], "give": [0, 2], "new": [0, 2, 12], "us": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12], "support": [0, 7, 8], "some": [0, 1, 2, 3, 6, 7, 10], "make": [0, 1, 2, 5, 12], "chang": [0, 1, 2, 5, 10, 12], "data": [0, 1, 4, 6, 7, 10], "thei": [0, 1, 2, 7, 10], "ar": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12], "These": [0, 1, 2, 3, 6, 7, 8, 9], "clean": [0, 6], "harmon": 0, "The": [0, 1, 2, 3, 5, 7, 8, 9, 10, 12], "avail": [0, 1, 2, 3, 5, 7, 8, 9, 12], "list": [0, 1, 2, 3, 4, 8, 10, 11], "below": [0, 1, 2, 3, 8, 9, 10], "section": [0, 1, 2, 12], "By": [0, 2, 10], "default": [0, 1, 2, 7, 8, 10], "must": [0, 1, 2, 8, 9, 11], "same": [0, 1, 2, 6, 7, 10], "With": [0, 9], "override_column_a": [0, 2, 3], "override_column_b": [0, 2, 3], "you": [0, 1, 2, 5, 10, 11, 12], "can": [0, 1, 2, 5, 6, 7, 8, 10, 12], "differ": [0, 1, 2, 4, 6, 7], "either": [0, 1, 2, 6, 11], "A": [0, 1, 2, 9, 10], "b": [0, 1, 2, 10], "when": [0, 1, 2, 3, 7, 12], "do": [0, 1, 3, 10, 12], "thi": [0, 1, 2, 5, 6, 7, 9, 10, 12], "appli": [0, 2, 3, 7, 12], "onli": [0, 1, 2, 7, 12], "non": 0, "overrid": [0, 2], "also": [0, 1, 2, 5, 6, 7, 9, 10, 12], "provid": [0, 2, 3, 6, 7, 9, 10], "override_transform": [0, 2], "describ": [0, 2, 10], "type": [0, 2, 3, 4, 7, 8, 10, 11, 12], "oper": [0, 2], "singl": [0, 2, 10, 12], "output": [0, 1, 2, 3, 6, 7, 10, 12], "more": [0, 1, 2, 9, 10, 12], "than": [0, 1, 2, 8], "one": [0, 1, 2, 7], "order": [0, 2, 7], "so": [0, 1, 2, 5, 12], "anoth": [0, 1, 3, 7], "format": 0, "letter": 0, "t": [0, 1, 2, 3, 12], "u": 0, "repres": [0, 1, 2, 3, 9, 10], "arbitrari": 0, "requir": [0, 1, 2, 3, 4, 7, 9, 10, 11], "addit": [0, 1, 2, 3, 5, 6, 10], "vari": [0, 2], "inform": [0, 1, 2, 10], "appear": [0, 1], "its": [0, 1, 6, 10], "suffix": 0, "mean": [0, 2], "two": [0, 1, 2, 3, 6, 7, 9, 10, 12], "link": [0, 1, 2, 4, 6, 8], "most": [0, 1, 7, 10], "independ": [0, 2], "For": [0, 1, 2, 7, 10, 12], "exampl": [0, 1, 2], "taken": [0, 1], "10": [0, 2, 5, 12], "year": [0, 1, 2, 3, 4], "apart": 0, "want": [0, 1, 2, 10, 12], "standard": [0, 1, 11], "ag": [0, 1, 2, 3], "variabl": [0, 1, 2, 12], "i": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12], "compar": [0, 1, 2, 6, 7], "between": [0, 1, 2, 6, 7, 10, 12], "To": [0, 1, 5, 7, 10], "could": [0, 2], "creat": [0, 2, 6, 7, 9, 10, 11, 12], "age_at_dataset_b": 0, "ad": [0, 1, 2], "column_map": [0, 2, 7], "valu": [0, 1, 2, 8, 9, 10, 11], "As": 0, "suppos": 0, "record": [0, 1, 2, 6, 7], "person": [0, 1, 6], "": [0, 1, 2, 6, 7, 10, 11], "first": [0, 1, 2, 5, 7, 10, 11], "string": [0, 1, 2, 3, 7, 8, 10, 11], "In": [0, 1, 6, 10, 12], "call": 0, "namefrst": [0, 1, 2], "entir": 0, "lowercas": 0, "first_nam": 0, "uppercas": 0, "follow": [0, 1, 6, 10, 11, 12], "configur": [0, 1, 6, 10, 12], "add": [0, 4], "_": [0, 1, 2, 3, 4, 8, 9, 10], "given": [0, 1, 2, 3, 8, 12], "numer": [0, 1], "11": [0, 2, 5, 9], "concat": 0, "concaten": [0, 1], "end": [0, 1, 2, 3, 11], "col": [0, 1], "togeth": [0, 1, 2], "take": [0, 1, 2, 3, 7, 10], "column_to_append": 0, "multipl": [0, 1, 2, 10], "time": [0, 2, 7, 10], "row": [0, 2], "If": [0, 1, 2, 3, 7, 8, 10, 11], "automat": [0, 2, 5, 7], "convert": [0, 1, 2], "befor": [0, 1, 2, 3, 5, 7], "statefip": [0, 1, 2], "counti": [0, 1], "strip": [0, 7], "alphabet": 0, "charact": 0, "lower": [0, 1], "case": [0, 1, 2, 3, 6], "white": 0, "space": [0, 2, 3, 11], "start": [0, 11], "ration": 0, "word": [0, 4], "replac": [0, 1, 4], "sinc": [0, 2], "peopl": [0, 1, 6, 10], "raw": [0, 2, 7, 10], "censu": [0, 7, 12], "contain": [0, 1, 11], "lead": 0, "better": [0, 6], "match": [0, 1, 4, 6, 10, 11, 12], "remov": 0, "qmark": 0, "hyphen": 0, "punctuat": 0, "apostroph": 0, "altern": [0, 2], "surround": 0, "all": [0, 1, 2, 3, 7, 8, 9, 10], "them": [0, 1, 2, 7], "jr": [0, 2], "sr": [0, 2], "ii": [0, 2], "iii": [0, 2], "stop": 0, "last": [0, 1, 7, 9], "street": [0, 1], "avenu": [0, 11], "blvd": 0, "circl": 0, "court": 0, "road": 0, "prefix": 0, "like": [0, 2, 7, 10], "m": [0, 1], "mr": 0, "ah": 0, "chines": 0, "condens": 0, "whitespac": [0, 7], "leav": 0, "behind": 0, "arrai": [0, 2, 4, 8, 9], "namefrst_split": [0, 2], "namefrst_clean": [0, 2], "index": [0, 5], "select": [0, 1, 4, 6, 10, 12], "element": 0, "posit": [0, 1, 2, 4, 6], "second": [0, 1, 2, 11], "1": [0, 1, 2, 4, 7, 8, 9, 10, 12], "item": 0, "set": [0, 1, 2, 3, 6, 7, 10, 12], "Then": [0, 5], "0": [0, 1, 2, 7, 8, 9, 10, 12], "initi": [0, 1, 10], "probabl": [0, 2, 8], "middl": [0, 1], "namefrst_mid_init": [0, 1], "otherwis": [0, 1, 9, 12], "known": 0, "recod": 0, "birthyr": [0, 2], "clean_birthyr": [0, 2, 3], "9999": [0, 2, 9], "1999": [0, 2], "9998": 0, "divid": 0, "int": [0, 1, 2, 3, 8], "integ": [0, 1, 2, 9], "result": [0, 1, 6, 9, 10, 12], "instanc": [0, 8], "birthplac": 0, "detail": [0, 2, 10], "version": [0, 5, 12], "gener": [0, 1, 4, 6, 7, 10], "least": [0, 1], "signific": 0, "digit": 0, "we": [0, 1, 10, 12], "simpli": [0, 2], "drop": [0, 2, 10], "100": [0, 2, 12], "round": [0, 2], "lowest": 0, "whole": [0, 6], "number": [0, 1, 2, 7, 8, 10], "floor": 0, "function": [0, 1, 2, 6, 10], "bpl": [0, 1, 2], "bpl_root": 0, "condit": [0, 1, 2, 3, 4, 7], "logic": 0, "work": [0, 1, 2, 5, 7, 10, 12], "sql": [0, 1, 2, 3, 4, 7, 10], "express": [0, 1, 2], "claus": [0, 1], "if_valu": 0, "else_valu": 0, "race": [0, 1, 2, 9, 12], "ipum": [0, 6], "code": [0, 1, 2, 5], "categori": [0, 8], "get": [0, 1, 10], "down": [0, 6, 12], "nearest": 0, "produc": [0, 10], "relat": [0, 1, 2], "hundr": 0, "300": 0, "child": [0, 8], "household": [0, 4, 6, 8, 10, 12], "head": 0, "301": 0, "302": 0, "adopt": 0, "303": 0, "step": [0, 1, 2, 6], "usual": [0, 7, 12], "need": [0, 1, 2, 7, 10, 12], "2": [0, 1, 2, 3, 7, 8, 11, 12], "spous": 0, "3": [0, 1, 2, 5, 7, 8, 9, 12], "4": [0, 1, 8], "law": 0, "5": [0, 1, 2, 8, 9, 10, 12], "parent": [0, 1, 11], "6": [0, 2, 8, 9, 12], "7": [0, 1, 2, 8, 12], "sibl": 0, "12": [0, 5], "relate_div_100": [0, 1, 2], "page": [1, 2, 10], "comparison_featur": [1, 2, 7], "along": 1, "header": [1, 2, 3, 9, 11], "context": [1, 3, 9], "relatematch": [1, 2], "comparison_typ": [1, 2], "categor": [1, 2, 8, 9], "true": [1, 2, 3, 7, 9, 11, 12], "maximum": [1, 8], "jaro": [1, 9], "winkler": [1, 9], "find": [1, 7, 12], "greatest": 1, "among": 1, "cartesian": 1, "product": [1, 6, 12], "column": [1, 3, 4, 7, 9, 10, 11, 12], "namelast": [1, 2], "would": [1, 2, 12], "return": [1, 3, 8, 10], "four": 1, "namefrst_a": 1, "namefrst_b": 1, "namelast_b": 1, "namelast_a": 1, "maximum_jw": 1, "score": [1, 2, 7, 9], "namefrst_jw": [1, 2, 12], "geograph": 1, "filter": [1, 4, 7, 11], "major": [1, 10], "locat": [1, 2, 10], "boundari": 1, "zero": 1, "jw_street": 1, "enum_dist": 1, "max": [1, 8, 10], "member": [1, 7], "neighborhood": 1, "surnam": 1, "related_individual_max_jw": 1, "namefrst_rel": 1, "assert": [1, 10], "NOT": 1, "distinct": 1, "f1": 1, "evalu": [1, 2, 6, 7, 8], "ani": [1, 2, 5, 8], "potenti": [1, 4, 7], "mismatch": 1, "queri": [1, 2], "fi": 1, "OR": 1, "mi0": 1, "mi1": 1, "THEN": 1, "els": [1, 2, 3], "first_init_col": 1, "namefrst_init": 1, "mid_init_col": 1, "namefrst_mid_init_2": 1, "f2": 1, "empti": 1, "null": [1, 2, 3], "AND": [1, 2], "individu": [1, 2, 7, 12], "mainli": 1, "caution": [1, 9], "flag": [1, 9, 10, 12], "f": [1, 10], "sp": 1, "m_caution": [1, 2, 9, 12], "mbpl": 1, "mother_birthyr": 1, "stepmom": 1, "momloc": 1, "comp_a": [1, 2], "comp_b": [1, 2], "comp_c": 1, "parent_step_chang": 1, "comp_d": 1, "check": [1, 10], "sign": 1, "boolean": [1, 2, 3, 11, 12], "form": [1, 7, 11], "cast": 1, "namelast_equal_as_int": 1, "namelast_clean": [1, 2, 3], "whether": [1, 2, 11], "join": [1, 11], "across": 1, "being": [1, 7], "exact": [1, 2], "namefrst_unstd": [1, 2], "present": [1, 2, 9], "nonzero": 1, "primarili": [1, 7], "indic": [1, 12], "kind": 1, "incompar": 1, "akin": 1, "miss": [1, 10], "see": [1, 2, 5, 10, 12], "univers": [1, 4, 7], "similar": 1, "fbpl_nomatch": 1, "fbpl": 1, "allow": [1, 2, 7, 12], "up": [1, 2, 10, 11], "sub": 1, "object": [1, 2, 6, 10], "document": [1, 8, 10, 12], "sp_caution": [1, 2, 12], "spouse_bpl": 1, "spouse_birthyr": 1, "durmarr": [1, 2], "new_marr": [1, 2], "street_jw": [1, 2, 12], "9": 1, "multipli": 1, "after": [1, 2, 4, 8, 10], "float": [1, 2, 8], "comp": 1, "c": 1, "sploc": 1, "012": 1, "fals": [1, 2, 3, 4, 6, 10], "d": 1, "under": [1, 2], "specif": [1, 2, 10], "circumst": 1, "should": [1, 2, 8, 9, 10], "mid_init_match": 1, "either_1": 1, "nativ": 1, "either_0": 1, "gen": 1, "imm": [1, 2, 12], "immigr": 1, "look": [1, 10, 11], "foreign": 1, "born": 1, "sgen": [1, 2, 12], "rel": [1, 2, 12], "scala": 1, "determin": [1, 7], "greater": [1, 5], "jw_threshold": 1, "less": [1, 2], "age_threshold": 1, "sex": [1, 2, 11], "sampl": 1, "related_individual_row": 1, "unrel": 1, "depend": [1, 2, 5, 12], "name_col": 1, "birthyr_col": 1, "namefrst_related_row": 1, "replaced_birthyr": [1, 2, 3], "extra": 1, "children": 1, "who": 1, "base": [1, 2, 7], "expect": 1, "count": [1, 10, 12], "suspect": [1, 6], "relate_col": 1, "histid_col": 1, "id": [1, 2], "birth": 1, "year_b": 1, "wa": [1, 12], "minimum": [1, 8], "accept": [1, 2, 12], "consid": [1, 8], "histid": [1, 2, 12], "1910": [1, 2, 12], "8": [1, 2, 5, 10], "rate": 1, "calcul": [1, 12], "percentag": 1, "seen": 1, "neighbor": 1, "meet": 1, "95": 1, "nbor": [1, 2, 12], "namelast_neighbor": 1, "05": [1, 2], "namelast_popularity_sum": 1, "namelast_popular": 1, "length": [1, 2, 9], "size": 1, "ab": 1, "diff": 1, "absolut": 1, "invalid": [1, 8], "instead": [1, 2, 5, 7], "marriag": 1, "durat": 1, "99": [1, 2], "placehold": 1, "unknown": 1, "exclud": 1, "those": [1, 2], "consider": 1, "byrdiff": [1, 2, 12], "mardurmatch": [1, 2], "14": 1, "minu": [1, 2], "subtract": 1, "geo": 1, "distanc": [1, 8], "lookup": 1, "tabl": [1, 2, 4, 7, 10, 12], "core": [1, 7, 10, 12], "dist_tabl": 1, "py": [1, 2], "There": [1, 2, 3, 7], "sever": [1, 6], "wai": [1, 5, 10], "file": [1, 4, 6, 7, 10, 11, 12], "kei": [1, 7, 10], "key_count": 1, "secondari": 1, "serv": 1, "back": 1, "primari": [1, 6], "doe": [1, 7, 12], "particularli": 1, "state": [1, 6], "much": [1, 7], "fewer": [1, 8], "combin": [1, 2, 3, 7], "thu": 1, "risk": 1, "fill": 1, "aren": 1, "ex": 1, "just": [1, 2, 10, 12], "even": 1, "though": 1, "distances_fil": 1, "path": [1, 2, 10, 11, 12], "table_nam": 1, "what": [1, 2, 10, 12], "onc": [1, 10], "loc_a": 1, "where": [1, 7, 10, 12], "come": 1, "loc_b": 1, "distance_col": 1, "source_column_a": 1, "sourc": [1, 4, 7, 10, 12], "source_column_b": 1, "loc_a_0": 1, "loc_a_1": 1, "loc_b_0": 1, "loc_b_1": 1, "secondary_key_count": 1, "backup": 1, "secondary_table_nam": 1, "secondary_distances_fil": 1, "secondary_source_column": 1, "secondary_loc_a": 1, "secondary_loc_b": 1, "secondary_distance_col": 1, "state_dist": 1, "state_distance_lookup": 1, "county_state_dist": 1, "csv": [1, 2, 7, 10, 11, 12], "statecode1": 1, "statecode2": 1, "dist": 1, "county_dist": [1, 2, 12], "county_distance_lookup": 1, "county_1900_1910_distances_km": 1, "from_icpsrctyi": 1, "to_icpsrctyi": 1, "from_statefip": 1, "to_statefip": 1, "distance_km": 1, "state_1900_1910_distances_km": 1, "fetch": 1, "neither": 1, "nor": 1, "mpre": 1, "m_namefrst": 1, "accord": 1, "niu": 1, "other": [1, 2, 12], "mfbplmatch": 1, "multi": 1, "search": 1, "special": 1, "simplifi": 1, "particular": [1, 2], "constraint": 1, "num_col": 1, "whose": 1, "templat": 1, "n": [1, 8, 9], "per": [1, 2, 8, 9, 10], "current": [1, 2, 10], "respect": [1, 7], "jw_col_templ": 1, "jw": 1, "pair": [1, 12], "equal_and_not_null_templ": 1, "final": [1, 2, 12], "comput": [1, 3, 7], "_namefrst": 1, "_bpl": 1, "_sex": 1, "25": 1, "nvl": 1, "sm_namefrst": 1, "sn_namefrst": 1, "sm_bpl": 1, "sn_bpl": 1, "sm_sex": 1, "sn_sex": 1, "pass": [1, 2, 7, 8], "flexibl": 1, "user": [1, 10], "write": [1, 10, 12], "own": [1, 2], "favor": 1, "reason": 1, "good": 1, "fallback": 1, "defin": [1, 7, 8, 9, 10], "spark": [1, 2, 5, 8, 9, 10, 12], "builtin": 1, "argument": [1, 10, 12], "namelast_jw_max": 1, "namelast1": 1, "namelast2": 1, "namelast3": 1, "abov": [1, 5], "extend": 1, "beyond": 1, "top": [1, 4], "level": [1, 4, 10], "everi": 1, "jw_f": [1, 2, 12], "father_namefrst": 1, "rais": [1, 3], "exponenti": 1, "squar": 1, "county_distance_squar": [1, 2, 12], "county_a": 1, "county_b": 1, "upper": 1, "gt": 1, "btwn": 1, "addl": 1, "var": [1, 2], "program": [1, 2, 7, 12], "report": [1, 4, 6, 10], "addl_var": 1, "check_val_expr": 1, "else_v": 1, "volumn": 1, "datasourc": [1, 2, 10], "yrimmig": 1, "immyear_diff": [1, 2, 9, 12], "includ": [1, 2, 7, 9, 10], "train": [1, 4, 6, 8, 10], "independent_var": [1, 2, 12], "config": [1, 4, 7, 10, 12], "id_column": [1, 2], "_a": 1, "mult": 1, "exist": [1, 2, 10], "within": [1, 2, 6, 10, 11], "hh_train": [1, 2, 7, 10, 12], "hh": 1, "highest": [1, 2], "against": [1, 11], "ten": [1, 2], "tell": 2, "how": [2, 7], "descript": [2, 8, 10], "refer": 2, "here": [2, 7, 10, 12], "tutori": [2, 10], "script": [2, 6, 10], "discuss": 2, "readm": 2, "note": 2, "written": [2, 6], "toml": [2, 6, 10], "abl": 2, "json": [2, 10], "datasource_a": [2, 7], "datasource_b": [2, 7], "transform": [2, 4, 6, 7], "lowercase_strip": 2, "add_to_a": 2, "age_2": 2, "derived_from": 2, "expand_length": 2, "explod": [2, 7], "jaro_winkl": 2, "namelast_jw": [2, 12], "threshold": [2, 8, 12], "feature_nam": 2, "79": 2, "84": 2, "complex": 2, "machin": [2, 6, 7, 10, 12], "learn": [2, 6, 7, 10, 12], "probabilist": [2, 6], "drop_data_from_scored_match": 2, "us1900": 2, "us1900m_usa": 2, "p": 2, "parquet": [2, 7], "us1910": 2, "us1910m_usa": 2, "training_data_subset": 2, "serialp": 2, "rationalize_name_word": 2, "remove_qmark_hyphen": 2, "replace_apostroph": 2, "remove_suffix": 2, "remove_alternate_nam": 2, "condense_strip_whitespac": 2, "split": [2, 3, 7, 8, 9, 12], "namefrst_std": [2, 11], "array_index": 2, "bpl_orig": 2, "divide_by_int": 2, "get_floor": 2, "statefip_h": 2, "output_typ": 2, "substitution_column": [2, 7, 11], "join_column": [2, 11], "join_valu": [2, 11], "substitution_fil": [2, 11], "name_std": [2, 11], "male": [2, 11], "femal": [2, 11], "feature_select": [2, 3, 7], "input_column": [2, 3, 9], "output_column": [2, 3, 9], "sql_condit": 2, "namelast_bigram": 2, "bigram": [2, 4], "bpl_clean": 2, "bpl_str": 2, "washington": 2, "bpl2_str": 2, "53": 2, "region": [2, 12], "attach_vari": 2, "region_dict": 2, "col_to_join_on": 2, "col_to_add": 2, "null_fil": 2, "col_typ": 2, "potential_matches_univers": [2, 7], "birthyr_3": 2, "namefrst_std_jw": [2, 12], "75": [2, 8, 12], "comparis": 2, "post": [2, 7], "hh_comparison": [2, 7], "threshold_expr": 2, "fetch_a": 2, "sex_equ": 2, "equal": [2, 11], "relate_a": [2, 9], "pipeline_featur": [2, 7, 9], "sex_region_interact": 2, "transformer_typ": [2, 9], "interact": [2, 4, 7, 12], "relatetyp": [2, 9], "bucket": [2, 7], "hit": [2, 10, 12], "scale_data": [2, 12], "training_data": [2, 10], "dependent_var": [2, 12], "score_with_model": [2, 12], "use_training_data_featur": [2, 7, 12], "split_by_id_a": [2, 12], "decis": [2, 4, 8, 12], "drop_duplicate_with_threshold_ratio": [2, 12], "n_training_iter": [2, 7, 12], "output_suspicious_td": [2, 12], "param_grid": [2, 12], "model_paramet": [2, 7, 8, 12], "random_forest": [2, 12], "maxdepth": [2, 8, 12], "numtre": [2, 8, 12], "005": 2, "threshold_ratio": [2, 8, 12], "logistic_regress": [2, 12], "50": [2, 12], "65": 2, "80": 2, "chosen_model": [2, 8, 12], "prediction_col": 2, "predict": [2, 12], "hh_col": 2, "hh_training_data_1900_1910": 2, "probit": [2, 4], "go": [2, 10], "your": [2, 5, 7, 10, 12], "uniqu": 2, "identifi": [2, 6, 12], "full": [2, 7, 12], "short": 2, "alphanumer": 2, "convert_ints_to_long": 2, "long": [2, 11], "especi": 2, "assum": 2, "schema": 2, "sometim": 2, "term": 2, "bigint": 2, "thing": 2, "my_fil": 2, "subset": [2, 11], "limit": 2, "extract": 2, "modifi": 2, "meant": 2, "usag": [2, 4, 10], "set_value_column_a": [2, 3], "liter": 2, "set_value_column_b": [2, 3], "iv": 2, "v": 2, "vi": 2, "vii": 2, "viii": 2, "namelast_clean_bigram": [2, 3], "fed": [2, 7], "prep": 2, "df": [2, 10], "men": 2, "newli": 2, "attempt": 2, "duplic": [2, 8], "conjuct": 2, "Will": 2, "conjunct": 2, "rang": [2, 9], "original_valu": 2, "plu": 2, "1870": 2, "expand": 2, "1867": 2, "1868": 2, "1869": 2, "1871": 2, "1872": 2, "1873": 2, "kept": 2, "keep": 2, "appropri": 2, "treat": [2, 9], "import": [2, 7, 10, 12], "dure": [2, 7], "hot": 2, "encod": [2, 3], "vector": [2, 9], "stage": 2, "well": 2, "upper_threshold": 2, "cannot": 2, "robust": 2, "ml": [2, 4, 8, 9], "typic": [2, 7], "leverag": 2, "api": [2, 6, 9], "piplin": 2, "regionf": 2, "sex_regionf_interact": 2, "immyear_caut": [2, 9], "myriad": 2, "explor": [2, 4, 6, 10], "part": [2, 7], "task": [2, 4, 6, 8, 12], "drop_duplicate_a": 2, "out": [2, 7, 12], "best": [2, 7], "smallest": 2, "possibl": 2, "ratio": [2, 8], "beta": [2, 8], "test": [2, 7, 12], "model_explor": [2, 10, 12], "hyper": [2, 6, 12], "paramet": [2, 6, 7, 8, 10, 12], "eval": 2, "skip": [2, 7], "apply_model": 2, "run_all_step": [2, 10, 12], "command": [2, 6, 10, 12], "try": 2, "creation": 2, "iter": 2, "scale": 2, "error": [2, 9], "1900": [2, 12], "about": [2, 10, 12], "1930": [2, 12], "1940": [2, 12], "fail": 2, "were": 2, "sure": [2, 5, 10], "scratch": 2, "although": 2, "know": 2, "haven": 2, "save": [2, 7, 12], "small": 2, "amount": 2, "process": [2, 6, 10], "repeatedli": 2, "help": [2, 7, 10], "neg": [2, 4, 6], "area": 2, "coverag": 2, "increas": [2, 9], "represent": [2, 7], "ensur": 2, "group": [2, 7], "a304bt": 2, "three": [2, 7], "b200": 2, "c201": 2, "d425": 2, "perform": [2, 6, 7, 11], "feature_import": [2, 7, 12], "coeffici": [2, 7], "enabl": [2, 7, 10], "srace": [2, 9, 12], "race_interacted_srac": [2, 9, 12], "hits2": [2, 12], "exact_mult": [2, 12], "ncount": [2, 3, 12], "ncount2": [2, 3, 12], "f_interacted_jw_f": [2, 12], "f_caution": [2, 12], "f_pre": [2, 12], "fbplmatch": [2, 12], "m_interacted_jw_m": [2, 9, 12], "jw_m": [2, 9, 12], "m_pre": [2, 9, 12], "mbplmatch": [2, 12], "sp_interacted_jw_sp": [2, 12], "jw_sp": [2, 12], "sp_pre": [2, 12], "mi": [2, 12], "fsoundex": [2, 12], "lsoundex": [2, 12], "oth": [2, 12], "imm_interacted_immyear_caut": [2, 12], "1900_1910_training_data_20191023": 2, "jw_max_a": 2, "jw_max_b": 2, "f1_match": 2, "f2_match": 2, "byrdifcat": 2, "racematch": 2, "bplmatch": 2, "imm_interacted_bplmatch": 2, "sexmatch": 2, "relatetype_interacted_relatematch": 2, "checkpoint": 3, "no_first_pad": 3, "don": 3, "prepend": 3, "namefrst_unstd_bigram": 3, "namelast_frst_bigram": 3, "namelast_clean_soundex": 3, "input_col": 3, "output_col": 3, "expon": 3, "introduct": 4, "overview": 4, "instal": 4, "pypi": 4, "preprocess": [4, 6, 10, 12], "model": [4, 6, 10], "run": [4, 5, 6, 7, 12], "librari": [4, 6], "mode": [4, 5, 12], "advanc": 4, "workflow": 4, "export": [4, 7, 10], "featur": [4, 6, 7, 8, 10], "reus": 4, "basic": 4, "map": [4, 7, 9], "substitut": [4, 7], "block": [4, 7], "comparison": [4, 7], "pipelin": 4, "ons": 4, "aggreg": 4, "union": 4, "soundex": 4, "power": 4, "regex": 4, "random": [4, 8], "forest": [4, 8], "logist": [4, 8], "regress": [4, 8], "tree": [4, 8], "gradient": [4, 8], "boost": [4, 8], "system": 5, "python": [5, 6, 10], "java": 5, "integr": 5, "apach": 5, "via": [5, 6], "pyspark": [5, 8, 9, 10], "packag": 5, "org": 5, "latest": 5, "pip": 5, "easiest": [5, 10], "through": [5, 7, 9, 10], "instruct": [5, 10], "But": 5, "clone": 5, "github": 5, "repositori": 5, "root": 5, "project": 5, "directori": [5, 10, 12], "develop": [5, 6], "e": 5, "dev": 5, "edit": 5, "made": 5, "built": 5, "tool": [5, 6], "line": [6, 10], "share": 6, "characterist": [6, 7], "correspond": [6, 7], "real": 6, "world": 6, "determinist": [6, 7], "rule": [6, 7], "algorithm": [6, 7], "At": [6, 7], "been": 6, "unit": 6, "census": 6, "hierarch": [6, 10], "structur": 6, "nest": 6, "howev": [6, 12], "tailor": 6, "ignor": 6, "common": [6, 7, 12], "highli": [6, 7], "languag": 6, "further": [6, 12], "broken": 6, "smaller": 6, "sequenc": 6, "linkrun": [6, 10], "prepar": [6, 7, 10], "research": 6, "experi": 6, "understand": 6, "tune": [6, 12], "relationship": 6, "varieti": 7, "normal": 7, "abbrevi": [7, 11], "regist": [7, 10], "datafram": [7, 10, 12], "request": 7, "classif": [7, 8], "metadata": 7, "introspect": 7, "ingest": 7, "inspect": 7, "mani": [7, 10], "aspect": [7, 10], "extens": 7, "longest": 7, "definit": 7, "reduc": 7, "drastic": 7, "improv": 7, "runtim": 7, "separ": 7, "total": 7, "potential_match": [7, 10], "satisfi": 7, "elig": 7, "reshap": 7, "thought": 7, "ahead": 7, "chosen": 7, "experiment": [7, 10], "focus": 7, "demograph": 7, "moment": 7, "veri": [7, 12], "anyon": 7, "percent": 7, "remain": 7, "popul": 7, "pull": 7, "fix": 7, "width": 7, "crosswalk": 7, "construct": 7, "alpha": 8, "hyperparamet": [8, 12], "de": 8, "param": [8, 12], "label": 8, "doc": [8, 9], "commonli": 8, "explan": 8, "randomforestclassifi": 8, "depth": 8, "20": 8, "featuresubsetstrategi": 8, "node": 8, "auto": 8, "onethird": 8, "sqrt": 8, "log2": 8, "15": 8, "generalizedlinearregress": 8, "famili": 8, "binomi": 8, "85": [8, 10], "logisticregress": 8, "decisiontreeclassifi": 8, "mininstancespernod": 8, "caus": 8, "left": 8, "right": [8, 10], "discard": 8, "maxbin": 8, "bin": 8, "discret": 8, "continu": [8, 9, 12], "gbtclassifi": 8, "mother": 9, "point": [9, 12], "x": [9, 10], "y": 9, "hold": 9, "except": 9, "strictli": 9, "inf": 9, "explicitli": 9, "cover": 9, "doubl": 9, "outsid": 9, "job": 10, "high": 10, "class": 10, "handl": 10, "main": 10, "complet": 10, "access": [10, 12], "link_run": 10, "factori": 10, "sparkfactori": 10, "load_config": 10, "load_conf_fil": 10, "sparksess": 10, "now": 10, "let": 10, "load": 10, "our": 10, "my_conf": 10, "lr": 10, "prep_step": 10, "get_step": 10, "enumer": 10, "print": 10, "input_table_nam": 10, "output_table_nam": 10, "run_step": 10, "get_tabl": 10, "matches_df": 10, "hh_model_explor": 10, "method": [10, 12], "interfac": 10, "easili": 10, "conveni": 10, "adjust": 10, "set_loc": 10, "set_num_cor": 10, "set_executor_memori": 10, "5g": 10, "ll": 10, "dictionari": 10, "often": 10, "modul": 10, "pleas": 10, "reproduc": 10, "consol": 10, "cpu": 10, "h": 10, "executor_memori": [10, 12], "execute_task": 10, "execute_command": 10, "conf": [10, 12], "show": 10, "messag": 10, "exit": 10, "memori": 10, "executor": 10, "begin": 10, "execut": 10, "seri": 10, "excute_command": 10, "filepath": 10, "sai": 10, "fullcount_1870_1880": 10, "pattern": 10, "full_count_1870_1880": 10, "prompt": 10, "enter": 10, "text": 10, "unstabl": 10, "topic": 10, "analyz": [10, 12], "set_preexisting_t": 10, "x_persist": 10, "borrow_t": 10, "get_task": 10, "set_print_sql": 10, "x_sql": 10, "x_sqlf": 10, "ipython": 10, "showf": 10, "x_summari": 10, "desc": 10, "x_crosswalk": 10, "x_tab": 10, "q": [10, 12], "x_hh_tfam": 10, "x_tfam": 10, "drop_al": 10, "reload": 10, "x_hh_tfam_2a": 10, "x_tfam_raw": 10, "drop_all_prc": 10, "x_hh_tfam_2b": 10, "x_union": 10, "drop_all_temp": 10, "x_load": 10, "get_set": 10, "set_link_task": 10, "x_parquet_from_csv": 10, "organ": 10, "hierarchi": 10, "five": 10, "hh_match": 10, "someth": 10, "choic": 10, "preexist": 10, "prepped_df_a": 10, "prepped_df_b": 10, "raw_df_b": 10, "raw_df_a": 10, "training_featur": [10, 12], "scored_potential_match": 10, "potential_matches_prep": 10, "exploded_df_b": 10, "exploded_df_a": 10, "predicted_match": 10, "hh_training_featur": [10, 12], "hh_training_data": 10, "hh_predicted_match": 10, "hh_scored_potential_match": 10, "hh_potential_match": 10, "hh_blocked_match": 10, "hh_potential_matchs_prep": 10, "model_eval_training_vector": 10, "model_eval_training_data": 10, "model_eval_repeat_fp": 10, "model_eval_training_featur": 10, "model_eval_training_result": 10, "model_eval_repeat_fn": 10, "hh_model_eval_training_vector": 10, "hh_model_eval_repeat_fp": 10, "hh_model_eval_repeat_fn": 10, "hh_model_eval_training_result": 10, "hh_model_eval_training_featur": 10, "hh_model_eval_training_data": 10, "persist": 10, "hidden": 10, "intermedi": 10, "yet": 10, "databas": 10, "tablenam": 10, "istemporari": 10, "task_nam": 10, "num": 10, "finish": 10, "put": [10, 12], "launch": [10, 12], "my": [10, 12], "subhead": 11, "suppli": 11, "regex_word_replac": 11, "variant": 11, "av": 11, "7th": 11, "swap": 11, "still": 11, "anywher": 11, "proceed": 11, "street_unstd": 11, "dir": 11, "substitutions_street_abbrev": 11, "span": 12, "1920": 12, "deriv": 12, "necessari": 12, "scenario": 12, "copi": 12, "use_potential_matches_featur": 12, "full_count_1900_1910": 12, "50g": 12, "ask": 12, "arg": 12, "partit": 12, "training_data_1900_1910_hlink_featur": 12, "might": 12, "shut": 12, "framework": 12, "etc": 12, "relev": 12, "matrix": 12, "implement": 12, "regular": 12, "training_data_1900_1910": 12, "weren": 12, "ident": 12, "manual": 12, "updat": 12, "isn": 12, "analysi": 12, "training_result": 12, "hh_training_result": 12, "1900_1910_training_result": 12, "repeat_fp": 12, "repeat_fn": 12, "hh_repeat_fp": 12, "hh_repeat_fn": 12, "1900_1910_potential_fp": 12, "1900_1910_potential_fn": 12, "prefer": 12, "ve": 12}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"column": [0, 2], "map": [0, 2], "basic": [0, 2], "usag": 0, "advanc": [0, 2, 12], "transform": [0, 1, 3, 9], "add_to_a": 0, "concat_to_a": 0, "concat_to_b": 0, "concat_two_col": 0, "lowercase_strip": 0, "rationalize_name_word": 0, "remove_qmark_hyphen": 0, "remove_punctu": 0, "replace_apostroph": 0, "remove_alternate_nam": 0, "remove_suffix": 0, "remove_stop_word": 0, "remove_prefix": 0, "condense_strip_whitespac": 0, "remove_one_letter_nam": 0, "split": 0, "array_index": 0, "substr": 0, "divide_by_int": 0, "when_valu": 0, "get_floor": 0, "comparison": [1, 2], "type": [1, 9], "add": 1, "ons": 1, "aggreg": 1, "featur": [1, 2, 3, 9, 12], "household": [1, 2, 7], "maximum_jaro_winkl": 1, "jaro_winkl": 1, "jaro_winkler_street": 1, "max_jaro_winkl": 1, "equal": 1, "f1_match": 1, "f2_match": 1, "not_equ": 1, "equals_as_int": 1, "all_equ": 1, "not_zero_and_not_equ": 1, "time": 1, "caution_comp_3": 1, "caution_comp_3_012": 1, "caution_comp_4": 1, "caution_comp_4_012": 1, "any_equ": 1, "either_are_1": 1, "either_are_0": 1, "second_gen_imm": 1, "rel_jaro_winkl": 1, "extra_children": 1, "jaro_winkler_r": 1, "sum": 1, "length_b": 1, "abs_diff": 1, "b_minus_a": 1, "geo_dist": 1, "fetch_a": 1, "fetch_b": 1, "present_both_year": 1, "neither_are_nul": 1, "present_and_matching_categor": 1, "present_and_not_equ": 1, "present_and_equal_categorical_in_univers": 1, "multi_jaro_winkler_search": 1, "sql_condit": [1, 3], "alia": 1, "power": [1, 3], "threshold": 1, "lower_threshold": 1, "upper_threshold": 1, "gt_threshold": 1, "btwn_threshold": 1, "look_at_addl_var": 1, "hit": 1, "hits2": 1, "exact_mult": 1, "jw_max_a": 1, "jw_max_b": 1, "configur": [2, 4, 7], "config": 2, "file": 2, "top": 2, "level": 2, "data": [2, 11, 12], "sourc": [2, 5], "filter": 2, "substitut": [2, 11], "select": [2, 3], "potenti": [2, 12], "match": [2, 7], "univers": 2, "block": 2, "pipelin": [2, 9], "gener": [2, 9, 12], "train": [2, 7, 12], "model": [2, 7, 8, 12], "bigram": 3, "arrai": 3, "union": 3, "soundex": 3, "welcom": 4, "hlink": [4, 10], "": 4, "document": 4, "api": 4, "instal": 5, "requir": 5, "from": 5, "pypi": 5, "introduct": 6, "overview": [6, 7], "link": [7, 10, 12], "task": [7, 10], "preprocess": 7, "step": [7, 10], "relat": 7, "section": 7, "explor": [7, 12], "report": 7, "random_forest": 8, "probit": 8, "logistic_regress": 8, "decision_tre": 8, "gradient_boosted_tre": 8, "interact": [9, 10], "bucket": 9, "run": 10, "us": 10, "librari": 10, "mode": 10, "start": 10, "program": 10, "exampl": [10, 12], "workflow": [10, 12], "1": 11, "tabl": 11, "regex": 11, "word": 11, "replac": 11, "export": 12, "after": 12, "reus": 12, "differ": 12, "year": 12, "ml": 12, "list": 12, "fals": 12, "posit": 12, "neg": 12, "fp": 12, "fn": 12}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 60}, "alltitles": {"Column Mappings": [[0, "column-mappings"], [2, "column-mappings"]], "Basic Usage": [[0, "basic-usage"]], "Advanced Usage": [[0, "advanced-usage"]], "Transforms": [[0, "transforms"]], "add_to_a": [[0, "add-to-a"]], "concat_to_a": [[0, "concat-to-a"]], "concat_to_b": [[0, "concat-to-b"]], "concat_two_cols": [[0, "concat-two-cols"]], "lowercase_strip": [[0, "lowercase-strip"]], "rationalize_name_words": [[0, "rationalize-name-words"]], "remove_qmark_hyphen": [[0, "remove-qmark-hyphen"]], "remove_punctuation": [[0, "remove-punctuation"]], "replace_apostrophe": [[0, "replace-apostrophe"]], "remove_alternate_names": [[0, "remove-alternate-names"]], "remove_suffixes": [[0, "remove-suffixes"]], "remove_stop_words": [[0, "remove-stop-words"]], "remove_prefixes": [[0, "remove-prefixes"]], "condense_strip_whitespace": [[0, "condense-strip-whitespace"]], "remove_one_letter_names": [[0, "remove-one-letter-names"]], "split": [[0, "split"]], "array_index": [[0, "array-index"]], "mapping": [[0, "mapping"]], "substring": [[0, "substring"]], "divide_by_int": [[0, "divide-by-int"]], "when_value": [[0, "when-value"]], "get_floor": [[0, "get-floor"]], "Comparison types, transform add-ons, aggregate features, and household aggregate features": [[1, "comparison-types-transform-add-ons-aggregate-features-and-household-aggregate-features"]], "Comparison types": [[1, "comparison-types"]], "maximum_jaro_winkler": [[1, "maximum-jaro-winkler"]], "jaro_winkler": [[1, "jaro-winkler"]], "jaro_winkler_street": [[1, "jaro-winkler-street"]], "max_jaro_winkler": [[1, "max-jaro-winkler"]], "equals": [[1, "equals"]], "f1_match": [[1, "f1-match"]], "f2_match": [[1, "f2-match"]], "not_equals": [[1, "not-equals"]], "equals_as_int": [[1, "equals-as-int"]], "all_equals": [[1, "all-equals"]], "not_zero_and_not_equals": [[1, "not-zero-and-not-equals"]], "or": [[1, "or"]], "and": [[1, "and"]], "times": [[1, "times"]], "caution_comp_3": [[1, "caution-comp-3"]], "caution_comp_3_012": [[1, "caution-comp-3-012"]], "caution_comp_4": [[1, "caution-comp-4"]], "caution_comp_4_012": [[1, "caution-comp-4-012"]], "any_equals": [[1, "any-equals"]], "either_are_1": [[1, "either-are-1"]], "either_are_0": [[1, "either-are-0"]], "second_gen_imm": [[1, "second-gen-imm"]], "rel_jaro_winkler": [[1, "rel-jaro-winkler"]], "extra_children": [[1, "extra-children"]], "jaro_winkler_rate": [[1, "jaro-winkler-rate"]], "sum": [[1, "sum"]], "length_b": [[1, "length-b"]], "abs_diff": [[1, "abs-diff"]], "b_minus_a": [[1, "b-minus-a"]], "geo_distance": [[1, "geo-distance"]], "fetch_a": [[1, "fetch-a"]], "fetch_b": [[1, "fetch-b"]], "present_both_years": [[1, "present-both-years"]], "neither_are_null": [[1, "neither-are-null"]], "present_and_matching_categorical": [[1, "present-and-matching-categorical"]], "present_and_not_equal": [[1, "present-and-not-equal"]], "present_and_equal_categorical_in_universe": [[1, "present-and-equal-categorical-in-universe"]], "multi_jaro_winkler_search": [[1, "multi-jaro-winkler-search"]], "sql_condition": [[1, "sql-condition"], [3, "sql-condition"]], "Feature add-ons": [[1, "feature-add-ons"]], "alias": [[1, "alias"]], "power": [[1, "power"], [3, "power"]], "threshold": [[1, "threshold"]], "lower_threshold": [[1, "lower-threshold"]], "upper_threshold": [[1, "upper-threshold"]], "gt_threshold": [[1, "gt-threshold"]], "btwn_threshold": [[1, "btwn-threshold"]], "look_at_addl_var": [[1, "look-at-addl-var"]], "Aggregate Features": [[1, "aggregate-features"]], "hits": [[1, "hits"]], "hits2": [[1, "hits2"]], "exact_mult": [[1, "exact-mult"]], "Household Aggregate Features": [[1, "household-aggregate-features"]], "jw_max_a": [[1, "jw-max-a"]], "jw_max_b": [[1, "jw-max-b"]], "Configuration": [[2, "configuration"]], "Basic Config File": [[2, "basic-config-file"]], "Advanced Config File": [[2, "advanced-config-file"]], "Top level configs": [[2, "top-level-configs"]], "Data sources": [[2, "data-sources"]], "Filter": [[2, "filter"]], "Substitution Columns": [[2, "substitution-columns"]], "Feature Selections": [[2, "feature-selections"]], "Potential Matches Universe": [[2, "potential-matches-universe"]], "Blocking": [[2, "blocking"]], "Comparisons": [[2, "comparisons"]], "Household Comparisons": [[2, "household-comparisons"]], "Comparison Features": [[2, "comparison-features"]], "Pipeline-generated Features": [[2, "pipeline-generated-features"]], "Training and models": [[2, "training-and-models"]], "Household training and models": [[2, "household-training-and-models"]], "Feature Selection transforms": [[3, "feature-selection-transforms"]], "bigrams": [[3, "bigrams"]], "array": [[3, "array"]], "union": [[3, "union"]], "soundex": [[3, "soundex"]], "Welcome to hlink\u2019s documentation!": [[4, "welcome-to-hlink-s-documentation"]], "Configuration API": [[4, "configuration-api"], [4, null]], "Installation": [[5, "installation"]], "Requirements": [[5, "requirements"]], "Installing from PyPI": [[5, "installing-from-pypi"]], "Installing from source": [[5, "installing-from-source"]], "Introduction": [[6, "introduction"]], "Overview": [[6, "overview"], [7, "overview"], [7, "id1"], [7, "id4"], [7, "id7"], [7, "id10"], [7, "id13"]], "Link Tasks": [[7, "link-tasks"]], "Preprocessing": [[7, "preprocessing"]], "Task steps": [[7, "task-steps"], [7, "id2"], [7, "id5"], [7, "id8"], [7, "id11"], [7, "id14"]], "Related Configuration Sections": [[7, "related-configuration-sections"], [7, "id3"], [7, "id6"], [7, "id9"], [7, "id12"], [7, "id15"]], "Training and Household Training": [[7, "training-and-household-training"]], "Matching": [[7, "matching"]], "Household Matching": [[7, "household-matching"]], "Model Exploration and Household Model Exploration": [[7, "model-exploration-and-household-model-exploration"]], "Reporting": [[7, "reporting"]], "Models": [[8, "models"]], "random_forest": [[8, "random-forest"]], "probit": [[8, "probit"]], "logistic_regression": [[8, "logistic-regression"]], "decision_tree": [[8, "decision-tree"]], "gradient_boosted_trees": [[8, "gradient-boosted-trees"]], "Pipeline generated features": [[9, "pipeline-generated-features"]], "Transformer types": [[9, "transformer-types"]], "interaction": [[9, "interaction"]], "bucketizer": [[9, "bucketizer"]], "Running hlink": [[10, "running-hlink"]], "Using hlink as a Library": [[10, "using-hlink-as-a-library"]], "Interactive Mode": [[10, "interactive-mode"]], "Starting the program": [[10, "starting-the-program"]], "Running Linking Tasks and Steps": [[10, "running-linking-tasks-and-steps"]], "Example interactive mode workflow": [[10, "example-interactive-mode-workflow"]], "Substitutions": [[11, "substitutions"]], "1:1 substitution by data table": [[11, "substitution-by-data-table"]], "Substitution by regex word replace": [[11, "substitution-by-regex-word-replace"]], "Advanced Workflow Examples": [[12, "advanced-workflow-examples"]], "Export training data after generating features to reuse in different linking years": [[12, "export-training-data-after-generating-features-to-reuse-in-different-linking-years"]], "Example training data export with generated ML features": [[12, "example-training-data-export-with-generated-ml-features"]], "ML model exploration and export of lists of potential false positives/negatives in training data": [[12, "ml-model-exploration-and-export-of-lists-of-potential-false-positives-negatives-in-training-data"]], "Example model exploration and FP/FN export workflow": [[12, "example-model-exploration-and-fp-fn-export-workflow"]]}, "indexentries": {}})
\ No newline at end of file
+Search.setIndex({"docnames": ["column_mappings", "comparison_types", "config", "feature_selection_transforms", "index", "installation", "introduction", "link_tasks", "models", "pipeline_features", "running_the_program", "substitutions", "use_examples"], "filenames": ["column_mappings.md", "comparison_types.md", "config.md", "feature_selection_transforms.md", "index.rst", "installation.md", "introduction.md", "link_tasks.md", "models.md", "pipeline_features.md", "running_the_program.md", "substitutions.md", "use_examples.md"], "titles": ["Column Mappings", "Comparison types, transform add-ons, aggregate features, and household aggregate features", "Configuration", "Feature Selection transforms", "Welcome to hlink\u2019s documentation!", "Installation", "Introduction", "Link Tasks", "Models", "Pipeline generated features", "Running hlink", "Substitutions", "Advanced Workflow Examples"], "terms": {"each": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10], "read": [0, 1, 2, 7, 10], "from": [0, 1, 2, 4, 6, 7, 8, 9, 10, 12], "input": [0, 1, 2, 3, 6, 7, 10, 11], "dataset": [0, 1, 2, 6, 7, 10, 12], "hlink": [0, 1, 2, 5, 6, 7, 12], "It": [0, 1, 2, 6, 10, 12], "ha": [0, 1, 2, 6, 10, 12], "column_nam": [0, 1, 2, 11], "attribut": [0, 1, 2, 3, 7, 8, 9, 10, 11], "which": [0, 1, 2, 3, 6, 7, 9, 10, 12], "specifi": [0, 1, 2, 6, 7, 9, 10, 11], "name": [0, 1, 2, 10, 11], "both": [0, 1, 2, 7, 12], "option": [0, 1, 2, 3, 6, 7, 8, 10, 12], "mai": [0, 2, 6, 7, 10], "have": [0, 1, 2, 5, 6, 7, 8, 10, 12], "an": [0, 1, 2, 6, 8, 10], "alia": [0, 2, 7], "give": [0, 2], "new": [0, 2, 12], "us": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12], "support": [0, 7, 8], "some": [0, 1, 2, 3, 6, 7, 10], "make": [0, 1, 2, 5, 12], "chang": [0, 1, 2, 5, 10, 12], "data": [0, 1, 4, 6, 7, 10], "thei": [0, 1, 2, 7, 10], "ar": [0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 12], "These": [0, 1, 2, 3, 6, 7, 8, 9], "clean": [0, 6], "harmon": 0, "The": [0, 1, 2, 3, 5, 7, 8, 9, 10, 12], "avail": [0, 1, 2, 3, 5, 7, 8, 9, 12], "list": [0, 1, 2, 3, 4, 8, 10, 11], "below": [0, 1, 2, 3, 8, 9, 10], "section": [0, 1, 2, 12], "By": [0, 2, 10], "default": [0, 1, 2, 7, 8, 10], "must": [0, 1, 2, 8, 9, 11], "same": [0, 1, 2, 6, 7, 10], "With": [0, 9], "override_column_a": [0, 2, 3], "override_column_b": [0, 2, 3], "you": [0, 1, 2, 5, 10, 11, 12], "can": [0, 1, 2, 5, 6, 7, 8, 10, 12], "differ": [0, 1, 2, 4, 6, 7], "either": [0, 1, 2, 6, 11], "A": [0, 1, 2, 9, 10], "b": [0, 1, 2, 10], "when": [0, 1, 2, 3, 7, 12], "do": [0, 1, 3, 10, 12], "thi": [0, 1, 2, 5, 6, 7, 9, 10, 12], "appli": [0, 2, 3, 7, 12], "onli": [0, 1, 2, 7, 12], "non": 0, "overrid": [0, 2], "also": [0, 1, 2, 5, 6, 7, 9, 10, 12], "provid": [0, 2, 3, 6, 7, 9, 10], "override_transform": [0, 2], "describ": [0, 2, 10], "type": [0, 2, 3, 4, 7, 8, 10, 11, 12], "oper": [0, 2], "singl": [0, 2, 3, 10, 12], "output": [0, 1, 2, 3, 6, 7, 10, 12], "more": [0, 1, 2, 9, 10, 12], "than": [0, 1, 2, 8], "one": [0, 1, 2, 7], "order": [0, 2, 7], "so": [0, 1, 2, 5, 12], "anoth": [0, 1, 3, 7], "format": 0, "letter": 0, "t": [0, 1, 2, 3, 12], "u": 0, "repres": [0, 1, 2, 3, 9, 10], "arbitrari": 0, "requir": [0, 1, 2, 3, 4, 7, 9, 10, 11], "addit": [0, 1, 2, 3, 5, 6, 10], "vari": [0, 2], "inform": [0, 1, 2, 10], "appear": [0, 1], "its": [0, 1, 6, 10], "suffix": 0, "mean": [0, 2], "two": [0, 1, 2, 3, 6, 7, 9, 10, 12], "link": [0, 1, 2, 4, 6, 8], "most": [0, 1, 7, 10], "independ": [0, 2], "For": [0, 1, 2, 7, 10, 12], "exampl": [0, 1, 2], "taken": [0, 1], "10": [0, 2, 5, 12], "year": [0, 1, 2, 3, 4], "apart": 0, "want": [0, 1, 2, 10, 12], "standard": [0, 1, 11], "ag": [0, 1, 2, 3], "variabl": [0, 1, 2, 12], "i": [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12], "compar": [0, 1, 2, 6, 7], "between": [0, 1, 2, 6, 7, 10, 12], "To": [0, 1, 5, 7, 10], "could": [0, 2], "creat": [0, 2, 6, 7, 9, 10, 11, 12], "age_at_dataset_b": 0, "ad": [0, 1, 2], "column_map": [0, 2, 7], "valu": [0, 1, 2, 8, 9, 10, 11], "As": 0, "suppos": 0, "record": [0, 1, 2, 6, 7], "person": [0, 1, 6], "": [0, 1, 2, 6, 7, 10, 11], "first": [0, 1, 2, 5, 7, 10, 11], "string": [0, 1, 2, 3, 7, 8, 10, 11], "In": [0, 1, 6, 10, 12], "call": 0, "namefrst": [0, 1, 2], "entir": 0, "lowercas": 0, "first_nam": 0, "uppercas": 0, "follow": [0, 1, 6, 10, 11, 12], "configur": [0, 1, 6, 10, 12], "add": [0, 4], "_": [0, 1, 2, 3, 4, 8, 9, 10], "given": [0, 1, 2, 3, 8, 12], "numer": [0, 1], "11": [0, 2, 5, 9], "concat": 0, "concaten": [0, 1], "end": [0, 1, 2, 3, 11], "col": [0, 1], "togeth": [0, 1, 2], "take": [0, 1, 2, 3, 7, 10], "column_to_append": 0, "multipl": [0, 1, 2, 10], "time": [0, 2, 7, 10], "row": [0, 2], "If": [0, 1, 2, 3, 7, 8, 10, 11], "automat": [0, 2, 5, 7], "convert": [0, 1, 2], "befor": [0, 1, 2, 3, 5, 7], "statefip": [0, 1, 2], "counti": [0, 1], "strip": [0, 7], "alphabet": 0, "charact": 0, "lower": [0, 1], "case": [0, 1, 2, 3, 6], "white": 0, "space": [0, 2, 3, 11], "start": [0, 11], "ration": 0, "word": [0, 4], "replac": [0, 1, 4], "sinc": [0, 2], "peopl": [0, 1, 6, 10], "raw": [0, 2, 7, 10], "censu": [0, 7, 12], "contain": [0, 1, 11], "lead": 0, "better": [0, 6], "match": [0, 1, 4, 6, 10, 11, 12], "remov": 0, "qmark": 0, "hyphen": 0, "punctuat": 0, "apostroph": 0, "altern": [0, 2], "surround": 0, "all": [0, 1, 2, 3, 7, 8, 9, 10], "them": [0, 1, 2, 7], "jr": [0, 2], "sr": [0, 2], "ii": [0, 2], "iii": [0, 2], "stop": 0, "last": [0, 1, 7, 9], "street": [0, 1], "avenu": [0, 11], "blvd": 0, "circl": 0, "court": 0, "road": 0, "prefix": 0, "like": [0, 2, 7, 10], "m": [0, 1], "mr": 0, "ah": 0, "chines": 0, "condens": 0, "whitespac": [0, 7], "leav": 0, "behind": 0, "arrai": [0, 2, 4, 8, 9], "namefrst_split": [0, 2], "namefrst_clean": [0, 2], "index": [0, 5], "select": [0, 1, 4, 6, 10, 12], "element": 0, "posit": [0, 1, 2, 4, 6], "second": [0, 1, 2, 11], "1": [0, 1, 2, 4, 7, 8, 9, 10, 12], "item": 0, "set": [0, 1, 2, 3, 6, 7, 10, 12], "Then": [0, 5], "0": [0, 1, 2, 7, 8, 9, 10, 12], "initi": [0, 1, 10], "probabl": [0, 2, 8], "middl": [0, 1], "namefrst_mid_init": [0, 1], "otherwis": [0, 1, 9, 12], "known": 0, "recod": 0, "birthyr": [0, 2], "clean_birthyr": [0, 2, 3], "9999": [0, 2, 9], "1999": [0, 2], "9998": 0, "divid": 0, "int": [0, 1, 2, 3, 8], "integ": [0, 1, 2, 9], "result": [0, 1, 6, 9, 10, 12], "instanc": [0, 8], "birthplac": 0, "detail": [0, 2, 10], "version": [0, 5, 12], "gener": [0, 1, 4, 6, 7, 10], "least": [0, 1], "signific": 0, "digit": 0, "we": [0, 1, 10, 12], "simpli": [0, 2], "drop": [0, 2, 10], "100": [0, 2, 12], "round": [0, 2], "lowest": 0, "whole": [0, 6], "number": [0, 1, 2, 3, 7, 8, 10], "floor": 0, "function": [0, 1, 2, 6, 10], "bpl": [0, 1, 2], "bpl_root": 0, "condit": [0, 1, 2, 3, 4, 7], "logic": 0, "work": [0, 1, 2, 5, 7, 10, 12], "sql": [0, 1, 2, 3, 4, 7, 10], "express": [0, 1, 2], "claus": [0, 1], "if_valu": 0, "else_valu": 0, "race": [0, 1, 2, 9, 12], "ipum": [0, 6], "code": [0, 1, 2, 5], "categori": [0, 8], "get": [0, 1, 10], "down": [0, 6, 12], "nearest": 0, "produc": [0, 10], "relat": [0, 1, 2], "hundr": 0, "300": 0, "child": [0, 8], "household": [0, 4, 6, 8, 10, 12], "head": 0, "301": 0, "302": 0, "adopt": 0, "303": 0, "step": [0, 1, 2, 6], "usual": [0, 7, 12], "need": [0, 1, 2, 7, 10, 12], "2": [0, 1, 2, 3, 7, 8, 11, 12], "spous": 0, "3": [0, 1, 2, 5, 7, 8, 9, 12], "4": [0, 1, 8], "law": 0, "5": [0, 1, 2, 8, 9, 10, 12], "parent": [0, 1, 11], "6": [0, 2, 8, 9, 12], "7": [0, 1, 2, 8, 12], "sibl": 0, "12": [0, 5], "relate_div_100": [0, 1, 2], "page": [1, 2, 10], "comparison_featur": [1, 2, 7], "along": 1, "header": [1, 2, 3, 9, 11], "context": [1, 3, 9], "relatematch": [1, 2], "comparison_typ": [1, 2], "categor": [1, 2, 8, 9], "true": [1, 2, 3, 7, 9, 11, 12], "maximum": [1, 8], "jaro": [1, 9], "winkler": [1, 9], "find": [1, 7, 12], "greatest": 1, "among": 1, "cartesian": 1, "product": [1, 6, 12], "column": [1, 3, 4, 7, 9, 10, 11, 12], "namelast": [1, 2], "would": [1, 2, 12], "return": [1, 3, 8, 10], "four": 1, "namefrst_a": 1, "namefrst_b": 1, "namelast_b": 1, "namelast_a": 1, "maximum_jw": 1, "score": [1, 2, 7, 9], "namefrst_jw": [1, 2, 12], "geograph": 1, "filter": [1, 4, 7, 11], "major": [1, 10], "locat": [1, 2, 10], "boundari": 1, "zero": 1, "jw_street": 1, "enum_dist": 1, "max": [1, 8, 10], "member": [1, 7], "neighborhood": 1, "surnam": 1, "related_individual_max_jw": 1, "namefrst_rel": 1, "assert": [1, 10], "NOT": 1, "distinct": 1, "f1": 1, "evalu": [1, 2, 6, 7, 8], "ani": [1, 2, 3, 5, 8], "potenti": [1, 4, 7], "mismatch": 1, "queri": [1, 2], "fi": 1, "OR": 1, "mi0": 1, "mi1": 1, "THEN": 1, "els": [1, 2, 3], "first_init_col": 1, "namefrst_init": 1, "mid_init_col": 1, "namefrst_mid_init_2": 1, "f2": 1, "empti": 1, "null": [1, 2, 3], "AND": [1, 2], "individu": [1, 2, 7, 12], "mainli": 1, "caution": [1, 9], "flag": [1, 9, 10, 12], "f": [1, 10], "sp": 1, "m_caution": [1, 2, 9, 12], "mbpl": 1, "mother_birthyr": 1, "stepmom": 1, "momloc": 1, "comp_a": [1, 2], "comp_b": [1, 2], "comp_c": 1, "parent_step_chang": 1, "comp_d": 1, "check": [1, 10], "sign": 1, "boolean": [1, 2, 3, 11, 12], "form": [1, 7, 11], "cast": 1, "namelast_equal_as_int": 1, "namelast_clean": [1, 2, 3], "whether": [1, 2, 11], "join": [1, 11], "across": 1, "being": [1, 7], "exact": [1, 2], "namefrst_unstd": [1, 2], "present": [1, 2, 9], "nonzero": 1, "primarili": [1, 7], "indic": [1, 12], "kind": 1, "incompar": 1, "akin": 1, "miss": [1, 10], "see": [1, 2, 5, 10, 12], "univers": [1, 4, 7], "similar": 1, "fbpl_nomatch": 1, "fbpl": 1, "allow": [1, 2, 7, 12], "up": [1, 2, 10, 11], "sub": 1, "object": [1, 2, 6, 10], "document": [1, 8, 10, 12], "sp_caution": [1, 2, 12], "spouse_bpl": 1, "spouse_birthyr": 1, "durmarr": [1, 2], "new_marr": [1, 2], "street_jw": [1, 2, 12], "9": 1, "multipli": 1, "after": [1, 2, 4, 8, 10], "float": [1, 2, 8], "comp": 1, "c": 1, "sploc": 1, "012": 1, "fals": [1, 2, 3, 4, 6, 10], "d": 1, "under": [1, 2], "specif": [1, 2, 10], "circumst": 1, "should": [1, 2, 8, 9, 10], "mid_init_match": 1, "either_1": 1, "nativ": 1, "either_0": 1, "gen": 1, "imm": [1, 2, 12], "immigr": 1, "look": [1, 10, 11], "foreign": 1, "born": 1, "sgen": [1, 2, 12], "rel": [1, 2, 12], "scala": 1, "determin": [1, 7], "greater": [1, 5], "jw_threshold": 1, "less": [1, 2], "age_threshold": 1, "sex": [1, 2, 11], "sampl": 1, "related_individual_row": 1, "unrel": 1, "depend": [1, 2, 5, 12], "name_col": 1, "birthyr_col": 1, "namefrst_related_row": 1, "replaced_birthyr": [1, 2, 3], "extra": 1, "children": 1, "who": 1, "base": [1, 2, 7], "expect": 1, "count": [1, 10, 12], "suspect": [1, 6], "relate_col": 1, "histid_col": 1, "id": [1, 2], "birth": 1, "year_b": 1, "wa": [1, 12], "minimum": [1, 8], "accept": [1, 2, 12], "consid": [1, 8], "histid": [1, 2, 12], "1910": [1, 2, 12], "8": [1, 2, 5, 10], "rate": 1, "calcul": [1, 12], "percentag": 1, "seen": 1, "neighbor": 1, "meet": 1, "95": 1, "nbor": [1, 2, 12], "namelast_neighbor": 1, "05": [1, 2], "namelast_popularity_sum": 1, "namelast_popular": 1, "length": [1, 2, 9], "size": 1, "ab": 1, "diff": 1, "absolut": 1, "invalid": [1, 8], "instead": [1, 2, 5, 7], "marriag": 1, "durat": 1, "99": [1, 2], "placehold": 1, "unknown": 1, "exclud": 1, "those": [1, 2], "consider": 1, "byrdiff": [1, 2, 12], "mardurmatch": [1, 2], "14": 1, "minu": [1, 2], "subtract": 1, "geo": 1, "distanc": [1, 8], "lookup": 1, "tabl": [1, 2, 4, 7, 10, 12], "core": [1, 7, 10, 12], "dist_tabl": 1, "py": [1, 2], "There": [1, 2, 3, 7], "sever": [1, 6], "wai": [1, 5, 10], "file": [1, 4, 6, 7, 10, 11, 12], "kei": [1, 7, 10], "key_count": 1, "secondari": 1, "serv": 1, "back": 1, "primari": [1, 6], "doe": [1, 7, 12], "particularli": 1, "state": [1, 6], "much": [1, 7], "fewer": [1, 8], "combin": [1, 2, 3, 7], "thu": 1, "risk": 1, "fill": 1, "aren": 1, "ex": 1, "just": [1, 2, 10, 12], "even": 1, "though": 1, "distances_fil": 1, "path": [1, 2, 10, 11, 12], "table_nam": 1, "what": [1, 2, 10, 12], "onc": [1, 10], "loc_a": 1, "where": [1, 7, 10, 12], "come": 1, "loc_b": 1, "distance_col": 1, "source_column_a": 1, "sourc": [1, 4, 7, 10, 12], "source_column_b": 1, "loc_a_0": 1, "loc_a_1": 1, "loc_b_0": 1, "loc_b_1": 1, "secondary_key_count": 1, "backup": 1, "secondary_table_nam": 1, "secondary_distances_fil": 1, "secondary_source_column": 1, "secondary_loc_a": 1, "secondary_loc_b": 1, "secondary_distance_col": 1, "state_dist": 1, "state_distance_lookup": 1, "county_state_dist": 1, "csv": [1, 2, 7, 10, 11, 12], "statecode1": 1, "statecode2": 1, "dist": 1, "county_dist": [1, 2, 12], "county_distance_lookup": 1, "county_1900_1910_distances_km": 1, "from_icpsrctyi": 1, "to_icpsrctyi": 1, "from_statefip": 1, "to_statefip": 1, "distance_km": 1, "state_1900_1910_distances_km": 1, "fetch": 1, "neither": 1, "nor": 1, "mpre": 1, "m_namefrst": 1, "accord": 1, "niu": 1, "other": [1, 2, 12], "mfbplmatch": 1, "multi": 1, "search": 1, "special": 1, "simplifi": 1, "particular": [1, 2], "constraint": 1, "num_col": 1, "whose": 1, "templat": 1, "n": [1, 8, 9], "per": [1, 2, 8, 9, 10], "current": [1, 2, 10], "respect": [1, 7], "jw_col_templ": 1, "jw": 1, "pair": [1, 12], "equal_and_not_null_templ": 1, "final": [1, 2, 12], "comput": [1, 3, 7], "_namefrst": 1, "_bpl": 1, "_sex": 1, "25": 1, "nvl": 1, "sm_namefrst": 1, "sn_namefrst": 1, "sm_bpl": 1, "sn_bpl": 1, "sm_sex": 1, "sn_sex": 1, "pass": [1, 2, 7, 8], "flexibl": 1, "user": [1, 10], "write": [1, 10, 12], "own": [1, 2], "favor": 1, "reason": 1, "good": 1, "fallback": 1, "defin": [1, 7, 8, 9, 10], "spark": [1, 2, 5, 8, 9, 10, 12], "builtin": 1, "argument": [1, 10, 12], "namelast_jw_max": 1, "namelast1": 1, "namelast2": 1, "namelast3": 1, "abov": [1, 5], "extend": 1, "beyond": 1, "top": [1, 4], "level": [1, 4, 10], "everi": 1, "jw_f": [1, 2, 12], "father_namefrst": 1, "rais": [1, 3], "exponenti": 1, "squar": 1, "county_distance_squar": [1, 2, 12], "county_a": 1, "county_b": 1, "upper": 1, "gt": 1, "btwn": 1, "addl": 1, "var": [1, 2], "program": [1, 2, 7, 12], "report": [1, 4, 6, 10], "addl_var": 1, "check_val_expr": 1, "else_v": 1, "volumn": 1, "datasourc": [1, 2, 10], "yrimmig": 1, "immyear_diff": [1, 2, 9, 12], "includ": [1, 2, 7, 9, 10], "train": [1, 4, 6, 8, 10], "independent_var": [1, 2, 12], "config": [1, 4, 7, 10, 12], "id_column": [1, 2], "_a": 1, "mult": 1, "exist": [1, 2, 10], "within": [1, 2, 6, 10, 11], "hh_train": [1, 2, 7, 10, 12], "hh": 1, "highest": [1, 2], "against": [1, 11], "ten": [1, 2], "tell": 2, "how": [2, 7], "descript": [2, 8, 10], "refer": 2, "here": [2, 7, 10, 12], "tutori": [2, 10], "script": [2, 6, 10], "discuss": 2, "readm": 2, "note": 2, "written": [2, 6], "toml": [2, 6, 10], "abl": 2, "json": [2, 10], "datasource_a": [2, 7], "datasource_b": [2, 7], "transform": [2, 4, 6, 7], "lowercase_strip": 2, "add_to_a": 2, "age_2": 2, "derived_from": 2, "expand_length": 2, "explod": [2, 7], "jaro_winkl": 2, "namelast_jw": [2, 12], "threshold": [2, 8, 12], "feature_nam": 2, "79": 2, "84": 2, "complex": 2, "machin": [2, 6, 7, 10, 12], "learn": [2, 6, 7, 10, 12], "probabilist": [2, 6], "drop_data_from_scored_match": 2, "us1900": 2, "us1900m_usa": 2, "p": 2, "parquet": [2, 7], "us1910": 2, "us1910m_usa": 2, "training_data_subset": 2, "serialp": 2, "rationalize_name_word": 2, "remove_qmark_hyphen": 2, "replace_apostroph": 2, "remove_suffix": 2, "remove_alternate_nam": 2, "condense_strip_whitespac": 2, "split": [2, 3, 7, 8, 9, 12], "namefrst_std": [2, 11], "array_index": 2, "bpl_orig": 2, "divide_by_int": 2, "get_floor": 2, "statefip_h": 2, "output_typ": 2, "substitution_column": [2, 7, 11], "join_column": [2, 11], "join_valu": [2, 11], "substitution_fil": [2, 11], "name_std": [2, 11], "male": [2, 11], "femal": [2, 11], "feature_select": [2, 3, 7], "input_column": [2, 3, 9], "output_column": [2, 3, 9], "sql_condit": 2, "namelast_bigram": 2, "bigram": [2, 4], "bpl_clean": 2, "bpl_str": 2, "washington": 2, "bpl2_str": 2, "53": 2, "region": [2, 12], "attach_vari": 2, "region_dict": 2, "col_to_join_on": 2, "col_to_add": 2, "null_fil": 2, "col_typ": 2, "potential_matches_univers": [2, 7], "birthyr_3": 2, "namefrst_std_jw": [2, 12], "75": [2, 8, 12], "comparis": 2, "post": [2, 7], "hh_comparison": [2, 7], "threshold_expr": 2, "fetch_a": 2, "sex_equ": 2, "equal": [2, 11], "relate_a": [2, 9], "pipeline_featur": [2, 7, 9], "sex_region_interact": 2, "transformer_typ": [2, 9], "interact": [2, 4, 7, 12], "relatetyp": [2, 9], "bucket": [2, 7], "hit": [2, 10, 12], "scale_data": [2, 12], "training_data": [2, 10], "dependent_var": [2, 12], "score_with_model": [2, 12], "use_training_data_featur": [2, 7, 12], "split_by_id_a": [2, 12], "decis": [2, 4, 8, 12], "drop_duplicate_with_threshold_ratio": [2, 12], "n_training_iter": [2, 7, 12], "output_suspicious_td": [2, 12], "param_grid": [2, 12], "model_paramet": [2, 7, 8, 12], "random_forest": [2, 12], "maxdepth": [2, 8, 12], "numtre": [2, 8, 12], "005": 2, "threshold_ratio": [2, 8, 12], "logistic_regress": [2, 12], "50": [2, 12], "65": 2, "80": 2, "chosen_model": [2, 8, 12], "prediction_col": 2, "predict": [2, 12], "hh_col": 2, "hh_training_data_1900_1910": 2, "probit": [2, 4], "go": [2, 10], "your": [2, 5, 7, 10, 12], "uniqu": 2, "identifi": [2, 6, 12], "full": [2, 7, 12], "short": 2, "alphanumer": 2, "convert_ints_to_long": 2, "long": [2, 11], "especi": 2, "assum": 2, "schema": 2, "sometim": 2, "term": 2, "bigint": 2, "thing": 2, "my_fil": 2, "subset": [2, 11], "limit": 2, "extract": 2, "modifi": 2, "meant": 2, "usag": [2, 4, 10], "set_value_column_a": [2, 3], "liter": 2, "set_value_column_b": [2, 3], "iv": 2, "v": 2, "vi": 2, "vii": 2, "viii": 2, "namelast_clean_bigram": [2, 3], "fed": [2, 7], "prep": 2, "df": [2, 10], "men": 2, "newli": 2, "attempt": 2, "duplic": [2, 8], "conjuct": 2, "Will": 2, "conjunct": 2, "rang": [2, 9], "original_valu": 2, "plu": 2, "1870": 2, "expand": 2, "1867": 2, "1868": 2, "1869": 2, "1871": 2, "1872": 2, "1873": 2, "kept": 2, "keep": 2, "appropri": 2, "treat": [2, 9], "import": [2, 7, 10, 12], "dure": [2, 7], "hot": 2, "encod": [2, 3], "vector": [2, 9], "stage": 2, "well": 2, "upper_threshold": 2, "cannot": 2, "robust": 2, "ml": [2, 4, 8, 9], "typic": [2, 7], "leverag": 2, "api": [2, 6, 9], "piplin": 2, "regionf": 2, "sex_regionf_interact": 2, "immyear_caut": [2, 9], "myriad": 2, "explor": [2, 4, 6, 10], "part": [2, 7], "task": [2, 4, 6, 8, 12], "drop_duplicate_a": 2, "out": [2, 7, 12], "best": [2, 7], "smallest": 2, "possibl": 2, "ratio": [2, 8], "beta": [2, 8], "test": [2, 7, 12], "model_explor": [2, 10, 12], "hyper": [2, 6, 12], "paramet": [2, 6, 7, 8, 10, 12], "eval": 2, "skip": [2, 7], "apply_model": 2, "run_all_step": [2, 10, 12], "command": [2, 6, 10, 12], "try": 2, "creation": 2, "iter": 2, "scale": 2, "error": [2, 9], "1900": [2, 12], "about": [2, 10, 12], "1930": [2, 12], "1940": [2, 12], "fail": 2, "were": 2, "sure": [2, 5, 10], "scratch": 2, "although": 2, "know": 2, "haven": 2, "save": [2, 7, 12], "small": 2, "amount": 2, "process": [2, 6, 10], "repeatedli": 2, "help": [2, 7, 10], "neg": [2, 4, 6], "area": 2, "coverag": 2, "increas": [2, 9], "represent": [2, 7], "ensur": 2, "group": [2, 7], "a304bt": 2, "three": [2, 7], "b200": 2, "c201": 2, "d425": 2, "perform": [2, 6, 7, 11], "feature_import": [2, 7, 12], "coeffici": [2, 7], "enabl": [2, 7, 10], "srace": [2, 9, 12], "race_interacted_srac": [2, 9, 12], "hits2": [2, 12], "exact_mult": [2, 12], "ncount": [2, 3, 12], "ncount2": [2, 3, 12], "f_interacted_jw_f": [2, 12], "f_caution": [2, 12], "f_pre": [2, 12], "fbplmatch": [2, 12], "m_interacted_jw_m": [2, 9, 12], "jw_m": [2, 9, 12], "m_pre": [2, 9, 12], "mbplmatch": [2, 12], "sp_interacted_jw_sp": [2, 12], "jw_sp": [2, 12], "sp_pre": [2, 12], "mi": [2, 12], "fsoundex": [2, 12], "lsoundex": [2, 12], "oth": [2, 12], "imm_interacted_immyear_caut": [2, 12], "1900_1910_training_data_20191023": 2, "jw_max_a": 2, "jw_max_b": 2, "f1_match": 2, "f2_match": 2, "byrdifcat": 2, "racematch": 2, "bplmatch": 2, "imm_interacted_bplmatch": 2, "sexmatch": 2, "relatetype_interacted_relatematch": 2, "checkpoint": 3, "no_first_pad": 3, "don": 3, "prepend": 3, "namefrst_unstd_bigram": 3, "namelast_frst_bigram": 3, "namelast_clean_soundex": 3, "input_col": 3, "output_col": 3, "expon": 3, "introduct": 4, "overview": 4, "instal": 4, "pypi": 4, "preprocess": [4, 6, 10, 12], "model": [4, 6, 10], "run": [4, 5, 6, 7, 12], "librari": [4, 6], "mode": [4, 5, 12], "advanc": 4, "workflow": 4, "export": [4, 7, 10], "featur": [4, 6, 7, 8, 10], "reus": 4, "basic": 4, "map": [4, 7, 9], "substitut": [4, 7], "block": [4, 7], "comparison": [4, 7], "pipelin": 4, "ons": 4, "aggreg": 4, "union": 4, "soundex": 4, "power": 4, "regex": 4, "random": [4, 8], "forest": [4, 8], "logist": [4, 8], "regress": [4, 8], "tree": [4, 8], "gradient": [4, 8], "boost": [4, 8], "system": 5, "python": [5, 6, 10], "java": 5, "integr": 5, "apach": 5, "via": [5, 6], "pyspark": [5, 8, 9, 10], "packag": 5, "org": 5, "latest": 5, "pip": 5, "easiest": [5, 10], "through": [5, 7, 9, 10], "instruct": [5, 10], "But": 5, "clone": 5, "github": 5, "repositori": 5, "root": 5, "project": 5, "directori": [5, 10, 12], "develop": [5, 6], "e": 5, "dev": 5, "edit": 5, "made": 5, "built": 5, "tool": [5, 6], "line": [6, 10], "share": 6, "characterist": [6, 7], "correspond": [6, 7], "real": 6, "world": 6, "determinist": [6, 7], "rule": [6, 7], "algorithm": [6, 7], "At": [6, 7], "been": 6, "unit": 6, "census": 6, "hierarch": [6, 10], "structur": 6, "nest": 6, "howev": [6, 12], "tailor": 6, "ignor": 6, "common": [6, 7, 12], "highli": [6, 7], "languag": 6, "further": [6, 12], "broken": 6, "smaller": 6, "sequenc": 6, "linkrun": [6, 10], "prepar": [6, 7, 10], "research": 6, "experi": 6, "understand": 6, "tune": [6, 12], "relationship": 6, "varieti": 7, "normal": 7, "abbrevi": [7, 11], "regist": [7, 10], "datafram": [7, 10, 12], "request": 7, "classif": [7, 8], "metadata": 7, "introspect": 7, "ingest": 7, "inspect": 7, "mani": [7, 10], "aspect": [7, 10], "extens": 7, "longest": 7, "definit": 7, "reduc": 7, "drastic": 7, "improv": 7, "runtim": 7, "separ": 7, "total": 7, "potential_match": [7, 10], "satisfi": 7, "elig": 7, "reshap": 7, "thought": 7, "ahead": 7, "chosen": 7, "experiment": [7, 10], "focus": 7, "demograph": 7, "moment": 7, "veri": [7, 12], "anyon": 7, "percent": 7, "remain": 7, "popul": 7, "pull": 7, "fix": 7, "width": 7, "crosswalk": 7, "construct": 7, "alpha": 8, "hyperparamet": [8, 12], "de": 8, "param": [8, 12], "label": 8, "doc": [8, 9], "commonli": 8, "explan": 8, "randomforestclassifi": 8, "depth": 8, "20": 8, "featuresubsetstrategi": 8, "node": 8, "auto": 8, "onethird": 8, "sqrt": 8, "log2": 8, "15": 8, "generalizedlinearregress": 8, "famili": 8, "binomi": 8, "85": [8, 10], "logisticregress": 8, "decisiontreeclassifi": 8, "mininstancespernod": 8, "caus": 8, "left": 8, "right": [8, 10], "discard": 8, "maxbin": 8, "bin": 8, "discret": 8, "continu": [8, 9, 12], "gbtclassifi": 8, "mother": 9, "point": [9, 12], "x": [9, 10], "y": 9, "hold": 9, "except": 9, "strictli": 9, "inf": 9, "explicitli": 9, "cover": 9, "doubl": 9, "outsid": 9, "job": 10, "high": 10, "class": 10, "handl": 10, "main": 10, "complet": 10, "access": [10, 12], "link_run": 10, "factori": 10, "sparkfactori": 10, "load_config": 10, "load_conf_fil": 10, "sparksess": 10, "now": 10, "let": 10, "load": 10, "our": 10, "my_conf": 10, "lr": 10, "prep_step": 10, "get_step": 10, "enumer": 10, "print": 10, "input_table_nam": 10, "output_table_nam": 10, "run_step": 10, "get_tabl": 10, "matches_df": 10, "hh_model_explor": 10, "method": [10, 12], "interfac": 10, "easili": 10, "conveni": 10, "adjust": 10, "set_loc": 10, "set_num_cor": 10, "set_executor_memori": 10, "5g": 10, "ll": 10, "dictionari": 10, "often": 10, "modul": 10, "pleas": 10, "reproduc": 10, "consol": 10, "cpu": 10, "h": 10, "executor_memori": [10, 12], "execute_task": 10, "execute_command": 10, "conf": [10, 12], "show": 10, "messag": 10, "exit": 10, "memori": 10, "executor": 10, "begin": 10, "execut": 10, "seri": 10, "excute_command": 10, "filepath": 10, "sai": 10, "fullcount_1870_1880": 10, "pattern": 10, "full_count_1870_1880": 10, "prompt": 10, "enter": 10, "text": 10, "unstabl": 10, "topic": 10, "analyz": [10, 12], "set_preexisting_t": 10, "x_persist": 10, "borrow_t": 10, "get_task": 10, "set_print_sql": 10, "x_sql": 10, "x_sqlf": 10, "ipython": 10, "showf": 10, "x_summari": 10, "desc": 10, "x_crosswalk": 10, "x_tab": 10, "q": [10, 12], "x_hh_tfam": 10, "x_tfam": 10, "drop_al": 10, "reload": 10, "x_hh_tfam_2a": 10, "x_tfam_raw": 10, "drop_all_prc": 10, "x_hh_tfam_2b": 10, "x_union": 10, "drop_all_temp": 10, "x_load": 10, "get_set": 10, "set_link_task": 10, "x_parquet_from_csv": 10, "organ": 10, "hierarchi": 10, "five": 10, "hh_match": 10, "someth": 10, "choic": 10, "preexist": 10, "prepped_df_a": 10, "prepped_df_b": 10, "raw_df_b": 10, "raw_df_a": 10, "training_featur": [10, 12], "scored_potential_match": 10, "potential_matches_prep": 10, "exploded_df_b": 10, "exploded_df_a": 10, "predicted_match": 10, "hh_training_featur": [10, 12], "hh_training_data": 10, "hh_predicted_match": 10, "hh_scored_potential_match": 10, "hh_potential_match": 10, "hh_blocked_match": 10, "hh_potential_matchs_prep": 10, "model_eval_training_vector": 10, "model_eval_training_data": 10, "model_eval_repeat_fp": 10, "model_eval_training_featur": 10, "model_eval_training_result": 10, "model_eval_repeat_fn": 10, "hh_model_eval_training_vector": 10, "hh_model_eval_repeat_fp": 10, "hh_model_eval_repeat_fn": 10, "hh_model_eval_training_result": 10, "hh_model_eval_training_featur": 10, "hh_model_eval_training_data": 10, "persist": 10, "hidden": 10, "intermedi": 10, "yet": 10, "databas": 10, "tablenam": 10, "istemporari": 10, "task_nam": 10, "num": 10, "finish": 10, "put": [10, 12], "launch": [10, 12], "my": [10, 12], "subhead": 11, "suppli": 11, "regex_word_replac": 11, "variant": 11, "av": 11, "7th": 11, "swap": 11, "still": 11, "anywher": 11, "proceed": 11, "street_unstd": 11, "dir": 11, "substitutions_street_abbrev": 11, "span": 12, "1920": 12, "deriv": 12, "necessari": 12, "scenario": 12, "copi": 12, "use_potential_matches_featur": 12, "full_count_1900_1910": 12, "50g": 12, "ask": 12, "arg": 12, "partit": 12, "training_data_1900_1910_hlink_featur": 12, "might": 12, "shut": 12, "framework": 12, "etc": 12, "relev": 12, "matrix": 12, "implement": 12, "regular": 12, "training_data_1900_1910": 12, "weren": 12, "ident": 12, "manual": 12, "updat": 12, "isn": 12, "analysi": 12, "training_result": 12, "hh_training_result": 12, "1900_1910_training_result": 12, "repeat_fp": 12, "repeat_fn": 12, "hh_repeat_fp": 12, "hh_repeat_fn": 12, "1900_1910_potential_fp": 12, "1900_1910_potential_fn": 12, "prefer": 12, "ve": 12}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"column": [0, 2], "map": [0, 2], "basic": [0, 2], "usag": 0, "advanc": [0, 2, 12], "transform": [0, 1, 3, 9], "add_to_a": 0, "concat_to_a": 0, "concat_to_b": 0, "concat_two_col": 0, "lowercase_strip": 0, "rationalize_name_word": 0, "remove_qmark_hyphen": 0, "remove_punctu": 0, "replace_apostroph": 0, "remove_alternate_nam": 0, "remove_suffix": 0, "remove_stop_word": 0, "remove_prefix": 0, "condense_strip_whitespac": 0, "remove_one_letter_nam": 0, "split": 0, "array_index": 0, "substr": 0, "divide_by_int": 0, "when_valu": 0, "get_floor": 0, "comparison": [1, 2], "type": [1, 9], "add": 1, "ons": 1, "aggreg": 1, "featur": [1, 2, 3, 9, 12], "household": [1, 2, 7], "maximum_jaro_winkl": 1, "jaro_winkl": 1, "jaro_winkler_street": 1, "max_jaro_winkl": 1, "equal": 1, "f1_match": 1, "f2_match": 1, "not_equ": 1, "equals_as_int": 1, "all_equ": 1, "not_zero_and_not_equ": 1, "time": 1, "caution_comp_3": 1, "caution_comp_3_012": 1, "caution_comp_4": 1, "caution_comp_4_012": 1, "any_equ": 1, "either_are_1": 1, "either_are_0": 1, "second_gen_imm": 1, "rel_jaro_winkl": 1, "extra_children": 1, "jaro_winkler_r": 1, "sum": 1, "length_b": 1, "abs_diff": 1, "b_minus_a": 1, "geo_dist": 1, "fetch_a": 1, "fetch_b": 1, "present_both_year": 1, "neither_are_nul": 1, "present_and_matching_categor": 1, "present_and_not_equ": 1, "present_and_equal_categorical_in_univers": 1, "multi_jaro_winkler_search": 1, "sql_condit": [1, 3], "alia": 1, "power": [1, 3], "threshold": 1, "lower_threshold": 1, "upper_threshold": 1, "gt_threshold": 1, "btwn_threshold": 1, "look_at_addl_var": 1, "hit": 1, "hits2": 1, "exact_mult": 1, "jw_max_a": 1, "jw_max_b": 1, "configur": [2, 4, 7], "config": 2, "file": 2, "top": 2, "level": 2, "data": [2, 11, 12], "sourc": [2, 5], "filter": 2, "substitut": [2, 11], "select": [2, 3], "potenti": [2, 12], "match": [2, 7], "univers": 2, "block": 2, "pipelin": [2, 9], "gener": [2, 9, 12], "train": [2, 7, 12], "model": [2, 7, 8, 12], "bigram": 3, "arrai": 3, "union": 3, "soundex": 3, "welcom": 4, "hlink": [4, 10], "": 4, "document": 4, "api": 4, "instal": 5, "requir": 5, "from": 5, "pypi": 5, "introduct": 6, "overview": [6, 7], "link": [7, 10, 12], "task": [7, 10], "preprocess": 7, "step": [7, 10], "relat": 7, "section": 7, "explor": [7, 12], "report": 7, "random_forest": 8, "probit": 8, "logistic_regress": 8, "decision_tre": 8, "gradient_boosted_tre": 8, "interact": [9, 10], "bucket": 9, "run": 10, "us": 10, "librari": 10, "mode": 10, "start": 10, "program": 10, "exampl": [10, 12], "workflow": [10, 12], "1": 11, "tabl": 11, "regex": 11, "word": 11, "replac": 11, "export": 12, "after": 12, "reus": 12, "differ": 12, "year": 12, "ml": 12, "list": 12, "fals": 12, "posit": 12, "neg": 12, "fp": 12, "fn": 12}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 60}, "alltitles": {"Column Mappings": [[0, "column-mappings"], [2, "column-mappings"]], "Basic Usage": [[0, "basic-usage"]], "Advanced Usage": [[0, "advanced-usage"]], "Transforms": [[0, "transforms"]], "add_to_a": [[0, "add-to-a"]], "concat_to_a": [[0, "concat-to-a"]], "concat_to_b": [[0, "concat-to-b"]], "concat_two_cols": [[0, "concat-two-cols"]], "lowercase_strip": [[0, "lowercase-strip"]], "rationalize_name_words": [[0, "rationalize-name-words"]], "remove_qmark_hyphen": [[0, "remove-qmark-hyphen"]], "remove_punctuation": [[0, "remove-punctuation"]], "replace_apostrophe": [[0, "replace-apostrophe"]], "remove_alternate_names": [[0, "remove-alternate-names"]], "remove_suffixes": [[0, "remove-suffixes"]], "remove_stop_words": [[0, "remove-stop-words"]], "remove_prefixes": [[0, "remove-prefixes"]], "condense_strip_whitespace": [[0, "condense-strip-whitespace"]], "remove_one_letter_names": [[0, "remove-one-letter-names"]], "split": [[0, "split"]], "array_index": [[0, "array-index"]], "mapping": [[0, "mapping"]], "substring": [[0, "substring"]], "divide_by_int": [[0, "divide-by-int"]], "when_value": [[0, "when-value"]], "get_floor": [[0, "get-floor"]], "Comparison types, transform add-ons, aggregate features, and household aggregate features": [[1, "comparison-types-transform-add-ons-aggregate-features-and-household-aggregate-features"]], "Comparison types": [[1, "comparison-types"]], "maximum_jaro_winkler": [[1, "maximum-jaro-winkler"]], "jaro_winkler": [[1, "jaro-winkler"]], "jaro_winkler_street": [[1, "jaro-winkler-street"]], "max_jaro_winkler": [[1, "max-jaro-winkler"]], "equals": [[1, "equals"]], "f1_match": [[1, "f1-match"]], "f2_match": [[1, "f2-match"]], "not_equals": [[1, "not-equals"]], "equals_as_int": [[1, "equals-as-int"]], "all_equals": [[1, "all-equals"]], "not_zero_and_not_equals": [[1, "not-zero-and-not-equals"]], "or": [[1, "or"]], "and": [[1, "and"]], "times": [[1, "times"]], "caution_comp_3": [[1, "caution-comp-3"]], "caution_comp_3_012": [[1, "caution-comp-3-012"]], "caution_comp_4": [[1, "caution-comp-4"]], "caution_comp_4_012": [[1, "caution-comp-4-012"]], "any_equals": [[1, "any-equals"]], "either_are_1": [[1, "either-are-1"]], "either_are_0": [[1, "either-are-0"]], "second_gen_imm": [[1, "second-gen-imm"]], "rel_jaro_winkler": [[1, "rel-jaro-winkler"]], "extra_children": [[1, "extra-children"]], "jaro_winkler_rate": [[1, "jaro-winkler-rate"]], "sum": [[1, "sum"]], "length_b": [[1, "length-b"]], "abs_diff": [[1, "abs-diff"]], "b_minus_a": [[1, "b-minus-a"]], "geo_distance": [[1, "geo-distance"]], "fetch_a": [[1, "fetch-a"]], "fetch_b": [[1, "fetch-b"]], "present_both_years": [[1, "present-both-years"]], "neither_are_null": [[1, "neither-are-null"]], "present_and_matching_categorical": [[1, "present-and-matching-categorical"]], "present_and_not_equal": [[1, "present-and-not-equal"]], "present_and_equal_categorical_in_universe": [[1, "present-and-equal-categorical-in-universe"]], "multi_jaro_winkler_search": [[1, "multi-jaro-winkler-search"]], "sql_condition": [[1, "sql-condition"], [3, "sql-condition"]], "Feature add-ons": [[1, "feature-add-ons"]], "alias": [[1, "alias"]], "power": [[1, "power"], [3, "power"]], "threshold": [[1, "threshold"]], "lower_threshold": [[1, "lower-threshold"]], "upper_threshold": [[1, "upper-threshold"]], "gt_threshold": [[1, "gt-threshold"]], "btwn_threshold": [[1, "btwn-threshold"]], "look_at_addl_var": [[1, "look-at-addl-var"]], "Aggregate Features": [[1, "aggregate-features"]], "hits": [[1, "hits"]], "hits2": [[1, "hits2"]], "exact_mult": [[1, "exact-mult"]], "Household Aggregate Features": [[1, "household-aggregate-features"]], "jw_max_a": [[1, "jw-max-a"]], "jw_max_b": [[1, "jw-max-b"]], "Configuration": [[2, "configuration"]], "Basic Config File": [[2, "basic-config-file"]], "Advanced Config File": [[2, "advanced-config-file"]], "Top level configs": [[2, "top-level-configs"]], "Data sources": [[2, "data-sources"]], "Filter": [[2, "filter"]], "Substitution Columns": [[2, "substitution-columns"]], "Feature Selections": [[2, "feature-selections"]], "Potential Matches Universe": [[2, "potential-matches-universe"]], "Blocking": [[2, "blocking"]], "Comparisons": [[2, "comparisons"]], "Household Comparisons": [[2, "household-comparisons"]], "Comparison Features": [[2, "comparison-features"]], "Pipeline-generated Features": [[2, "pipeline-generated-features"]], "Training and models": [[2, "training-and-models"]], "Household training and models": [[2, "household-training-and-models"]], "Welcome to hlink\u2019s documentation!": [[4, "welcome-to-hlink-s-documentation"]], "Configuration API": [[4, "configuration-api"], [4, null]], "Installation": [[5, "installation"]], "Requirements": [[5, "requirements"]], "Installing from PyPI": [[5, "installing-from-pypi"]], "Installing from source": [[5, "installing-from-source"]], "Introduction": [[6, "introduction"]], "Overview": [[6, "overview"], [7, "overview"], [7, "id1"], [7, "id4"], [7, "id7"], [7, "id10"], [7, "id13"]], "Link Tasks": [[7, "link-tasks"]], "Preprocessing": [[7, "preprocessing"]], "Task steps": [[7, "task-steps"], [7, "id2"], [7, "id5"], [7, "id8"], [7, "id11"], [7, "id14"]], "Related Configuration Sections": [[7, "related-configuration-sections"], [7, "id3"], [7, "id6"], [7, "id9"], [7, "id12"], [7, "id15"]], "Training and Household Training": [[7, "training-and-household-training"]], "Matching": [[7, "matching"]], "Household Matching": [[7, "household-matching"]], "Model Exploration and Household Model Exploration": [[7, "model-exploration-and-household-model-exploration"]], "Reporting": [[7, "reporting"]], "Models": [[8, "models"]], "random_forest": [[8, "random-forest"]], "probit": [[8, "probit"]], "logistic_regression": [[8, "logistic-regression"]], "decision_tree": [[8, "decision-tree"]], "gradient_boosted_trees": [[8, "gradient-boosted-trees"]], "Pipeline generated features": [[9, "pipeline-generated-features"]], "Transformer types": [[9, "transformer-types"]], "interaction": [[9, "interaction"]], "bucketizer": [[9, "bucketizer"]], "Running hlink": [[10, "running-hlink"]], "Using hlink as a Library": [[10, "using-hlink-as-a-library"]], "Interactive Mode": [[10, "interactive-mode"]], "Starting the program": [[10, "starting-the-program"]], "Running Linking Tasks and Steps": [[10, "running-linking-tasks-and-steps"]], "Example interactive mode workflow": [[10, "example-interactive-mode-workflow"]], "Substitutions": [[11, "substitutions"]], "1:1 substitution by data table": [[11, "substitution-by-data-table"]], "Substitution by regex word replace": [[11, "substitution-by-regex-word-replace"]], "Advanced Workflow Examples": [[12, "advanced-workflow-examples"]], "Export training data after generating features to reuse in different linking years": [[12, "export-training-data-after-generating-features-to-reuse-in-different-linking-years"]], "Example training data export with generated ML features": [[12, "example-training-data-export-with-generated-ml-features"]], "ML model exploration and export of lists of potential false positives/negatives in training data": [[12, "ml-model-exploration-and-export-of-lists-of-potential-false-positives-negatives-in-training-data"]], "Example model exploration and FP/FN export workflow": [[12, "example-model-exploration-and-fp-fn-export-workflow"]], "Feature Selection transforms": [[3, "feature-selection-transforms"]], "bigrams": [[3, "bigrams"]], "array": [[3, "array"]], "union": [[3, "union"]], "soundex": [[3, "soundex"]]}, "indexentries": {}})
\ No newline at end of file
diff --git a/hlink/linking/core/transforms.py b/hlink/linking/core/transforms.py
index 6df980b..e7073b5 100755
--- a/hlink/linking/core/transforms.py
+++ b/hlink/linking/core/transforms.py
@@ -3,6 +3,8 @@
 # in this project's top-level directory, and also on-line at:
 #   https://github.com/ipums/hlink
 
+from typing import Any
+
 from pyspark.sql.functions import (
     array,
     collect_list,
@@ -24,13 +26,18 @@
 )
 from pyspark.sql.types import ArrayType, LongType, StringType
 from pyspark.ml import Pipeline
-from pyspark.sql import Window
+from pyspark.sql import DataFrame, SparkSession, Window
 from pyspark.ml.feature import NGram, RegexTokenizer, CountVectorizer, MinHashLSH
 
 
 def generate_transforms(
-    spark, df_selected, feature_selections, link_task, is_a, id_col
-):
+    spark: SparkSession,
+    df_selected: DataFrame,
+    feature_selections: list[dict[str, Any]],
+    link_task,
+    is_a: bool,
+    id_col: str,
+) -> DataFrame:
     not_skipped_feature_selections = [
         c
         for c in feature_selections
@@ -43,7 +50,9 @@ def generate_transforms(
         if ("post_agg_feature" in c) and c["post_agg_feature"]
     ]
 
-    def parse_feature_selections(df_selected, feature_selection, is_a):
+    def parse_feature_selections(
+        df_selected: DataFrame, feature_selection: dict[str, Any], is_a: bool
+    ) -> DataFrame:
         transform = feature_selection["transform"]
 
         if not feature_selection.get("output_column", False):
@@ -114,9 +123,9 @@ def parse_feature_selections(df_selected, feature_selection, is_a):
             return df_selected
 
         elif transform == "array":
-            col1, col2 = feature_selection["input_columns"]
+            input_cols = feature_selection["input_columns"]
             output_col = feature_selection["output_column"]
-            df_selected = df_selected.withColumn(output_col, array(col1, col2))
+            df_selected = df_selected.withColumn(output_col, array(input_cols))
             return df_selected
 
         elif transform == "union":
@@ -300,7 +309,7 @@ def union_list(list_a, list_b):
     for feature_selection in not_skipped_feature_selections:
         df_selected = parse_feature_selections(df_selected, feature_selection, is_a)
 
-    def get_transforms(name, is_a):
+    def get_transforms(name: str, is_a: bool) -> list[dict[str, Any]]:
         to_process = []
         for f in not_skipped_feature_selections:
             if ("override_column_a" in f) and is_a:
diff --git a/sphinx-docs/feature_selection_transforms.md b/sphinx-docs/feature_selection_transforms.md
index 0e7e332..c87440a 100644
--- a/sphinx-docs/feature_selection_transforms.md
+++ b/sphinx-docs/feature_selection_transforms.md
@@ -46,10 +46,10 @@ transform = "sql_condition"
 
 ## array
 
-Combine two input columns into an array output column.
+Combine any number of input columns into a single array output column.
 
 * Attributes:
-  * `input_columns` - Type: list of strings. Required. The two input columns.
+  * `input_columns` - Type: list of strings. Required. The list of input columns.
   * `output_column` - Type: `string`. Required.
 
 ```


Test patch:
diff --git a/hlink/tests/core/transforms_test.py b/hlink/tests/core/transforms_test.py
new file mode 100644
index 0000000..f940dbb
--- /dev/null
+++ b/hlink/tests/core/transforms_test.py
@@ -0,0 +1,92 @@
+from pyspark.sql import Row, SparkSession
+import pytest
+
+from hlink.linking.core.transforms import generate_transforms
+from hlink.linking.link_task import LinkTask
+
+
+@pytest.mark.parametrize("is_a", [True, False])
+def test_generate_transforms_array_transform_1_col(
+    spark: SparkSession, preprocessing: LinkTask, is_a: bool
+) -> None:
+    df = spark.createDataFrame(
+        [[1, "Leto II", 3508], [2, "Hwi", 26], [3, "Siona", 25]],
+        schema=["id", "name", "age"],
+    )
+    feature_selections = [
+        {
+            "transform": "array",
+            "input_columns": ["name"],
+            "output_column": "array_column",
+        }
+    ]
+
+    df_result = generate_transforms(
+        spark, df, feature_selections, preprocessing, is_a, "id"
+    )
+    array_column = df_result.select("array_column").collect()
+    assert array_column == [
+        Row(array_column=["Leto II"]),
+        Row(array_column=["Hwi"]),
+        Row(array_column=["Siona"]),
+    ]
+
+
+@pytest.mark.parametrize("is_a", [True, False])
+def test_generate_transforms_array_transform_2_cols(
+    spark: SparkSession, preprocessing: LinkTask, is_a: bool
+) -> None:
+    df = spark.createDataFrame(
+        [[1, "Leto II", 3508], [2, "Hwi", 26], [3, "Siona", 25]],
+        schema=["id", "name", "age"],
+    )
+    feature_selections = [
+        {
+            "transform": "array",
+            "input_columns": ["name", "age"],
+            "output_column": "array_column",
+        }
+    ]
+
+    df_result = generate_transforms(
+        spark, df, feature_selections, preprocessing, is_a, "id"
+    )
+    array_column = df_result.select("array_column").collect()
+    assert array_column == [
+        Row(array_column=["Leto II", "3508"]),
+        Row(array_column=["Hwi", "26"]),
+        Row(array_column=["Siona", "25"]),
+    ]
+
+
+@pytest.mark.parametrize("is_a", [True, False])
+def test_generate_transforms_array_transform_3_cols(
+    spark: SparkSession,
+    preprocessing: LinkTask,
+    is_a: bool,
+) -> None:
+    df = spark.createDataFrame(
+        [
+            [1, "Leto II", 3508, "Arrakis"],
+            [2, "Hwi", 26, "Ix"],
+            [3, "Siona", 25, "Arrakis"],
+        ],
+        schema=["id", "name", "age", "home"],
+    )
+    feature_selections = [
+        {
+            "transform": "array",
+            "input_columns": ["home", "age", "name"],
+            "output_column": "array_column",
+        }
+    ]
+
+    df_result = generate_transforms(
+        spark, df, feature_selections, preprocessing, is_a, "id"
+    )
+    array_column = df_result.select("array_column").collect()
+    assert array_column == [
+        Row(array_column=["Arrakis", "3508", "Leto II"]),
+        Row(array_column=["Ix", "26", "Hwi"]),
+        Row(array_column=["Arrakis", "25", "Siona"]),
+    ]