Project: YugabyteDB (App Definition and Development / Database)
File: cr-assert-assumptions-ok-sql.md

title: Create procedure assertassumptionsok() linkTitle: Create assertassumptionsok() headerTitle: Create the procedure assertassumptionsok() description: Creates a procedure checks that all of the assumptions made about the data imported from the .csv files holds menu: v2.18: identifier: cr-assert-assumptions-ok-sql parent: ingest-scripts weight: 30 type: docs The background for the tests that this procedure performs is explained in the section . It makes use of these features of the "array" data type: the constructor. the function to get all the values returned by `SELECT` execution as a `text[]` array in a single PL/pgSQL-to-SQL round trip. the function to return the number of elements in an array. the terse construct to iterate of the array's values. Save this script as "cr-assert-assumptions-ok.sql" ```plpgsql drop procedure if exists assertassumptionsok(date, date) cascade; create procedure assertassumptionsok(startsurveydate in date, endsurveydate in date) language plpgsql as $body$ declare -- Each survey date (i.e. "timevalue") has exactly the same number of states (i.e. "geovalue"). -- Each state has the same number, of survey date values. expected_states constant text[] not null := array[ 'ak', 'al', 'ar', 'az', 'ca', 'co', 'ct', 'dc', 'de', 'fl', 'ga', 'hi', 'ia', 'id', 'il', 'in', 'ks', 'ky', 'la', 'ma', 'md', 'me', 'mi', 'mn', 'mo', 'ms', 'mt', 'nc', 'nd', 'ne', 'nh', 'nj', 'nm', 'nv', 'ny', 'oh', 'ok', 'or', 'pa', 'ri', 'sc', 'sd', 'tn', 'tx', 'ut', 'va', 'vt', 'wa', 'wi', 'wv', 'wy' ]; expectedstatecount constant int := cardinality(expected_states); actualstatesqry constant text not null := 'select arrayagg(distinct geovalue order by geo_value) from ?'; actual_states text[] not null := '{}'; expecteddates date[] not null := array[startsurvey_date]; actualdatesqry constant text not null := 'select arrayagg(distinct timevalue order by time_value) from ?'; actual_dates date[] not null := '{}'; expecteddatecount int not null := 0; names constant covidcast_names[] not null := ( select arrayagg((csvfile, stagingtable, signal)::covidcastnames) from covidcast_names); expectedtotalcount int not null := 0; r covidcast_names not null := ('', '', ''); d date not null := startsurveydate; t text not null := ''; n int not null := 0; b boolean not null := false; begin loop d := d + interval '1 day'; expecteddates := expecteddates||d; exit when d >= endsurveydate; end loop; expecteddatecount := cardinality(expected_dates); expectedtotalcount := expectedstatecount*expecteddatecount; foreach r in array names loop -- signal: One of covidcast_names.signal. execute replace('select distinct signal from ?', '?', r.staging_table) into t; assert t = r.signal, 'signal from '||r.staging_table||' <> "'||r.signal||'"'; -- geo_type: state. execute 'select distinct geotype from '||r.stagingtable into t; assert t = 'state', 'geotype from '||r.stagingtable||' <> "state"'; -- data_source: fb-survey. execute 'select distinct datasource from '||r.stagingtable into t; assert t = 'fb-survey', 'datasource from '||r.stagingtable||' <> "fb-survey"'; -- direction: IS NULL. execute $$select distinct coalesce(direction, '<null>') from $$||r.staging_table into t; assert t = '<null>', 'direction from '||r.staging_table||' <> "<null>"'; -- Expected total count(*). execute 'select count(*) from '||r.staging_table into n; assert n = expectedtotalcount, 'count from '||r.stagingtable||' <> expectedtotal_count'; -- geo_value: Check list of actual distinct states is as expected. execute replace(actualstatesqry, '?', r.stagingtable) into actualstates; assert actualstates = expectedstates, 'actualstates <> expectedstates'; -- geovalue: Expected distinct state (i.e. "geovalue") count(*). execute 'select count(distinct geovalue) from '||r.stagingtable into n; assert n = expectedstatecount, 'distinct state count per survey date from

<> expectedstate_count'; -- time_value: Check list of actual distinct survey dates is as expected. execute replace(actualdatesqry, '?', r.stagingtable) into actualdates; assert actualdates = expecteddates, 'actualdates <> expecteddates'; -- timevalue: Expected distinct survey date (i.e. "timevalue") count(*). execute 'select count(distinct timevalue) from '||r.stagingtable into n; assert n = expecteddatecount, 'distinct survey date count per state from '||r.stagingtable||' <> expecteddate_count'; -- Same number of states (i.e. "geovalue") for each distinct survey date (i.e. "timevalue"). execute ' with r as ( select timevalue, count(timevalue) as n from '||r.staging_table||' group by time_value) select distinct n from r' into n; assert n = expectedstatecount, 'distinct state count from '||r.stagingtable||' <> expectedstate_count'; -- Same number of survey dates (i.e. "timevalue") for each distinct state (i.e. geovalue). execute ' with r as ( select geovalue, count(geovalue) as n from '||r.staging_table||' group by geo_value) select distinct n from r' into n; assert n = expecteddatecount, 'distinct state count from '||r.stagingtable||' <> expecteddate_count'; -- value: check is legal percentage value. execute ' select max(value) between 0 and 100 and min(value) between 0 and 100 from '||r.staging_table into b; assert b, 'max(value), min(value) from '||r.staging_table||' both < 100 FALSE'; end loop; -- code and geo_value: check same exact one-to-one correspondence in all staging tables. declare chkcodeandgeovalues constant text := $$ with a1 as ( select tochar(code, '90')||' '||geovalue as v from ?1), v1 as ( select v, count(v) as n from a1 group by v), a2 as ( select tochar(code, '90')||' '||geovalue as v from ?2), v2 as ( select v, count(v) as n from a2 group by v), a3 as ( select tochar(code, '90')||' '||geovalue as v from ?3), v3 as ( select v, count(v) as n from a3 group by v), v4 as (select v, n from v1 except select v, n from v2), v5 as (select v, n from v2 except select v, n from v1), v6 as (select v, n from v1 except select v, n from v3), v7 as (select v, n from v3 except select v, n from v1), r as ( select v, n from v4 union all select v, n from v5 union all select v, n from v6 union all select v, n from v6) select count(*) from r$$; begin execute replace(replace(replace(chkcodeandgeovalues, '?1', names[1].staging_table), '?2', names[2].staging_table), '?3', names[3].staging_table ) into n; assert n = 0, '(code, geo_value) tuples from the three staging tables disagree'; end; -- Check set of (geovalue, timevalue) values same in each staging table. declare chkputativepks constant text := ' with v1 as ( select geovalue, timevalue from ?1 except select geovalue, timevalue from ?2), v2 as ( select geovalue, timevalue from ?2 except select geovalue, timevalue from ?1), v3 as ( select geovalue, timevalue from ?1 except select geovalue, timevalue from ?3), v4 as ( select geovalue, timevalue from ?3 except select geovalue, timevalue from ?1), v5 as ( select geovalue, timevalue from v1 union all select geovalue, timevalue from v2 union all select geovalue, timevalue from v3 union all select geovalue, timevalue from v4) select count(*) from v5'; begin execute replace(replace(replace(chkputativepks, '?1', names[1].staging_table), '?2', names[2].staging_table), '?3', names[3].staging_table) into n; assert n = 0, 'pk values from ' || replace(replace(replace('?1, ?2, ?3', '?1', names[1].staging_table), '?2', names[2].staging_table), '?3', names[3].staging_table) || ' do not line up'; end; end; $body$; ```