Module Expr Str Import alias Import alias Import alias Import alias ImportFrom alias ImportFrom alias ClassDef bases NameLoad body FunctionDef arguments args NameParam NameParam NameParam defaults body Assign AttributeStore NameLoad attr NameLoad Assign AttributeStore NameLoad attr NameLoad decorator_list decorator_list ClassDef bases AttributeLoad NameLoad attr body Expr Str FunctionDef arguments args NameParam NameParam defaults body TryExcept body Expr Call AttributeLoad NameLoad attr NameLoad handlers ExceptHandler type NameLoad name NameStore body If Call AttributeLoad NameLoad attr Str AttributeLoad NameLoad attr body Return AttributeLoad NameLoad attr orelse Raise Call NameLoad BinOpMod Str TupleLoad NameLoad AttributeLoad NameLoad attr Raise Call NameLoad BinOpMod Str NameLoad decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore TupleLoad TupleLoad NameLoad Str TupleLoad Str Str TupleLoad Str Str TupleLoad Str BinOpMod Str Call AttributeLoad NameLoad attr Str TupleLoad Str Str For TupleStore NameStore NameStore NameLoad body Expr Call AttributeLoad NameLoad attr Str NameLoad Assign NameStore Call AttributeLoad NameLoad attr NameLoad Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Dict Str NameLoad Dict Str Str NameLoad NameLoad Dict Str Str NameLoad AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr BinOpMod Str NameLoad decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str Str NameLoad BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr NameLoad Dict Str Str NameLoad AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Dict Str NameLoad Dict Str NameLoad Dict Str Str NameLoad AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str Str NameLoad Str Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Dict Str NameLoad Dict Str Str NameLoad BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str Str NameLoad AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Dict Str Dict Str Num Expr Call AttributeLoad NameLoad attr NameLoad Dict Str Dict Str Num Expr Call AttributeLoad NameLoad attr Dict Str Dict Str Str Num Num NameLoad Assign NameStore Dict Str Dict Str Num Expr Call AttributeLoad NameLoad attr NameLoad Dict Str Dict Str Num Expr Call AttributeLoad NameLoad attr Dict Str Str Dict Str Num Dict Str Num NameLoad Assign NameStore Dict Str Str Expr Call AttributeLoad NameLoad attr NameLoad Dict Str Str Expr Call AttributeLoad NameLoad attr Dict Str Str NameLoad decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str Str NameLoad BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str Str Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str Str NameLoad Str Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore Call AttributeLoad NameLoad attr Str Assign NameStore Dict AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str Str ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad ListLoad Dict BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr Str BinOpMod AttributeLoad AttributeLoad NameLoad attr attr ListLoad Str Str Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Dict Str NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr Expr Call AttributeLoad NameLoad attr NameLoad AttributeLoad AttributeLoad NameLoad attr attr decorator_list FunctionDef arguments args NameParam defaults body Assign NameStore ListLoad Str Str For NameStore NameLoad body Assign NameStore Call AttributeLoad AttributeLoad NameLoad attr attr Call AttributeLoad AttributeLoad NameLoad attr attr NameLoad Str NameLoad Assign NameStore ListComp NameLoad comprehension NameStore Call AttributeLoad NameLoad attr NameLoad Call AttributeLoad NameLoad attr Str Expr Call AttributeLoad NameLoad attr Str NameLoad NameLoad For NameStore NameLoad body Expr Call AttributeLoad NameLoad attr Str NameLoad Call AttributeLoad AttributeLoad NameLoad attr attr NameLoad Assign NameStore Call AttributeLoad Call NameLoad Call AttributeLoad AttributeLoad NameLoad attr attr NameLoad NameLoad attr Assign NameStore Call AttributeLoad NameLoad attr NameLoad Expr Call AttributeLoad NameLoad attr AttributeLoad AttributeLoad NameLoad attr attr decorator_list decorator_list If CompareEq NameLoad Str body Expr Call AttributeLoad NameLoad attr
N/A N/A Pipeline config linter unit tests. N/A json N/A os N/A re N/A logging src basetest src.pipelines linter MockPipeline N/A object N/A __init__ N/A N/A self name config N/A N/A N/A N/A self name name N/A N/A self config config N/A N/A LintPipelineTest N/A N/A basetest TestCase N/A N/A Tests the pipeline linter. getUnterminatedStringMessage N/A N/A self input_string N/A N/A N/A N/A N/A N/A N/A json loads input_string N/A N/A N/A ValueError N/A e N/A N/A N/A N/A re match Unterminated string starting at: line \d+ column \d+ \(char \d+\) N/A e message N/A N/A N/A e message N/A N/A N/A ValueError N/A expected %r to raise a ValueError that looked like "Unterminated string" but got %r N/A input_string N/A e message N/A N/A ValueError N/A expected %r to raise a ValueError input_string N/A testJunk N/A N/A self N/A N/A N/A bad_strings N/A N/A None PreTemplate: expected string or buffer N/A PreTemplate: No JSON object could be decoded N/A " PreTemplate: end is out of bounds N/A "fish N/A PreTemplate: %s N/A N/A self getUnterminatedStringMessage "fish N/A fish PreTemplate: No JSON object could be decoded N/A N/A bad_string reason bad_strings N/A N/A N/A N/A logging info testing: %r bad_string N/A pl N/A N/A linter PipelineLinter bad_string N/A expect N/A N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES N/A pass True N/A pass reason False reason N/A pass reason False N/A N/A linter PipelineLinter MSG_MISSING_IO_STAGES N/A N/A N/A self assertFalse N/A N/A pl results valid N/A Config:%r  bad_string N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A Config:%r bad_string N/A testSimpleFailure N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "GcsInput"}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A pass True N/A pass reason False N/A N/A N/A linter StageLinter MSG_REQUIRE_AT_LEAST_ONE_FMT N/A object objects N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testMissingStageType N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT None N/A pass reason False N/A N/A linter StageLinter MSG_TYPE_NOT_FOUND N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testMissingInputStageType N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES N/A pass True N/A pass True N/A pass reason False N/A N/A linter PipelineLinter MSG_MISSING_IO_STAGES N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testBadStageType N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "UnknownInput"}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT UnknownInput N/A pass reason False No module named unknowninput N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testBadSection N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"input": [{"type": "UnknownInput"}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES N/A pass True N/A pass reason False N/A N/A N/A linter PipelineLinter MSG_UNKNOWN_CONFIG_KEYS_FMT input N/A pass reason False N/A N/A linter PipelineLinter MSG_MISSING_IO_STAGES N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testUpdateNestedDict N/A N/A self N/A N/A N/A o N/A h N/A b 1 N/A N/A N/A linter UpdateNestedDict o N/A h N/A c 2 N/A N/A N/A self assertSameStructure N/A h N/A b c 1 2 o N/A o N/A h N/A b 1 N/A N/A N/A linter UpdateNestedDict o N/A z N/A c 2 N/A N/A N/A self assertSameStructure N/A h z N/A b 1 N/A c 2 o N/A o N/A h j N/A N/A N/A linter UpdateNestedDict o N/A h k N/A N/A N/A self assertSameStructure N/A h j o N/A testGcsInputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "GcsInput", "object": "gs://b1/o1", "sinks": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testGcsInputNullSink N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "GcsInput", "object": "gs://b1/o1", "sinks": [null]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A pass True N/A pass reason False N/A N/A N/A linter StageLinter MSG_FIELD_INVALID_FMT null N/A pass True N/A pass True N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testGcsOutputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"outputs": [{"type": "GcsOutput", "object": "gs://b1/o1", "sources": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A outputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsOutput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sources N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testGcsCompositorOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "GcsInput", "object": "gs://b1/o1", "sinks": ["gs:/b2/o2"]}],"transforms": [{"type": "GcsCompositor", "sources": ["gs:/b2/o2"], "sinks": ["gs:/b3/o3"], "contentType": "text/plain"}],"outputs": [{"type": "GcsOutput", "object": "gs://b3/o3", "sources": ["gs:/b4/o4"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs transforms outputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsCompositor N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sources N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT contentType N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsOutput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sources N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testHttpInputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "HttpInput", "url": "http://foo/data.csv", "sinks": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT HttpInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT url N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testHttpInputBadShardSize N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "HttpInput", "url": "http://foo/data.csv", "shardSize": 33554433, "sinks": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT HttpInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT url N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT shardSize N/A pass True N/A pass True N/A pass True N/A pass reason False Invalid value: 'Size exceeds App Engine response limit.' N/A N/A N/A self assertFalse N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testDatastoreInputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "DatastoreInput", "gql": "SELECT *", "params": {"projection": ["a", "b"]}, "sinks": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT DatastoreInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_TYPE_FMT params N/A N/A N/A linter StageLinter CHECK_TYPE_FMT params.projection N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A gql object N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testS3InputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "S3Input", "object": "s3://b/o", "s3Credentials": {"accessKey": "123", "accessSecret": "abc"}, "sinks": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT S3Input N/A N/A N/A linter StageLinter CHECK_TYPE_FMT s3Credentials N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT s3Credentials N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT s3Credentials.accessKey N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT s3Credentials.accessSecret N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testBigQueryOutputOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"outputs": [{"type": "BigQueryOutput", "destinationTable": {"projectId": "123", "tableId": "abc", "datasetId": "xyz"}, "schema": {"fields": [{"type": "STRING"}]}, "sources": ["gs:/b2/o2"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A outputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT BigQueryOutput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sources N/A N/A N/A linter StageLinter CHECK_TYPE_FMT destinationTable N/A N/A N/A linter StageLinter CHECK_TYPE_FMT schema N/A N/A N/A linter StageLinter CHECK_TYPE_FMT schema.fields N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT destinationTable N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT schema N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT destinationTable.projectId N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT destinationTable.tableId N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT destinationTable.datasetId N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT schema.fields N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testCsvMatchReplaceOk N/A N/A self N/A N/A N/A pl N/A N/A linter PipelineLinter {"inputs": [{"type": "GcsInput", "object": "gs://b1/o1", "sinks": ["gs:/b2/o2"]}], "transforms": [{"type": "CsvMatchReplace", "fieldDelimiter": ",", "columns": [{"wanted": true,  "type": "STRING", "name": "col1"}], "sources": ["gs://bucket/foo.csv"], "sinks": ["gs://bucket/results", "gs://bucket/badrows"]}]} N/A expect N/A N/A N/A linter PipelineLinter CHECK_SYNTAX_VALID N/A N/A linter PipelineLinter CHECK_UNKNOWN_CONFIG_KEYS N/A N/A linter PipelineLinter CHECK_REQ_IO_STAGES stages N/A pass True N/A pass True N/A pass True N/A transforms inputs N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT CsvMatchReplace N/A N/A N/A linter StageLinter CHECK_TYPE_FMT columns N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sources N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT fieldDelimiter N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT columns N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A N/A N/A linter StageLinter CHECK_TYPE_FMT GcsInput N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT sinks N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT object N/A N/A N/A linter StageLinter CHECK_FIELD_EXISTS_FMT N/A object objects N/A pass True N/A pass True N/A pass True N/A pass True N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A self assertSameStructure expect N/A N/A pl results results N/A testLintFullFiles N/A N/A self N/A N/A N/A directories N/A src/pipelines/testdata static/examples N/A directory directories N/A N/A directory N/A N/A N/A os path join N/A N/A N/A os path dirname __file__ ../.. directory N/A filenames N/A x N/A x N/A N/A os listdir directory N/A N/A x endswith .json N/A N/A N/A logging info directory %s files %r directory filenames N/A filename filenames N/A N/A N/A N/A logging info Linting %r from %r filename N/A N/A N/A os path basename directory N/A j N/A N/A N/A open N/A N/A N/A os path join directory filename read N/A pl N/A N/A linter PipelineLinter j N/A N/A N/A self assertTrue N/A N/A pl results valid N/A N/A N/A N/A __name__ __main__ N/A N/A N/A N/A basetest main