Package: dotifi
dotifiA utility to generateDOTfiles and images based ongraphvizfrom the canvas of anApache NiFiinstance.dotifi supportsconnecting to Apache Nifi usingnipyapiin the background and generating a graph from the canvas itemsProcess GroupsRemote Process GroupsInput and Output PortsProcessorsConnectionscreatingDOTfiles usingGraphvizandpygraphvizcreating a graphic file in Graphviz supported formatsstarting with an existing dot file as opposed to building from a live NiFi instancestarting from a specific process groupcontrolling the depth of recursionsupplying DOT files as templates for configuringthe root graph ( and all children)Any process group by IDsupplying name value pairs ( in yaml configuration file) to override and setattributesProcessors by idRemote Process Groups by idssl, certificates, and auth-tokensInstall>pipinstalldotifiwith python 3 as your defaultSample outputDOTstrict digraph "nifi flow" {
	graph [compound=true,
		label="nifi flow",
		outputorder=edgesfirst,
		rankdir=LR,
		ratio=1.0
	];
	node [fixedsize=false,
		fontsize=8,
		label="\N",
		shape=rectangle,
		style=filled
	];
	edge [color="#1100FF",
		style="setlinewidth(2)"
	];
	subgraph cluster_root_1 {
		graph [id="351b1dbc-0172-1000-056d-ec78a003b493",
			label=root_1
		];
		subgraph cluster_remote_target {
			graph [id="65f57b35-0172-1000-d805-dec5391bee7e",
				label=remote_target
			];
			"65f5b8db-0172-1000-407f-93e85d1d9f94"	[id="65f5b8db-0172-1000-407f-93e85d1d9f94",
				label="Input 1 : INPUT_PORT",
				pos="-768.000000,-768.000000"];
			"65f629b8-0172-1000-4ced-efa192c35536"	[id="65f629b8-0172-1000-4ced-efa192c35536",
				label="log_remote_attributes : LogAttribute",
				pos="-344.000000,-344.000000"];
			"65f5b8db-0172-1000-407f-93e85d1d9f94" -> "65f629b8-0172-1000-4ced-efa192c35536"	[label="Input to Processor"];
		}
		subgraph cluster_root_1_1 {
			graph [id="351dbb56-0172-1000-21fa-a4af9fc0dbb1",
				label=root_1_1
			];
			subgraph "cluster_NiFi Flow" {
				graph [color=blue,
					fontcolor=white,
					id="65f8c7d5-0172-1000-a916-0e5562295e08",
					label="http://localhost:8080/nifi : Remote Process Group",
					style=filled
				];
				"37b065f7-232b-30b9-a282-3ea28f75a2a2"	[id="37b065f7-232b-30b9-a282-3ea28f75a2a2",
					label="Input 1 : INPUT_PORT"];
			}
			"b268f335-2a91-44e8-b567-c676c292fd55"	[id="b268f335-2a91-44e8-b567-c676c292fd55",
				label="Reformat Date Column : ReplaceText",
				pos="1259.009165,1259.009165"];
			"25b139c3-db89-4272-59c3-774ce58f0973"	[id="25b139c3-db89-4272-59c3-774ce58f0973",
				label="No-Op Termination : UpdateAttribute",
				pos="1265.009165,1265.009165"];
			"b268f335-2a91-44e8-b567-c676c292fd55" -> "25b139c3-db89-4272-59c3-774ce58f0973"	[label=success];
			"b268f335-2a91-44e8-b567-c676c292fd55" -> "37b065f7-232b-30b9-a282-3ea28f75a2a2"	[label=success];
			"67626c71-910a-4342-627d-e38cfcbcb924"	[id="67626c71-910a-4342-627d-e38cfcbcb924",
				label="Set CSV Content : ReplaceText",
				pos="408.000000,408.000000"];
			"67626c71-910a-4342-627d-e38cfcbcb924" -> "b268f335-2a91-44e8-b567-c676c292fd55"	[label=success];
			"bc3ad300-c93c-430b-fc94-3a32c3020f53"	[id="bc3ad300-c93c-430b-fc94-3a32c3020f53",
				label="Generate Empty File : GenerateFlowFile",
				pos="356.009165,356.009165"];
			"bc3ad300-c93c-430b-fc94-3a32c3020f53" -> "67626c71-910a-4342-627d-e38cfcbcb924"	[label=success];
		}
		"719d5fb9-cd96-4491-fe7c-e81b3f3bb555"	[id="719d5fb9-cd96-4491-fe7c-e81b3f3bb555",
			label="q=nifi : UpdateAttribute",
			pos="1242.304169,1242.304169"];
		"e829b880-ef17-432a-7c5c-1e3ecdcd690d"	[id="e829b880-ef17-432a-7c5c-1e3ecdcd690d",
			label="Search Google : InvokeHTTP",
			pos="1245.295837,1245.295837"];
		"719d5fb9-cd96-4491-fe7c-e81b3f3bb555" -> "e829b880-ef17-432a-7c5c-1e3ecdcd690d"	[label=success];
		"f96be8b1-78b2-42f2-6ba5-2579f4f6c411"	[id="f96be8b1-78b2-42f2-6ba5-2579f4f6c411",
			label="Source : GenerateFlowFile",
			pos="425.000000,425.000000"];
		"f96be8b1-78b2-42f2-6ba5-2579f4f6c411" -> "719d5fb9-cd96-4491-fe7c-e81b3f3bb555"	[label=success];
		"83ca901a-efe0-4d97-dbca-5c557cefbe6f"	[id="83ca901a-efe0-4d97-dbca-5c557cefbe6f",
			label="HashContent : HashContent",
			pos="1264.422387,1264.422387"];
		"ffcc8819-75f5-4811-e906-434e71a31f26"	[id="ffcc8819-75f5-4811-e906-434e71a31f26",
			label="Route On Status Code : RouteOnAttribute",
			pos="1242.078949,1242.078949"];
		"ffcc8819-75f5-4811-e906-434e71a31f26" -> "83ca901a-efe0-4d97-dbca-5c557cefbe6f"	[label=200];
		"e829b880-ef17-432a-7c5c-1e3ecdcd690d" -> "ffcc8819-75f5-4811-e906-434e71a31f26"	[label=Original];
		"e829b880-ef17-432a-7c5c-1e3ecdcd690d" -> "e829b880-ef17-432a-7c5c-1e3ecdcd690d"	[label=Retry];
	}
}PNGpoetrydotifi is maintained usingpoetryfor dependency management and packaging.Getting started for developmentInstallGraphvizClone, fork, or download thesourceInstallpoetryIf required setuppyenvor your preference to get a python 3.8 environment, as poetry will use whatever the current python is.for example setup pyenv local to the project directoryIn the source route directory runpoetry install, this will install all the dependenciesRunpeotry run pytest -vto run the tests and ensure things are workingRunpoetry buildthenpoetry installthenpoetry run dotifiwith any options to runSetting upJetbrains PyCharmwith your virtual python environmentseethis reddit answerConfigurationdotifi usesconfusefor it's configuration management.
dotifi will take most of its configuration:from commandline parametersfrom a yaml configuration file specified with the --with-conf-file parametersome combination of the twoIf a configuration fileispresent, and some configuration value is set both from the commandline and in the configuration,
then the commandline is treated as an override for the configuration file.String values in the configuration file may be entered as shell environment variables, such as $PATH.  Variables are expanded usingos.path.expandvars.usage:dotifi[-h][--with-existing-dot-fileWITH_EXISTING_DOT_FILE][--with-conf-fileWITH_CONF_FILE][--output-dot-fileOUTPUT_DOT_FILE][--output-graphviz-format{canon,cmap,cmapx,cmapx_np,dia,dot,fig,gd,gd2,gif,hpgl,imap,imap_np,ismap,jpe,jpeg,jpg,mif,mp,pcl,pdf,pic,plain,plain-ext,png,ps,ps2,svg,svgz,vml,vmlz,vrml,vtx,wbmp,xdot,xlib}][--output-graphviz-program{neato,dot,twopi,circo,fdp,nop}][--output-graphviz-fileOUTPUT_GRAPHVIZ_FILE][--start-at-pgSTART_AT_PG][--depthDEPTH][--nifi-urlNIFI_URL][--using-ssl][--using-user-pw][--ca-fileCA_FILE][--client-cert-fileCLIENT_CERT_FILE][--client-key-fileCLIENT_KEY_FILE][--client-key-passwordCLIENT_KEY_PASSWORD][--auth-tokenAUTH_TOKEN][--auth-token-api-fieldAUTH_TOKEN_API_FIELD][--nifi-usernameNIFI_USERNAME][--nifi-user-passwordNIFI_USER_PASSWORD][--verbose][--generate-mock-data][--mock-data-fileMOCK_DATA_FILE]optionalarguments:-h,--helpshowthishelpmessageandexit--with-existing-dot-fileWITH_EXISTING_DOT_FILE,-wWITH_EXISTING_DOT_FILEOutputwillbebasedonanexistingDOTgraphdefinitionasopposedtobeingbuiltfromNiFi--with-conf-fileWITH_CONF_FILE,-cWITH_CONF_FILEPathtothe.yamlfilewiththeconfiguration.Alloptionscanbesetintheconfiguration,with--output-dot-fileOUTPUT_DOT_FILE,-oOUTPUT_DOT_FILEPathtothegvfiletostorethedotresultsto.--output-graphviz-format{canon,cmap,cmapx,cmapx_np,dia,dot,fig,gd,gd2,gif,hpgl,imap,imap_np,ismap,jpe,jpeg,jpg,mif,mp,pcl,pdf,pic,plain,plain-ext,png,ps,ps2,svg,svgz,vml,vmlz,vrml,vtx,wbmp,xdot,xlib},-f{canon,cmap,cmapx,cmapx_np,dia,dot,fig,gd,gd2,gif,hpgl,imap,imap_np,ismap,jpe,jpeg,jpg,mif,mp,pcl,pdf,pic,plain,plain-ext,png,ps,ps2,svg,svgz,vml,vmlz,vrml,vtx,wbmp,xdot,xlib}Theformatofthegraphvizgeneratedfile.Formats(notallmaybeavailableoneverysystemdependingonhowGraphvizwasbuilt)--output-graphviz-program{neato,dot,twopi,circo,fdp,nop}Graphvizlayoutmethodtouse.--output-graphviz-fileOUTPUT_GRAPHVIZ_FILE,-gOUTPUT_GRAPHVIZ_FILEPathtothegraphicalfiletostorethegraphvizresultsto.Resultswillbesavedwiththeextensionofthe--output-graphviz-formatoption--start-at-pgSTART_AT_PG,-sSTART_AT_PGTheidoftheprocessgrouptostartat.Thiswillbeauuid.Whensettheoutputwillstartwiththisprocessandit's descendents based on the depthsetting--depth DEPTH, -d DEPTHThe depth to descend to within nested process groups.Note that the top level canvas is the root processgroup. As such a depth of 0 will only output items inthe root canvas and not any process groups itcontains. A value of -1 means unlimited.--nifi-url NIFI_URL, -n NIFI_URLThe url of the NiFi instance to connect to. This isused if --with-existing is not set.--using-ssl           Flag, when specified it signals that the NiFiconnection requires SSL--using-user-pw       Flag, when specified it signals that the NiFiconnection requires a username and password--ca-file CA_FILE     A PEM file containing certs for the root CA(s) for theNiFi server--client-cert-file CLIENT_CERT_FILEA PEM file containing the public certificates for theuser / client identity--client-key-file CLIENT_KEY_FILEAn encrypted (password -protected PEM file containingthe client'ssecretkey--client-key-passwordCLIENT_KEY_PASSWORDThepasswordtodecrypttheclient_key_file--auth-tokenAUTH_TOKENAuthorizationtokenstring--auth-token-api-fieldAUTH_TOKEN_API_FIELDtheapi_keyfieldnametosetthetokento.Defaultsto'tokenAuth'--nifi-usernameNIFI_USERNAMETheNiFiusername--nifi-user-passwordNIFI_USER_PASSWORDTheNiFiuserpassword--verbose,-vSetstheloggingleveltoverbose--generate-mock-dataGeneratesmockdata--mock-data-fileMOCK_DATA_FILEWhen--generate-mock-dataisspecified,itwillbewrittentothisfileA sample of the yaml configuration ishere# All string values in this configuration may be replaced using# environmental variables in the form of $VARIABLENAME## the version of this configurationconfiguration_version:1#  Flag to set logging to verbose ( debugging level )verbose:#  Path to an existing DOT graph definition as opposed to being built from NiFiwith_existing_dot_file:# Path to the dot file to store the dot results tooutput_dot_file:#  The format of the graphviz generated file. Formats (not all may be available on every system#  depending on how Graphviz was built#  "canon", "cmap", "cmapx", "cmapx_np", "dia", "dot", "fig", "gd", "gd2", "gif",#  "hpgl", "imap", "imap_np", "ismap", "jpe", "jpeg", "jpg", "mif", "mp", "pcl", "pdf",#  "pic", "plain", "plain_ext", "png", "ps", "ps2", "svg", "svgz", "vml", "vmlz", "vrml",#  "vtx", "wbmp", "xdot", "xlib"output_graphviz_format:#  The Graphviz layout to use# ‘neato’|’dot’|’twopi’|’circo’|’fdp’|’nop'output_graphviz_program:#  Path to the dot file to store the graphviz results to. Results will be saved with the extension#  output_graphviz_format optionoutput_graphviz_file:#  The id of the process group to start at.  This will be a uuid.  When set the output will start#  with this process and it's decedents based on the depth settingstart_at_pg:#  The depth to descend to within nested process groups.  Note that the top level canvas#  is the root process group.  As such a depth of 0 will only output items in the root canvas and#  not any process groups it contains.  A value of _1 means unlimiteddepth:# The url of the nifi instance to connect to.  This is used if with_existing is not setnifi_url:# Flag, when specified it signals that the NiFi connection requires SSLusing_ssl:# Flag, when specified it signals that the NiFi connection requires a username and passwordusing_user_pw:# A PEM file containing certs for the root CA(s) for the NiFi serveca_file:# A PEM file containing the public certificates for the user / client identityclient_cert_file:# An encrypted (password _protected PEM file containing the client's secret keyclient_key_file:# The password to decrypt the client_key_fileclient_key_password:# The NiFi user namenifi_user_name:# The NiFi user passwordnifi_user_password:# Options for the entire graphgraph:# Path to a dot file that contains top level graph definition that sets the properties# and attributes at a graph level# see https://graphviz.org/documentation/ for information on the dot languagetemplate:bar.dot# Options for specific process groups, by idprocess_groups:# the uuid id of the process group351b1dbc-0172-1000-056d-ec78a003b493:# Path to a dot file the contains the graph definition that sets the properties# and attributes at a graph level for this process group and it's descendentstemplate:foo.dot# Options for specific remote process groups, by id# these values will override or mix in with the defaultsremote_process_groups:# the uuid of the remote process group35199793-0172-1000-02ea-52da1888a03d:# NODE attributes# see https://graphviz.gitlab.io/_pages/doc/info/attrs.htmlcolor:grey# Options for specific processors, by idprocessors:# the uuid of the processor351b1dbc-0172-1000-056d-ec78a003b49:# NODE attributes# see https://graphviz.gitlab.io/_pages/doc/info/attrs.htmlcolor:bluecommand line parameters that are described asflagsneed only be specified without a value to be set toTrue.
YAML configuration values however need to be explicitly set toTrueorFalse