Repository: projectdiscovery/katana
File: cmd/katana/main.go (Go)

Code before:
				gologger.Error().Msgf("Couldn't create resume file: %s\n", err)
			}

			os.Exit(0)
		}
	}()

	if err := katanaRunner.ExecuteCrawling(); err != nil {
		gologger.Fatal().Msgf("could not execute crawling: %s", err)
	}

	// on successful execution:

	// deduplicate the lines in each file in the store-field-dir
	//use options.StoreFieldDir once https://github.com/projectdiscovery/katana/pull/877 is merged
	storeFieldDir := "katana_field"
	_ = deduplicateLinesInFilesInDir(storeFieldDir)

	// remove the resume file in case it exists
	if fileutil.FileExists(resumeFilename) {
		os.Remove(resumeFilename)
	}

}

func deduplicateLinesInFilesInDir(dir string) error {
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			return deduplicateLinesInFile(path)
		}
		return nil
	})
	if err != nil {
		return errorutil.NewWithErr(err).Msgf("error processing directory %s", dir)
	}
	return nil
}

func deduplicateLinesInFile(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return errorutil.NewWithErr(err).Msgf("could not open file: %s", filename)
	}
	defer file.Close()

	seenLines := make(map[string]struct{})
	var deduplicatedLines []string


Review comment (suggestion):
What about moving them in utils with a note about potential high memory with large files (in our case the output of katana should never big that huge)

Code after:
				gologger.Error().Msgf("Couldn't create resume file: %s\n", err)
			}

			os.Exit(0)
		}
	}()

	if err := katanaRunner.ExecuteCrawling(); err != nil {
		gologger.Fatal().Msgf("could not execute crawling: %s", err)
	}

	// on successful execution:

	// deduplicate the lines in each file in the store-field-dir
	//use options.StoreFieldDir once https://github.com/projectdiscovery/katana/pull/877 is merged
	storeFieldDir := "katana_field"
	_ = folderutil.DedupeLinesInFiles(storeFieldDir)

	// remove the resume file in case it exists
	if fileutil.FileExists(resumeFilename) {
		os.Remove(resumeFilename)
	}

}

func readFlags() (*goflags.FlagSet, error) {
	flagSet := goflags.NewFlagSet()
	flagSet.SetDescription(`Katana is a fast crawler focused on execution in automation
pipelines offering both headless and non-headless crawling.`)

	flagSet.CreateGroup("input", "Input",
		flagSet.StringSliceVarP(&options.URLs, "list", "u", nil, "target url / list to crawl", goflags.FileCommaSeparatedStringSliceOptions),
		flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg"),
		flagSet.StringSliceVarP(&options.Exclude, "exclude", "e", nil, "exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)", goflags.CommaSeparatedStringSliceOptions),
	)

	flagSet.CreateGroup("config", "Configuration",
		flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.FileCommaSeparatedStringSliceOptions),
		flagSet.IntVarP(&options.MaxDepth, "depth", "d", 3, "maximum depth to crawl"),
		flagSet.BoolVarP(&options.ScrapeJSResponses, "js-crawl", "jc", false, "enable endpoint parsing / crawling in javascript file"),
		flagSet.BoolVarP(&options.ScrapeJSLuiceResponses, "jsluice", "jsl", false, "enable jsluice parsing in javascript file (memory intensive)"),
		flagSet.DurationVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for (s, m, h, d) (default s)"),
		flagSet.StringVarP(&options.KnownFiles, "known-files", "kf", "", "enable crawling of known files (all,robotstxt,sitemapxml), a minimum depth of 3 is required to ensure all known files are properly crawled."),
		flagSet.IntVarP(&options.BodyReadSize, "max-response-size", "mrs", math.MaxInt, "maximum response size to read"),
		flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait for request in seconds"),
		flagSet.BoolVarP(&options.AutomaticFormFill, "automatic-form-fill", "aff", false, "enable automatic form filling (experimental)"),
		flagSet.BoolVarP(&options.FormExtraction, "form-extraction", "fx", false, "extract form, input, textarea & select elements in jsonl output"),
		flagSet.IntVar(&options.Retries, "retry", 1, "number of times to retry the request"),
		flagSet.StringVar(&options.Proxy, "proxy", "", "http/socks5 proxy to use"),
		flagSet.StringSliceVarP(&options.CustomHeaders, "headers", "H", nil, "custom header/cookie to include in all http request in header:value format (file)", goflags.FileStringSliceOptions),
		flagSet.StringVar(&cfgFile, "config", "", "path to the katana configuration file"),