func sqlFiles(dirPath, repoBase string) ([]SQLFile, error) {
	fileInfos, err := ioutil.ReadDir(dirPath)
	if err != nil {
		return nil, err
	}
	result := make([]SQLFile, 0, len(fileInfos))
	for _, fi := range fileInfos {
		name := fi.Name()
		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
			dest, err := os.Readlink(filepath.Join(dirPath, name))
			if err != nil {
				continue
			}
			dest = filepath.Clean(dest)
			if !filepath.IsAbs(dest) {
				if dest, err = filepath.Abs(filepath.Join(dirPath, dest)); err != nil {
					continue
				}
			}
			if !strings.HasPrefix(dest, repoBase) {
				continue
			}
			if fi, err = os.Lstat(dest); err != nil {  
				continue
			}
		}
		destName := fi.Name()
		if strings.HasSuffix(destName, ".sql") && fi.Mode().IsRegular() {
			sf := SQLFile{
				Dir:      dirPath,
				FileName: name,  
			}
			result = append(result, sf)
		}
	}
	return result, nil
}
sqlFiles returns a slice of SQLFile for all *.sql files found in the supplied
path. This function does not recursively search subdirs, and does not parse
or validate the SQLFile contents in any way. An error will only be returned
if the directory cannot be read.
The repoBase affects evaluation of symlinks; any link destinations outside
of the repoBase are ignored.
package fs
import (
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	log "github.com/sirupsen/logrus"
	"github.com/skeema/mybase"
	"github.com/skeema/skeema/internal/util"
	"github.com/skeema/tengo"
)
type Dir struct {
	Path              string
	Config            *mybase.Config
	OptionFile        *mybase.File
	SQLFiles          []SQLFile
	LogicalSchemas    []*LogicalSchema  
	ParseError        error             
	IgnoredStatements []*Statement      
	repoBase          string            
}
type LogicalSchema struct {
	Name      string
	CharSet   string
	Collation string
	Creates   map[tengo.ObjectKey]*Statement
	Alters    []*Statement  
}
func (logicalSchema *LogicalSchema) AddStatement(stmt *Statement) error {
	switch stmt.Type {
	case StatementTypeCreate:
		if origStmt, already := logicalSchema.Creates[stmt.ObjectKey()]; already {
			return DuplicateDefinitionError{
				ObjectKey: stmt.ObjectKey(),
				FirstFile: origStmt.File,
				FirstLine: origStmt.LineNo,
				DupeFile:  stmt.File,
				DupeLine:  stmt.LineNo,
			}
		}
		logicalSchema.Creates[stmt.ObjectKey()] = stmt
		return nil
	case StatementTypeAlter:
		logicalSchema.Alters = append(logicalSchema.Alters, stmt)
		return nil
	default:
		return nil
	}
}
func ParseDir(dirPath string, globalConfig *mybase.Config) (*Dir, error) {
	cleaned, err := filepath.Abs(filepath.Clean(dirPath))
	if err != nil {
		return nil, err
	}
	dir := &Dir{
		Path:   cleaned,
		Config: globalConfig.Clone(),
	}
	var parentFiles []*mybase.File
	parentFiles, dir.repoBase, err = ParentOptionFiles(dirPath, globalConfig)
	if err != nil {
		return nil, err
	}
	for _, optionFile := range parentFiles {
		dir.Config.AddSource(optionFile)
	}
	dir.parseContents()
	return dir, dir.ParseError
}
func (dir *Dir) String() string {
	return dir.Path
}
func (dir *Dir) BaseName() string {
	return filepath.Base(dir.Path)
}
func (dir *Dir) RelPath() string {
	rel, err := filepath.Rel(dir.repoBase, dir.Path)
	if dir.repoBase == "" || err != nil {
		return dir.BaseName()
	}
	return rel
}
func (dir *Dir) Delete() error {
	return os.RemoveAll(dir.Path)
}
func (dir *Dir) HasFile(name string) (bool, error) {
	_, err := os.Lstat(filepath.Join(dir.Path, name))
	if err == nil {
		return true, nil
	} else if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
func (dir *Dir) Subdirs() ([]*Dir, error) {
	fileInfos, err := ioutil.ReadDir(dir.Path)
	if err != nil {
		return nil, err
	}
	result := make([]*Dir, 0, len(fileInfos))
	for _, fi := range fileInfos {
		if fi.IsDir() && fi.Name()[0] != '.' {
			sub := &Dir{
				Path:     filepath.Join(dir.Path, fi.Name()),
				Config:   dir.Config.Clone(),
				repoBase: dir.repoBase,
			}
			sub.parseContents()
			result = append(result, sub)
		}
	}
	return result, nil
}
func (dir *Dir) CreateSubdir(name string, optionFile *mybase.File) (*Dir, error) {
	dirPath := filepath.Join(dir.Path, name)
	if dir.OptionFile != nil && dir.OptionFile.SomeSectionHasOption("schema") {
		return nil, fmt.Errorf("Cannot use dir %s: parent option file %s defines schema option", dirPath, dir.OptionFile)
	} else if _, ok := dir.Config.Source("schema").(*mybase.File); ok {
		return nil, fmt.Errorf("Cannot use dir %s: an ancestor option file defines schema option", dirPath)
	}
	if fi, err := os.Stat(dirPath); os.IsNotExist(err) {
		err = os.MkdirAll(dirPath, 0777)
		if err != nil {
			return nil, fmt.Errorf("Unable to create directory %s: %s", dirPath, err)
		}
	} else if err != nil {
		return nil, err
	} else if !fi.IsDir() {
		return nil, fmt.Errorf("Path %s already exists but is not a directory", dirPath)
	} else {
		fileInfos, err := ioutil.ReadDir(dirPath)
		if err != nil {
			return nil, err
		}
		for _, fi := range fileInfos {
			if fi.Name() == ".skeema" {
				return nil, fmt.Errorf("Cannot use dir %s: already has .skeema file", dirPath)
			} else if strings.HasSuffix(fi.Name(), ".sql") {
				return nil, fmt.Errorf("Cannot use dir %s: Already contains *.sql files", dirPath)
			}
		}
	}
	if optionFile != nil {
		optionFile.Dir = dirPath
		if err := optionFile.Write(false); err != nil {
			return nil, fmt.Errorf("Cannot use dir %s: Unable to write to %s: %s", dirPath, optionFile.Path(), err)
		}
	}
	sub := &Dir{
		Path:     dirPath,
		Config:   dir.Config.Clone(),
		repoBase: dir.repoBase,
	}
	sub.parseContents()
	return sub, sub.ParseError
}
func (dir *Dir) CreateOptionFile(optionFile *mybase.File) (err error) {
	if dir.OptionFile != nil {
		return fmt.Errorf("Directory %s already has an option file", dir)
	}
	optionFile.Dir = dir.Path
	if err := optionFile.Write(false); err != nil {
		return fmt.Errorf("Unable to write to %s: %s", optionFile.Path(), err)
	}
	if dir.OptionFile, err = parseOptionFile(dir.Path, dir.repoBase, dir.Config); err != nil {
		return err
	}
	dir.Config.AddSource(dir.OptionFile)
	return nil
}
func (dir *Dir) Hostnames() ([]string, error) {
	if dir.Config.Changed("host-wrapper") {
		variables := map[string]string{
			"HOST":        dir.Config.Get("host"),
			"ENVIRONMENT": dir.Config.Get("environment"),
			"DIRNAME":     dir.BaseName(),
			"DIRPATH":     dir.Path,
			"SCHEMA":      dir.Config.Get("schema"),
		}
		shellOut, err := util.NewInterpolatedShellOut(dir.Config.Get("host-wrapper"), variables)
		if err != nil {
			return nil, err
		}
		return shellOut.RunCaptureSplit()
	}
	return dir.Config.GetSlice("host", ',', true), nil
}
func (dir *Dir) Instances() ([]*tengo.Instance, error) {
	hosts, err := dir.Hostnames()
	if err != nil {
		return nil, err
	} else if len(hosts) == 0 {
		return nil, nil
	}
	var userAndPass string
	if !dir.Config.Changed("password") {
		userAndPass = dir.Config.Get("user")
	} else {
		userAndPass = fmt.Sprintf("%s:%s", dir.Config.Get("user"), dir.Config.Get("password"))
	}
	params, err := dir.InstanceDefaultParams()
	if err != nil {
		return nil, fmt.Errorf("Invalid connection options: %s", err)
	}
	portValue := dir.Config.GetIntOrDefault("port")
	portWasSupplied := dir.Config.Supplied("port")
	portIsntDefault := dir.Config.Changed("port")
	socketValue := dir.Config.Get("socket")
	socketWasSupplied := dir.Config.Supplied("socket")
	var instances []*tengo.Instance
	for _, host := range hosts {
		var dsn string
		thisPortValue := portValue
		if host == "localhost" && (socketWasSupplied || !portWasSupplied) {
			dsn = fmt.Sprintf("%s@unix(%s)/?%s", userAndPass, socketValue, params)
		} else {
			splitHost, splitPort, err := tengo.SplitHostOptionalPort(host)
			if err != nil {
				return nil, err
			}
			if splitPort > 0 {
				if portIsntDefault && portValue != splitPort {
					return nil, fmt.Errorf("Port was supplied as %d inside hostname %s but as %d in option file", splitPort, host, portValue)
				}
				host = splitHost
				thisPortValue = splitPort
			}
			dsn = fmt.Sprintf("%s@tcp(%s:%d)/?%s", userAndPass, host, thisPortValue, params)
		}
		instance, err := util.NewInstance("mysql", dsn)
		if err != nil {
			if dir.Config.Changed("password") {
				safeUserPass := fmt.Sprintf("%s:*****", dir.Config.Get("user"))
				dsn = strings.Replace(dsn, userAndPass, safeUserPass, 1)
			}
			return nil, fmt.Errorf("Invalid connection information for %s (DSN=%s): %s", dir, dsn, err)
		}
		instances = append(instances, instance)
	}
	return instances, nil
}
func (dir *Dir) FirstInstance() (*tengo.Instance, error) {
	instances, err := dir.Instances()
	if len(instances) == 0 || err != nil {
		return nil, err
	}
	var lastErr error
	for _, instance := range instances {
		if lastErr = dir.ValidateInstance(instance); lastErr == nil {
			return instance, nil
		}
	}
	if len(instances) == 1 {
		return nil, fmt.Errorf("Unable to connect to %s for %s: %s", instances[0], dir, lastErr)
	}
	return nil, fmt.Errorf("Unable to connect to any of %d instances for %s; last error %s", len(instances), dir, lastErr)
}
func (dir *Dir) ValidateInstance(instance *tengo.Instance) error {
	ok, err := instance.Valid()
	if !ok {
		return err
	}
	instFlavor := instance.Flavor()
	confFlavor := tengo.NewFlavor(dir.Config.Get("flavor"))
	if instFlavor.Known() {
		if confFlavor != tengo.FlavorUnknown && instFlavor.Family() != confFlavor.Family() {
			log.Warnf("Instance %s actual flavor %s differs from dir %s configured flavor %s", instance, instFlavor, dir, confFlavor)
		}
	} else if confFlavor.Known() {
		log.Debugf("Instance %s flavor cannot be parsed; using dir %s configured flavor %s instead", instance, dir, confFlavor)
		instance.SetFlavor(confFlavor)
	} else {
		log.Warnf("Unable to determine database vendor/version of %s. To set manually, use the \"flavor\" option in %s", instance, filepath.Join(dir.Path, ".skeema"))
	}
	return nil
}
func (dir *Dir) SchemaNames(instance *tengo.Instance) (names []string, err error) {
	if !dir.Config.Changed("schema") {
		return nil, nil
	}
	schemaValue := dir.Config.Get("schema")                         
	rawSchemaValue := dir.Config.GetRaw("schema")                   
	if rawSchemaValue != schemaValue && rawSchemaValue[0] == '`' {  
		variables := map[string]string{
			"HOST":        instance.Host,
			"PORT":        strconv.Itoa(instance.Port),
			"USER":        dir.Config.Get("user"),
			"PASSWORD":    dir.Config.Get("password"),
			"ENVIRONMENT": dir.Config.Get("environment"),
			"DIRNAME":     dir.BaseName(),
			"DIRPATH":     dir.Path,
		}
		shellOut, err := util.NewInterpolatedShellOut(schemaValue, variables)
		if err == nil {
			names, err = shellOut.RunCaptureSplit()
		}
		if err != nil {
			return nil, err
		}
	} else if schemaValue == "*" || looksLikeRegex(schemaValue) {
		if names, err = instance.SchemaNames(); err != nil {
			return nil, err
		}
		sort.Strings(names)
		if schemaValue != "*" {
			re, err := regexp.Compile(schemaValue[1 : len(schemaValue)-1])
			if err != nil {
				return nil, err
			}
			keepNames := []string{}
			for _, name := range names {
				if re.MatchString(name) {
					keepNames = append(keepNames, name)
				}
			}
			names = keepNames
		}
	} else {
		names = dir.Config.GetSlice("schema", ',', true)
	}
	ignoreSchema, err := dir.Config.GetRegexp("ignore-schema")
	if err != nil {
		return nil, err
	}
	return filterSchemaNames(names, ignoreSchema), nil
}
func looksLikeRegex(input string) bool {
	return len(input) > 2 && input[0] == '/' && input[len(input)-1] == '/'
}
func filterSchemaNames(names []string, ignoreSchema *regexp.Regexp) []string {
	systemSchemas := map[string]bool{
		"information_schema": true,
		"performance_schema": true,
		"sys":                true,
		"mysql":              true,
	}
	keepNames := make([]string, 0, len(names))
	for _, name := range names {
		if ignoreSchema != nil && ignoreSchema.MatchString(name) {
			log.Debugf("Skipping schema %s because ignore-schema='%s'", name, ignoreSchema)
		} else if !systemSchemas[strings.ToLower(name)] {
			keepNames = append(keepNames, name)
		}
	}
	return keepNames
}
func (dir *Dir) HasSchema() bool {
	if dir.OptionFile != nil {
		if val, _ := dir.OptionFile.OptionValue("schema"); val != "" {
			return true
		}
	}
	for _, logicalSchema := range dir.LogicalSchemas {
		if logicalSchema.Name != "" {
			return true
		}
	}
	return false
}
var reUseStatement = regexp.MustCompile(`(?i)\bUSE\b`)
func (dir *Dir) NamedSchemaStatements() []*Statement {
	result := make([]*Statement, 0)
	checkFiles := make(map[*TokenizedSQLFile]bool, 0)
	for _, ls := range dir.LogicalSchemas {
		if ls.Name != "" {
			for _, stmt := range ls.Creates {
				if stmt.ObjectQualifier != "" {
					result = append(result, stmt)
				} else {
					checkFiles[stmt.FromFile] = true
				}
			}
			for _, stmt := range ls.Alters {
				if stmt.ObjectQualifier != "" {
					result = append(result, stmt)
				} else {
					checkFiles[stmt.FromFile] = true
				}
			}
		}
	}
	for tokenizedFile := range checkFiles {
		for _, stmt := range tokenizedFile.Statements {
			if stmt.Type == StatementTypeCommand && reUseStatement.MatchString(stmt.Text) {
				result = append(result, stmt)
			}
		}
	}
	return result
}
func (dir *Dir) InstanceDefaultParams() (string, error) {
	banned := map[string]bool{
		"allowallfiles":     true,
		"checkconnliveness": true,
		"clientfoundrows":   true,
		"columnswithalias":  true,
		"interpolateparams": true,  
		"loc":               true,
		"multistatements":   true,
		"parsetime":         true,
		"autocommit":             true,  
		"foreign_key_checks":     true,  
		"default_storage_engine": true,  
	}
	options, err := util.SplitConnectOptions(dir.Config.Get("connect-options"))
	if err != nil {
		return "", err
	}
	v := url.Values{}
	v.Set("timeout", "5s")
	v.Set("readTimeout", "20s")
	v.Set("writeTimeout", "5s")
	sslMode, err := dir.Config.GetEnum("ssl-mode", "disabled", "preferred", "required")
	if err != nil {
		return "", err
	}
	if sslMode == "disabled" || (dir.Config.IsTest && !dir.Config.Supplied("ssl-mode")) {
		v.Set("tls", "false")
	} else if sslMode == "required" {
		v.Set("tls", "skip-verify")
	} else {
		v.Set("tls", sslMode)
	}
	for name, value := range options {
		if banned[strings.ToLower(name)] {
			return "", fmt.Errorf("connect-options is not allowed to contain %s", name)
		}
		if name == "tls" && dir.Config.Supplied("ssl-mode") {
			return "", fmt.Errorf("connect-options is not allowed to contain %s; use only the newer ssl-mode option instead", name)
		}
		v.Set(name, value)
	}
	v.Set("interpolateParams", "true")
	v.Set("foreign_key_checks", "0")
	v.Set("default_storage_engine", "'InnoDB'")
	return v.Encode(), nil
}
func (dir *Dir) Generator() (major, minor, patch int, edition string) {
	generator := dir.Config.Get("generator")
	if !strings.HasPrefix(generator, "skeema:") {
		return 0, 0, 0, ""
	}
	generator = generator[7:]
	parts := strings.Split(generator, "-")
	if len(parts) > 1 {
		edition = parts[1]
	}
	tokens := strings.Split(parts[0], ".")
	major, _ = strconv.Atoi(tokens[0])  
	if len(tokens) > 1 {
		minor, _ = strconv.Atoi(tokens[1])
	}
	if len(tokens) > 2 {
		patch, _ = strconv.Atoi(tokens[2])
	}
	return
}
func (dir *Dir) parseContents() {
	var has bool
	if has, dir.ParseError = dir.HasFile(".skeema"); dir.ParseError != nil {
		return
	} else if has {
		if dir.OptionFile, dir.ParseError = parseOptionFile(dir.Path, dir.repoBase, dir.Config); dir.ParseError != nil {
			return
		}
		dir.Config.AddSource(dir.OptionFile)
	}
	if dir.SQLFiles, dir.ParseError = sqlFiles(dir.Path, dir.repoBase); dir.ParseError != nil {
		return
	}
	logicalSchemasByName := make(map[string]*LogicalSchema)
	for _, sf := range dir.SQLFiles {
		tokenizedFile, err := sf.Tokenize()
		if err != nil {
			log.Warnf(err.Error())
			dir.IgnoredStatements = append(dir.IgnoredStatements, tokenizedFile.Statements...)
			continue
		}
		for _, stmt := range tokenizedFile.Statements {
			if _, ok := logicalSchemasByName[stmt.Schema()]; !ok {
				logicalSchemasByName[stmt.Schema()] = &LogicalSchema{
					Creates: make(map[tengo.ObjectKey]*Statement),
				}
			}
			dir.ParseError = logicalSchemasByName[stmt.Schema()].AddStatement(stmt)
			if dir.ParseError != nil {
				return
			}
			if stmt.Type == StatementTypeUnknown {
				dir.IgnoredStatements = append(dir.IgnoredStatements, stmt)
			}
		}
	}
	if len(logicalSchemasByName) == 0 && dir.HasSchema() {
		dir.LogicalSchemas = []*LogicalSchema{
			{
				Creates:   make(map[tengo.ObjectKey]*Statement),
				CharSet:   dir.Config.Get("default-character-set"),
				Collation: dir.Config.Get("default-collation"),
			},
		}
		return
	}
	dir.LogicalSchemas = make([]*LogicalSchema, 0, len(logicalSchemasByName))
	if ls, ok := logicalSchemasByName[""]; ok && len(ls.Creates) > 0 {
		ls.CharSet = dir.Config.Get("default-character-set")
		ls.Collation = dir.Config.Get("default-collation")
		dir.LogicalSchemas = append(dir.LogicalSchemas, ls)
	}
	for name, ls := range logicalSchemasByName {
		if name != "" && len(ls.Creates) > 0 {
			ls.Name = name
			dir.LogicalSchemas = append(dir.LogicalSchemas, ls)
		}
	}
}
func ParentOptionFiles(dirPath string, baseConfig *mybase.Config) ([]*mybase.File, string, error) {
	dirs := ancestorPaths(dirPath)
	if len(dirs) == 0 {
		return nil, "", fmt.Errorf("Unable to search for option files in %s", dirPath)
	}
	filePaths := make([]string, 0, len(dirs)-1)
	home, _ := os.UserHomeDir()
	repoBase := dirs[0]  
	var atRepoBase bool
	for n, curPath := range dirs {
		if curPath == home {
			break
		}
		fileInfos, err := ioutil.ReadDir(curPath)
		if err != nil {
			break
		}
		for _, fi := range fileInfos {
			if fi.Name() == ".git" {
				repoBase = curPath
				atRepoBase = true
			} else if fi.Name() == ".skeema" && n > 0 {
				filePaths = append(filePaths, curPath)
				repoBase = curPath
			}
		}
		if atRepoBase {
			break
		}
	}
	files := make([]*mybase.File, 0, len(filePaths))
	for n := len(filePaths) - 1; n >= 0; n-- {
		f, err := parseOptionFile(filePaths[n], repoBase, baseConfig)
		if err != nil {
			return nil, repoBase, err
		}
		files = append(files, f)
	}
	return files, repoBase, nil
}
func HostDefaultDirName(hostname string, port int) string {
	sep := ':'
	if runtime.GOOS == "windows" {
		sep = '_'  
	}
	if port != 3306 && port != 0 {
		return fmt.Sprintf("%s%c%d", hostname, sep, port)
	}
	return hostname
}
func ancestorPaths(dirPath string) (result []string) {
	dirPath = filepath.Clean(dirPath)
	if abs, err := filepath.Abs(dirPath); err == nil {
		dirPath = abs
	}
	root := fmt.Sprintf("%s%c", filepath.VolumeName(dirPath), os.PathSeparator)
	for {
		result = append(result, dirPath)
		if dirPath == root {
			return
		}
		dirPath, _ = filepath.Split(dirPath)
		if dirPath != root {
			dirPath = strings.TrimRight(dirPath, string(os.PathSeparator))
		}
	}
}
func parseOptionFile(dirPath, repoBase string, baseConfig *mybase.Config) (*mybase.File, error) {
	f := mybase.NewFile(dirPath, ".skeema")
	fi, err := os.Lstat(f.Path())
	if err != nil {
		return nil, err
	} else if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
		dest, err := os.Readlink(f.Path())
		if err != nil {
			return nil, err
		}
		dest = filepath.Clean(dest)
		if !filepath.IsAbs(dest) {
			if dest, err = filepath.Abs(filepath.Join(dirPath, dest)); err != nil {
				return nil, err
			}
		}
		if !strings.HasPrefix(dest, repoBase) {
			return nil, fmt.Errorf("%s is a symlink pointing outside of its repo", f.Path())
		}
		if fi, err = os.Lstat(dest); err != nil {  
			return nil, err
		}
	}
	if !fi.Mode().IsRegular() {
		return nil, fmt.Errorf("%s is not a regular file, nor a symlink to a regular file", f.Path())
	}
	if err := f.Read(); err != nil {
		return nil, err
	}
	if err := f.Parse(baseConfig); err != nil {
		return nil, err
	}
	_ = f.UseSection(baseConfig.Get("environment"))  
	return f, nil
}