Repository: ariga/atlas
Pull request #1003 (2022-08-03 08:09:35)

sql/internal/sqlx: exclude resources based on patterns


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 24d0c47e..07367ce6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
     steps:
     - uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: 1.18
     - uses: actions/checkout@v2
     - name: Run linters
       uses: golangci/golangci-lint-action@v3
@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.17', '1.18' ]
+        go: [ '1.18', '1.19' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -74,7 +74,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.18' ]
+        go: [ '1.19' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -96,7 +96,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -128,7 +128,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -159,7 +159,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -190,7 +190,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -221,7 +221,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -252,7 +252,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -283,7 +283,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -313,7 +313,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -343,7 +343,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -373,7 +373,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -403,7 +403,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -433,7 +433,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -457,7 +457,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -481,7 +481,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -499,7 +499,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -523,7 +523,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
diff --git a/go.mod b/go.mod
index 10556d4f..9a69e23d 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module ariga.io/atlas
 
-go 1.17
+go 1.18
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
diff --git a/internal/ci/ci.tmpl b/internal/ci/ci.tmpl
index 32bdb3b2..5055ab72 100644
--- a/internal/ci/ci.tmpl
+++ b/internal/ci/ci.tmpl
@@ -14,7 +14,7 @@ jobs:
     steps:
     - uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: 1.18
     - uses: actions/checkout@v2
     - name: Run linters
       uses: golangci/golangci-lint-action@v3
@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.17', '1.18' ]
+        go: [ '1.18', '1.19' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -74,7 +74,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.18' ]
+        go: [ '1.19' ]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -96,7 +96,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
@@ -126,7 +126,7 @@ jobs:
       - uses: actions/checkout@v2.3.4
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/cache@v2.1.6
         with:
           path: ~/go/pkg/mod
diff --git a/sql/internal/sqlx/exclude.go b/sql/internal/sqlx/exclude.go
new file mode 100644
index 00000000..910e9d67
--- /dev/null
+++ b/sql/internal/sqlx/exclude.go
@@ -0,0 +1,149 @@
+// Copyright 2021-present The Atlas Authors. All rights reserved.
+// This source code is licensed under the Apache 2.0 license found
+// in the LICENSE file in the root directory of this source tree.
+
+package sqlx
+
+import (
+	"encoding/csv"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"ariga.io/atlas/sql/schema"
+)
+
+// ExcludeRealm filters resources in the realm based on the given patterns.
+func ExcludeRealm(r *schema.Realm, patterns []string) error {
+	var schemas []*schema.Schema
+	globs, err := split(patterns)
+	if err != nil {
+		return err
+	}
+Filter:
+	for _, s := range r.Schemas {
+		for i, g := range globs {
+			if len(g) > 3 {
+				return fmt.Errorf("too many parts in pattern: %q", patterns[i])
+			}
+			match, err := filepath.Match(g[0], s.Name)
+			if err != nil {
+				return err
+			}
+			if match {
+				// In case there is a match, and it is
+				// a single glob we exclude this schema.
+				if len(g) == 1 {
+					continue Filter
+				}
+				if err := excludeS(s, g[1:]); err != nil {
+					return err
+				}
+			}
+		}
+		schemas = append(schemas, s)
+	}
+	r.Schemas = schemas
+	return nil
+}
+
+// ExcludeSchema filters resources in the schema based on the given patterns.
+func ExcludeSchema(s *schema.Schema, patterns []string) error {
+	if s.Realm == nil {
+		return fmt.Errorf("missing realm for schema %q", s.Name)
+	}
+	for i, p := range patterns {
+		patterns[i] = fmt.Sprintf("%s.%s", s.Name, p)
+	}
+	return ExcludeRealm(s.Realm, patterns)
+}
+
+// split parses the list of patterns into chain of resource-globs.
+// For example, 's*.t.*' is split to ['s*', 't', *].
+func split(patterns []string) ([][]string, error) {
+	globs := make([][]string, len(patterns))
+	for i, p := range patterns {
+		r := csv.NewReader(strings.NewReader(p))
+		r.Comma = '.'
+		switch parts, err := r.ReadAll(); {
+		case err != nil:
+			return nil, err
+		case len(parts) != 1:
+			return nil, fmt.Errorf("unexpected pattern: %q", p)
+		case len(parts[0]) == 0:
+			return nil, fmt.Errorf("empty pattern: %q", p)
+		default:
+			globs[i] = parts[0]
+		}
+	}
+	return globs, nil
+}
+
+func excludeS(s *schema.Schema, glob []string) error {
+	var tables []*schema.Table
+	for _, t := range s.Tables {
+		match, err := filepath.Match(glob[0], t.Name)
+		if err != nil {
+			return err
+		}
+		if match {
+			// In case there is a match, and it is
+			// a single glob we exclude this table.
+			if len(glob) == 1 {
+				continue
+			}
+			if err := excludeT(t, glob[1]); err != nil {
+				return err
+			}
+		}
+		// No match or glob has more than one pattern.
+		tables = append(tables, t)
+	}
+	s.Tables = tables
+	return nil
+}
+
+func excludeT(t *schema.Table, pattern string) (err error) {
+	ex := make(map[*schema.Index]struct{})
+	ef := make(map[*schema.ForeignKey]struct{})
+	t.Columns, err = filter(t.Columns, func(c *schema.Column) (bool, error) {
+		match, err := filepath.Match(pattern, c.Name)
+		if !match || err != nil {
+			return false, err
+		}
+		for _, idx := range c.Indexes {
+			ex[idx] = struct{}{}
+		}
+		for _, fk := range c.ForeignKeys {
+			ef[fk] = struct{}{}
+		}
+		return true, nil
+	})
+	t.Indexes, err = filter(t.Indexes, func(idx *schema.Index) (bool, error) {
+		if _, ok := ex[idx]; ok {
+			return true, nil
+		}
+		return filepath.Match(pattern, idx.Name)
+	})
+	t.ForeignKeys, err = filter(t.ForeignKeys, func(fk *schema.ForeignKey) (bool, error) {
+		if _, ok := ef[fk]; ok {
+			return true, nil
+		}
+		return filepath.Match(pattern, fk.Symbol)
+	})
+	return
+}
+
+func filter[T any](s []T, f func(T) (bool, error)) ([]T, error) {
+	r := make([]T, 0, len(s))
+	for i := range s {
+		match, err := f(s[i])
+		if err != nil {
+			return nil, err
+		}
+		if !match {
+			r = append(r, s[i])
+		}
+	}
+	return r, nil
+}
diff --git a/sql/schema/inspect.go b/sql/schema/inspect.go
index cdbc1522..ea2de5f6 100644
--- a/sql/schema/inspect.go
+++ b/sql/schema/inspect.go
@@ -57,6 +57,18 @@ type (
 
 		// Tables to inspect. Empty means all tables in the schema.
 		Tables []string
+
+		// Exclude defines a list of glob patterns used to filter resources from inspection.
+		// The syntax used by the different drivers is implemented as follows:
+		//
+		//	t   // exclude table 't'.
+		//	*   // exclude all tables.
+		//	t.c // exclude column, index and foreign-key named 'c' in table 't'.
+		//	t.* // the last item defines the filtering; all resources under 't' are excluded.
+		//	*.c // the last item defines the filtering; all resourced named 'c' are excluded in all tables.
+		//	*.* // the last item defines the filtering; all resourced under all tables are excluded.
+		//
+		Exclude []string
 	}
 
 	// InspectRealmOption describes options for RealmInspector.
@@ -67,6 +79,20 @@ type (
 
 		// Schemas to inspect. Empty means all schemas in the realm.
 		Schemas []string
+
+		// Exclude defines a list of glob patterns used to filter resources from inspection.
+		// The syntax used by the different drivers is implemented as follows:
+		//
+		//	s     // exclude schema 't'.
+		//	*     // exclude all schemas.
+		//	s.t   // exclude table 't' under schema 's'.
+		//	s.*   // the last item defines the filtering; all tables under 's' are excluded.
+		//	*.t   // the last item defines the filtering; all tables named 't' are excluded in all schemas.
+		//	*.*   // the last item defines the filtering; all tables under all schemas are excluded.
+		//	*.*.c // the last item defines the filtering; all resourced named 'c' are excluded in all tables.
+		//	*.*.* // the last item defines the filtering; all resources are excluded in all tables.
+		//
+		Exclude []string
 	}
 
 	// Inspector is the interface implemented by the different database


Test patch:
diff --git a/sql/internal/sqlx/exclude_test.go b/sql/internal/sqlx/exclude_test.go
new file mode 100644
index 00000000..98de70fe
--- /dev/null
+++ b/sql/internal/sqlx/exclude_test.go
@@ -0,0 +1,181 @@
+// Copyright 2021-present The Atlas Authors. All rights reserved.
+// This source code is licensed under the Apache 2.0 license found
+// in the LICENSE file in the root directory of this source tree.
+
+package sqlx
+
+import (
+	"testing"
+
+	"ariga.io/atlas/sql/schema"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExcludeRealm_Schemas(t *testing.T) {
+	r := schema.NewRealm(schema.New("s1"), schema.New("s2"), schema.New("s3"))
+	require.NoError(t, ExcludeRealm(r, []string{"s4"}))
+	require.Len(t, r.Schemas, 3)
+
+	require.NoError(t, ExcludeRealm(r, []string{"s1", "s2.t2", "s3.t3.c3"}))
+	require.Len(t, r.Schemas, 2)
+	require.Equal(t, "s2", r.Schemas[0].Name)
+	require.Equal(t, "s3", r.Schemas[1].Name)
+
+	require.NoError(t, ExcludeRealm(r, []string{"*"}))
+	require.Empty(t, r.Schemas)
+
+	r = schema.NewRealm(schema.New("s1"), schema.New("s2"), schema.New("s3"))
+	require.NoError(t, ExcludeRealm(r, []string{"s*.*", "s*.*.*"}))
+	require.Len(t, r.Schemas, 3)
+	require.NoError(t, ExcludeRealm(r, []string{"s*"}))
+	require.Empty(t, r.Schemas)
+}
+
+func TestExcludeRealm_Tables(t *testing.T) {
+	r := schema.NewRealm(
+		schema.New("s0"),
+		schema.New("s1").AddTables(
+			schema.NewTable("t1"),
+		),
+		schema.New("s2").AddTables(
+			schema.NewTable("t1"),
+			schema.NewTable("t2"),
+		),
+		schema.New("s3").AddTables(
+			schema.NewTable("t1"),
+			schema.NewTable("t2"),
+			schema.NewTable("t3"),
+		),
+	)
+	require.NoError(t, ExcludeRealm(r, []string{"s4"}))
+	require.Len(t, r.Schemas, 4)
+	require.NoError(t, ExcludeRealm(r, []string{"s0"}))
+	require.Len(t, r.Schemas, 3)
+	require.Equal(t, "s1", r.Schemas[0].Name)
+	require.Equal(t, "s2", r.Schemas[1].Name)
+	require.Equal(t, "s3", r.Schemas[2].Name)
+
+	require.NoError(t, ExcludeRealm(r, []string{"*.t1.*"}))
+	require.Len(t, r.Schemas, 3)
+	require.Len(t, r.Schemas[0].Tables, 1)
+	require.Len(t, r.Schemas[1].Tables, 2)
+	require.Len(t, r.Schemas[2].Tables, 3)
+
+	require.NoError(t, ExcludeRealm(r, []string{"*.t1"}))
+	require.Len(t, r.Schemas, 3)
+	require.Empty(t, r.Schemas[0].Tables)
+	require.Len(t, r.Schemas[1].Tables, 1)
+	require.Len(t, r.Schemas[2].Tables, 2)
+
+	require.NoError(t, ExcludeRealm(r, []string{"s[12].t2"}))
+	require.Len(t, r.Schemas, 3)
+	require.Empty(t, r.Schemas[0].Tables)
+	require.Empty(t, r.Schemas[1].Tables)
+	require.Len(t, r.Schemas[2].Tables, 2)
+
+	require.NoError(t, ExcludeRealm(r, []string{"*.t[23].*"}))
+	require.Len(t, r.Schemas, 3)
+	require.Len(t, r.Schemas[2].Tables, 2)
+	require.NoError(t, ExcludeRealm(r, []string{"*.t[23]"}))
+	require.Len(t, r.Schemas, 3)
+	require.Empty(t, r.Schemas[2].Tables)
+}
+
+func TestExcludeRealm_Columns(t *testing.T) {
+	r := schema.NewRealm(
+		schema.New("s1").AddTables(
+			func() *schema.Table {
+				t := schema.NewTable("t1").AddColumns(schema.NewColumn("c1"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]))
+				return t
+			}(),
+		),
+		schema.New("s2").AddTables(
+			func() *schema.Table {
+				t := schema.NewTable("t1").AddColumns(schema.NewColumn("c1"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]))
+				return t
+			}(),
+			func() *schema.Table {
+				t := schema.NewTable("t2").AddColumns(schema.NewColumn("c1"), schema.NewColumn("c2"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]), schema.NewIndex("i2").AddColumns(t.Columns[1]))
+				return t
+			}(),
+		),
+		schema.New("s3").AddTables(
+			func() *schema.Table {
+				t := schema.NewTable("t1").AddColumns(schema.NewColumn("c1"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]))
+				return t
+			}(),
+			func() *schema.Table {
+				t := schema.NewTable("t2").AddColumns(schema.NewColumn("c1"), schema.NewColumn("c2"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]), schema.NewIndex("i2").AddColumns(t.Columns[1]))
+				return t
+			}(),
+			func() *schema.Table {
+				t := schema.NewTable("t3").AddColumns(schema.NewColumn("c1"), schema.NewColumn("c2"), schema.NewColumn("c3"))
+				t.AddIndexes(schema.NewIndex("i1").AddColumns(t.Columns[0]), schema.NewIndex("i2").AddColumns(t.Columns[1]))
+				return t
+			}(),
+		),
+	)
+	require.NoError(t, ExcludeRealm(r, []string{"s[23].t[23].c1"}))
+	require.Len(t, r.Schemas, 3)
+	require.Len(t, r.Schemas[0].Tables, 1)
+	require.Len(t, r.Schemas[0].Tables[0].Columns, 1)
+	require.Len(t, r.Schemas[0].Tables[0].Indexes, 1)
+
+	require.Len(t, r.Schemas[1].Tables, 2)
+	require.Len(t, r.Schemas[1].Tables[0].Columns, 1)
+	require.Len(t, r.Schemas[1].Tables[0].Indexes, 1)
+	require.Len(t, r.Schemas[1].Tables[1].Columns, 1)
+	require.Len(t, r.Schemas[1].Tables[1].Indexes, 1)
+
+	require.Len(t, r.Schemas[2].Tables, 3)
+	require.Len(t, r.Schemas[2].Tables[0].Columns, 1)
+	require.Len(t, r.Schemas[2].Tables[0].Indexes, 1)
+	require.Len(t, r.Schemas[2].Tables[1].Columns, 1)
+	require.Len(t, r.Schemas[2].Tables[1].Indexes, 1)
+	require.Len(t, r.Schemas[2].Tables[2].Columns, 2)
+	require.Len(t, r.Schemas[2].Tables[2].Indexes, 1)
+
+	require.NoError(t, ExcludeRealm(r, []string{"s[23].t*.*"}))
+	require.Len(t, r.Schemas, 3)
+	require.Len(t, r.Schemas[0].Tables, 1)
+	require.Len(t, r.Schemas[0].Tables[0].Columns, 1)
+	require.Len(t, r.Schemas[0].Tables[0].Indexes, 1)
+
+	require.Len(t, r.Schemas[1].Tables, 2)
+	require.Empty(t, r.Schemas[1].Tables[0].Columns)
+	require.Empty(t, r.Schemas[1].Tables[0].Indexes)
+	require.Empty(t, r.Schemas[1].Tables[1].Columns)
+	require.Empty(t, r.Schemas[1].Tables[1].Indexes)
+
+	require.Len(t, r.Schemas[2].Tables, 3)
+	require.Empty(t, r.Schemas[2].Tables[0].Columns)
+	require.Empty(t, r.Schemas[2].Tables[0].Indexes)
+	require.Empty(t, r.Schemas[2].Tables[1].Columns)
+	require.Empty(t, r.Schemas[2].Tables[1].Indexes)
+	require.Empty(t, r.Schemas[2].Tables[2].Columns)
+	require.Empty(t, r.Schemas[2].Tables[2].Indexes)
+
+	require.NoError(t, ExcludeRealm(r, []string{"*.*.*"}))
+	require.Len(t, r.Schemas, 3)
+	require.Len(t, r.Schemas[0].Tables, 1)
+	require.Empty(t, r.Schemas[0].Tables[0].Columns)
+	require.Empty(t, r.Schemas[0].Tables[0].Indexes)
+}
+
+func TestExcludeSchema(t *testing.T) {
+	r := schema.NewRealm(
+		schema.New("s1").AddTables(
+			schema.NewTable("t1"),
+			schema.NewTable("t2"),
+		),
+	)
+	require.NoError(t, ExcludeSchema(r.Schemas[0], []string{"t2"}))
+	require.Len(t, r.Schemas, 1)
+	require.Len(t, r.Schemas[0].Tables, 1)
+}