Repository: C2FO/vfs
Pull request #68 (2021-05-05 13:29:59)

Add OAuth token based authentication
Add the ability to authenticate with service accounts via OAuth tokens.


Patch:
diff --git a/backend/azure/client.go b/backend/azure/client.go
index c818b19..69b9e85 100644
--- a/backend/azure/client.go
+++ b/backend/azure/client.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"net/url"
 	"strings"
 	"time"
@@ -37,21 +36,21 @@ type Client interface {
 	// Copy should copy the file specified by srcFile to the file specified by tgtFile
 	Copy(srcFile vfs.File, tgtFile vfs.File) error
 
-	// List should return a listing for the specified location.  Listings should include the full path for the file.
+	// List should return a listing for the specified location. Listings should include the full path for the file.
 	List(l vfs.Location) ([]string, error)
 
 	// Delete should delete the file specified by the parameter file.
 	Delete(file vfs.File) error
 }
 
-// ClientImpl is the main implementation that actually makes the calls to Azure Blob Storage
-type ClientImpl struct {
+// DefaultClient is the main implementation that actually makes the calls to Azure Blob Storage
+type DefaultClient struct {
 	pipeline pipeline.Pipeline
 }
 
-// NewClient initializes a new ClientImpl
-func NewClient(accountName, accountKey string) (*ClientImpl, error) {
-	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+// NewClient initializes a new DefaultClient
+func NewClient(options *Options) (*DefaultClient, error) {
+	credential, err := options.Credential()
 	if err != nil {
 		return nil, err
 	}
@@ -59,13 +58,22 @@ func NewClient(accountName, accountKey string) (*ClientImpl, error) {
 	// This configures the client to use the default retry policy.  The default policy uses exponential backoff with
 	// maxRetries = 4.  If this behavior needs to be changed, add the Retry member to azblob.PipelineOptions.  For
 	// more information on azure retry policies see https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob#RetryOptions
-	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-
-	return &ClientImpl{pipeline: p}, nil
+	//
+	//  Example (this is not the default):
+	//	RetryOptions{
+	//		Policy:        RetryPolicyExponential, // Use exponential backoff as opposed to linear
+	//		MaxTries:      3,                      // Try at most 3 times to perform the operation (set to 1 to disable retries)
+	//		TryTimeout:    time.Second * 3,        // Maximum time allowed for any single try
+	//		RetryDelay:    time.Second * 1,        // Backoff amount for each retry (exponential or linear)
+	//		MaxRetryDelay: time.Second * 3,        // Max delay between retries
+	//	}
+	pl := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	return &DefaultClient{pl}, nil
 }
 
 // Properties fetches the properties for the blob specified by the parameters containerURI and filePath
-func (a *ClientImpl) Properties(containerURI, filePath string) (*BlobProperties, error) {
+func (a *DefaultClient) Properties(containerURI, filePath string) (*BlobProperties, error) {
 	URL, err := url.Parse(containerURI)
 	if err != nil {
 		return nil, err
@@ -91,7 +99,7 @@ func (a *ClientImpl) Properties(containerURI, filePath string) (*BlobProperties,
 }
 
 // Upload uploads a new file to Azure Blob Storage
-func (a *ClientImpl) Upload(file vfs.File, content io.ReadSeeker) error {
+func (a *DefaultClient) Upload(file vfs.File, content io.ReadSeeker) error {
 	URL, err := url.Parse(file.Location().(*Location).ContainerURL())
 	if err != nil {
 		return err
@@ -105,7 +113,7 @@ func (a *ClientImpl) Upload(file vfs.File, content io.ReadSeeker) error {
 }
 
 // SetMetadata sets the given metadata for the blob
-func (a *ClientImpl) SetMetadata(file vfs.File, metadata map[string]string) error {
+func (a *DefaultClient) SetMetadata(file vfs.File, metadata map[string]string) error {
 	URL, err := url.Parse(file.Location().(*Location).ContainerURL())
 	if err != nil {
 		return err
@@ -118,7 +126,7 @@ func (a *ClientImpl) SetMetadata(file vfs.File, metadata map[string]string) erro
 }
 
 // Download returns an io.ReadCloser for the given vfs.File
-func (a *ClientImpl) Download(file vfs.File) (io.ReadCloser, error) {
+func (a *DefaultClient) Download(file vfs.File) (io.ReadCloser, error) {
 	URL, err := url.Parse(file.Location().(*Location).ContainerURL())
 	if err != nil {
 		return nil, err
@@ -136,7 +144,7 @@ func (a *ClientImpl) Download(file vfs.File) (io.ReadCloser, error) {
 // Copy copies srcFile to the destination tgtFile within Azure Blob Storage.  Note that in the case where we get
 // encoded spaces in the file name (i.e. %20) the '%' must be encoded or the copy command will return a not found
 // error.
-func (a *ClientImpl) Copy(srcFile, tgtFile vfs.File) error {
+func (a *DefaultClient) Copy(srcFile, tgtFile vfs.File) error {
 	// Can't use url.PathEscape here since that will escape everything (even the directory separators)
 	srcURL, err := url.Parse(strings.Replace(srcFile.URI(), "%", "%25", -1))
 	if err != nil {
@@ -165,12 +173,12 @@ func (a *ClientImpl) Copy(srcFile, tgtFile vfs.File) error {
 		return nil
 	}
 
-	return fmt.Errorf("copy failed  ERROR[%s]", resp.ErrorCode())
+	return fmt.Errorf("copy failed ERROR[%s]", resp.ErrorCode())
 }
 
 // List will return a listing of the contents of the given location.  Each item in the list will contain the full key
 // as specified by the azure blob (incliding the virtual 'path').
-func (a *ClientImpl) List(l vfs.Location) ([]string, error) {
+func (a *DefaultClient) List(l vfs.Location) ([]string, error) {
 	URL, err := url.Parse(l.(*Location).ContainerURL())
 	if err != nil {
 		return []string{}, err
@@ -183,7 +191,7 @@ func (a *ClientImpl) List(l vfs.Location) ([]string, error) {
 		listBlob, err := containerURL.ListBlobsHierarchySegment(ctx, marker, "/",
 			azblob.ListBlobsSegmentOptions{Prefix: utils.RemoveLeadingSlash(l.Path())})
 		if err != nil {
-			log.Fatal("Unable to list blobs.  error: " + err.Error())
+			return []string{}, err
 		}
 
 		marker = listBlob.NextMarker
@@ -192,12 +200,11 @@ func (a *ClientImpl) List(l vfs.Location) ([]string, error) {
 			list = append(list, listBlob.Segment.BlobItems[i].Name)
 		}
 	}
-
 	return list, nil
 }
 
 // Delete deletes the given file from Azure Blob Storage.
-func (a *ClientImpl) Delete(file vfs.File) error {
+func (a *DefaultClient) Delete(file vfs.File) error {
 	URL, err := url.Parse(file.Location().(*Location).ContainerURL())
 	if err != nil {
 		return err
diff --git a/backend/azure/doc.go b/backend/azure/doc.go
index 9c7e95a..a9517fb 100644
--- a/backend/azure/doc.go
+++ b/backend/azure/doc.go
@@ -34,10 +34,6 @@ would have to be cast as azure.Filesystem to use the following:
       // cast if fs was created using backend.Backend().  Not necessary if created directly from azure.NewFilesystem().
       fs = fs.(azure.Filesystem)
 
-      // to use your own "context"
-      ctx := context.Background()
-      fs = fs.WithContext(ctx)
-
       // to pass in client options
       fs = fs.WithOptions(
           azure.Options{
@@ -47,8 +43,7 @@ would have to be cast as azure.Filesystem to use the following:
       )
 
       // to pass specific client, for instance mock client
-      ctx := context.Background()
-      client, _ := storage.NewClient(ctx, MockAzureClient{...})
+      client, _ := azure.NewClient(MockAzureClient{...})
       fs = fs.WithClient(client)
   }
 
@@ -57,7 +52,12 @@ Authentication
 Authentication, by default, occurs automatically when Client() is called. It looks for credentials in the following places,
 preferring the first location found:
 
-  1. The ENV vars AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY
-  2. More to come...
+  1. When the ENV vars VFS_AZURE_ENV_NAME, VFS_AZURE_STORAGE_ACCOUNT, VFS_AZURE_TENANT_ID, VFS_AZURE_CLIENT_ID, and
+     VFS_AZURE_CLIENT_SECRET, authentication is performed using an OAuth Token Authenticator.  This will allow access
+     to containers from multiple storage accounts.
+  2. The ENV vars VFS_AZURE_STORAGE_ACCOUNT and VFS_AZURE_STORAGE_KEY, a shared key authenticator is used.  This will
+     allow access to any containers owned by the designated storage account.
+  3. If none of the above are present, then an anonymous authenticator is created and only publicly accessible blobs
+     will be available
 */
 package azure
diff --git a/backend/azure/fileSystem.go b/backend/azure/fileSystem.go
index 87f3ee5..3a701b9 100644
--- a/backend/azure/fileSystem.go
+++ b/backend/azure/fileSystem.go
@@ -24,22 +24,20 @@ const errNilFileSystemReceiver = "azure.FileSystem receiver pointer must be non-
 
 // FileSystem implements the vfs.FileSystem interface for Azure Blob Storage
 type FileSystem struct {
-	options Options
+	options *Options
 	client  Client
 }
 
 // NewFileSystem creates a new default FileSystem.  This will set the options options.AccountName and
 // options.AccountKey with the env variables AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_ACCESS_KEY respectively.
 func NewFileSystem() *FileSystem {
-	fs := FileSystem{}
-	fs.options.AccountName, fs.options.AccountKey = os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
-	return &fs
+	return &FileSystem{options: NewOptions()}
 }
 
 // WithOptions allows the caller to override the default options
 func (fs *FileSystem) WithOptions(opts vfs.Options) *FileSystem {
 	azureOpts, _ := opts.(Options)
-	fs.options = azureOpts
+	fs.options = &azureOpts
 	return fs
 }
 
@@ -52,7 +50,7 @@ func (fs *FileSystem) WithClient(client Client) *FileSystem {
 // Client returns a Client to perform operations against Azure Blob Storage
 func (fs *FileSystem) Client() (Client, error) {
 	if fs.client == nil {
-		client, err := NewClient(fs.options.AccountName, fs.options.AccountKey)
+		client, err := NewClient(fs.options)
 		fs.client = client
 		return fs.client, err
 	}
diff --git a/backend/azure/mock_token.go b/backend/azure/mock_token.go
new file mode 100644
index 0000000..27af975
--- /dev/null
+++ b/backend/azure/mock_token.go
@@ -0,0 +1,15 @@
+package azure
+
+import (
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+)
+
+// MockTokenCredentialFactory knows how to create a "do-nothing" credential used for unit testing
+type MockTokenCredentialFactory struct{}
+
+// New creates a new azblob.TokenCredntial struct
+func (f *MockTokenCredentialFactory) New(tenantID, clientID, clientSecret, azureEnvName string) (azblob.TokenCredential, error) {
+	return azblob.NewTokenCredential("aaa", func(credential azblob.TokenCredential) time.Duration { return time.Second * 1 }), nil
+}
diff --git a/backend/azure/options.go b/backend/azure/options.go
index c6ee27f..932fa70 100644
--- a/backend/azure/options.go
+++ b/backend/azure/options.go
@@ -1,15 +1,83 @@
 package azure
 
-import "github.com/c2fo/vfs/v5"
+import (
+	"os"
+
+	"github.com/c2fo/vfs/v5"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+)
 
 // Options contains options necessary for the azure vfs implementation
 type Options struct {
-	// AccountName holds the Azure Blob Storage account name for authentication
+	// AccountName holds the Azure Blob Storage account name for authentication.  This field is required for all
+	// authentication types.
 	AccountName string
 
-	// AccountKey holds the Azure Blob Storage account key for authentication
+	// AccountKey holds the Azure Blob Storage account key for authentication.  This field is used for shared key
+	// authentication.
 	AccountKey string
 
+	// TenantID holds the Azure Service Account tenant id for authentication.  This field is used for OAuth token
+	// based authentication.
+	TenantID string
+
+	// ClientID holds the Azure Service Account client id for authentication.  This field is used for OAuth token
+	// based authentication.
+	ClientID string
+
+	// ClientSecret holds the Azure Service Account client secret for authentication.  This field is used for OAuth token
+	// based authentication.
+	ClientSecret string
+
+	// AzureEnvName holds the name for the Azure environment.  This field is used for OAuth token
+	// based authentication.
+	AzureEnvName string
+
 	// RetryFunc holds the retry function
 	RetryFunc vfs.Retry
+
+	tokenCredentialFactory TokenCredentialFactory
+}
+
+// NewOptions creates a new Options struct by populating values from environment variables.
+//   Env Vars:
+//     *VFS_AZURE_STORAGE_ACCOUNT
+//     *VFS_AZURE_STORAGE_ACCESS_KEY
+//     *VFS_AZURE_TENANT_ID
+//     *VFS_AZURE_CLIENT_ID
+//     *VFS_AZURE_CLIENT_SECRET
+//     *VFS_AZURE_ENV_NAME
+func NewOptions() *Options {
+	return &Options{
+		AccountName:            os.Getenv("VFS_AZURE_STORAGE_ACCOUNT"),
+		AccountKey:             os.Getenv("VFS_AZURE_STORAGE_ACCESS_KEY"),
+		TenantID:               os.Getenv("VFS_AZURE_TENANT_ID"),
+		ClientID:               os.Getenv("VFS_AZURE_CLIENT_ID"),
+		ClientSecret:           os.Getenv("VFS_AZURE_CLIENT_SECRET"),
+		AzureEnvName:           os.Getenv("VFS_AZURE_ENV_NAME"),
+		tokenCredentialFactory: &DefaultTokenCredentialFactory{},
+	}
+}
+
+// Credential returns an azblob.Credential struct based on how options are configured.  Options are checked
+// and evaluated in the following order:
+//    1. If TenantID, ClientID, and ClientSecret are non-empty, return azblob.TokenCredential.  This form of authentication
+//       is used with service accounts and can be used to access containers across multiple storage accounts.
+//    2. If AccountName, and AccountKey are non-empty, return azblob.SharedKeyCredential.  This form or authentication
+//       is used with storage accounts and only provides access to a single storage account.
+//    3. Returns an anonymous credential.  This allows access only to public blobs.
+func (o *Options) Credential() (azblob.Credential, error) {
+	// Check to see if we have service account credentials
+	if o.TenantID != "" && o.ClientID != "" && o.ClientSecret != "" {
+		return o.tokenCredentialFactory.New(o.TenantID, o.ClientID, o.ClientSecret, o.AzureEnvName)
+	}
+
+	// Check to see if we have storage account credentials
+	if o.AccountName != "" && o.AccountKey != "" {
+		return azblob.NewSharedKeyCredential(o.AccountName, o.AccountKey)
+	}
+
+	// 3. Return an anonymous credential
+	return azblob.NewAnonymousCredential(), nil
 }
diff --git a/backend/azure/token.go b/backend/azure/token.go
new file mode 100644
index 0000000..ab2b01f
--- /dev/null
+++ b/backend/azure/token.go
@@ -0,0 +1,55 @@
+package azure
+
+import (
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest/azure"
+)
+
+// TokenCredentialFactory is an interface that provides a single factory method to create azure.TokenCredentials.  This
+// interface is provided to allow for mocking in unit tests.
+type TokenCredentialFactory interface {
+	// New creates a new azblob.TokenCredntial struct
+	New(tenantID, clientID, clientSecret, azureEnvName string) (azblob.TokenCredential, error)
+}
+
+// DefaultTokenCredentialFactory knows how to make azblob.TokenCredential structs for OAuth authentication
+type DefaultTokenCredentialFactory struct{}
+
+// New creates a new azblob.TokenCredntial struct
+func (f *DefaultTokenCredentialFactory) New(tenantID, clientID, clientSecret, azureEnvName string) (azblob.TokenCredential, error) {
+	env, err := azure.EnvironmentFromName(azureEnvName)
+	if err != nil {
+		return nil, err
+	}
+
+	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, tenantID)
+	if err != nil {
+		return nil, err
+	}
+
+	spt, err := adal.NewServicePrincipalToken(*oauthConfig, clientID, clientSecret, env.ResourceIdentifiers.Storage)
+	if err != nil {
+		return nil, err
+	}
+
+	initialToken := spt.Token()
+
+	var tokenRefresher = func(credential azblob.TokenCredential) time.Duration {
+		err := spt.Refresh()
+		if err != nil {
+			// Not sure what else to do here except panic...
+			// The documentation suggests returning 0 so the token will never attempt to refresh again
+			// and then call cancel() on any context in the pipeline that have this credential object
+			panic(err)
+		}
+
+		token := spt.Token()
+		credential.SetToken(token.AccessToken)
+		return token.Expires().Sub(time.Now().Add(2 * time.Minute))
+	}
+
+	return azblob.NewTokenCredential(initialToken.AccessToken, tokenRefresher), nil
+}
diff --git a/go.mod b/go.mod
index 39302c8..ab89a8b 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,8 @@ require (
 	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.13.0
+	github.com/Azure/go-autorest/autorest v0.11.18
+	github.com/Azure/go-autorest/autorest/adal v0.9.13
 	github.com/aws/aws-sdk-go v1.36.12
 	github.com/fatih/color v1.10.0
 	github.com/mitchellh/go-homedir v1.1.0
diff --git a/go.sum b/go.sum
index fedb293..f8abc30 100644
--- a/go.sum
+++ b/go.sum
@@ -40,10 +40,19 @@ github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVt
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
 github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc=
 github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
+github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -69,6 +78,8 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -201,6 +212,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 h1:3wPMTskHO3+O6jqTEXyFcsnuxMQOqYSaHsDxcbUXpqA=
 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=


Test patch:
diff --git a/backend/azure/client_integration_test.go b/backend/azure/client_integration_test.go
index 7998e6f..31a7d70 100644
--- a/backend/azure/client_integration_test.go
+++ b/backend/azure/client_integration_test.go
@@ -16,219 +16,208 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
-type ClientTestSuite struct {
+type ClientIntegrationTestSuite struct {
 	suite.Suite
 	testContainerURL azblob.ContainerURL
 	accountName      string
 	accountKey       string
 }
 
-func (s *ClientTestSuite) SetupSuite() {
-	s.accountName, s.accountKey = os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+func (suite *ClientIntegrationTestSuite) SetupSuite() {
+	suite.accountName, suite.accountKey = os.Getenv("VFS_AZURE_STORAGE_ACCOUNT"), os.Getenv("VFS_AZURE_STORAGE_ACCESS_KEY")
 
-	credential, err := azblob.NewSharedKeyCredential(s.accountName, s.accountKey)
+	credential, err := azblob.NewSharedKeyCredential(suite.accountName, suite.accountKey)
 	if err != nil {
 		panic(err)
 	}
 
 	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-	baseURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", s.accountName))
-	s.NoError(err)
+	baseURL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", suite.accountName))
+	suite.NoError(err)
 	serviceURL := azblob.NewServiceURL(*baseURL, p)
-	s.testContainerURL = serviceURL.NewContainerURL("test-container")
-	_, err = s.testContainerURL.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessNone)
-	s.NoError(err)
+	suite.testContainerURL = serviceURL.NewContainerURL("test-container")
+	_, err = suite.testContainerURL.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessNone)
+	//suite.NoError(err)
 
 	// The create function claims to be synchronous but for some reason it does not exist for a little bit so
 	// we need to wait for it to be there.
-	_, err = s.testContainerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
+	_, err = suite.testContainerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
 	for {
 		time.Sleep(2 * time.Second)
 		if err == nil || err.(azblob.StorageError).ServiceCode() != "BlobNotFound" {
 			break
 		}
-		_, err = s.testContainerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
+		_, err = suite.testContainerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{})
 	}
 }
 
-func (s *ClientTestSuite) TearDownSuite() {
-	_, err := s.testContainerURL.Delete(context.Background(), azblob.ContainerAccessConditions{})
-	s.NoError(err)
+func (suite *ClientIntegrationTestSuite) TearDownSuite() {
+	_, err := suite.testContainerURL.Delete(context.Background(), azblob.ContainerAccessConditions{})
+	suite.NoError(err)
 }
 
-func (s *ClientTestSuite) TestNewClient() {
-	client, err := NewClient("", "&")
-	s.Error(err, "AccountKey must be composed of base 64 decodable characters")
-	s.Nil(client, "The call to NewClient resulted in an error so the returned client should be nil")
-
-	client, err = NewClient(s.accountName, s.accountKey)
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
-	s.NotNil(client, "No error is expected so the client should be non-nil")
-	s.NotNil(client.pipeline, "pipeline should be non-nil")
-}
-
-func (s *ClientTestSuite) TestAllTheThings_FileWithNoPath() {
+func (suite *ClientIntegrationTestSuite) TestAllTheThings_FileWithNoPath() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/test.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	// Create the new file
 	err = client.Upload(f, strings.NewReader("Hello world!"))
-	s.NoError(err, "The file should be successfully uploaded to azure")
+	suite.NoError(err, "The file should be successfully uploaded to azure")
 
 	// make sure it exists
 	_, err = client.Properties(f.Location().URI(), f.Name())
-	s.NoError(err, "If the file exists no error should be returned")
+	suite.NoError(err, "If the file exists no error should be returned")
 
 	// download it
 	reader, err := client.Download(f)
-	s.NoError(err)
+	suite.NoError(err)
 	dlContent, err := ioutil.ReadAll(reader)
 	err = reader.Close()
-	s.NoError(err)
-	s.NoError(err, "there should be no error reading the downloaded file")
-	s.Equal("Hello world!", string(dlContent))
+	suite.NoError(err)
+	suite.NoError(err, "there should be no error reading the downloaded file")
+	suite.Equal("Hello world!", string(dlContent))
 
 	// copy it
 	copyOf, err := fs.NewFile("test-container", "/copy_of_test.txt")
 	err = client.Copy(f, copyOf)
-	s.NoError(err, "Copy should succeed so there should be no error")
+	suite.NoError(err, "Copy should succeed so there should be no error")
 	_, err = client.Properties(copyOf.Location().URI(), copyOf.Name())
-	s.NoError(err, "The copy should succeed so we should not get an error on the properties call")
+	suite.NoError(err, "The copy should succeed so we should not get an error on the properties call")
 
 	// list the location
 	list, err := client.List(f.Location())
-	s.NoError(err)
-	s.Len(list, 2)
-	s.Equal("copy_of_test.txt", list[0])
-	s.Equal("test.txt", list[1])
+	suite.NoError(err)
+	suite.Len(list, 2)
+	suite.Equal("copy_of_test.txt", list[0])
+	suite.Equal("test.txt", list[1])
 
 	// delete it
 	err = client.Delete(f)
-	s.NoError(err, "if the file was deleted no error should be returned")
+	suite.NoError(err, "if the file was deleted no error should be returned")
 
 	// make sure it got deleted
 	_, err = client.Properties(f.Location().URI(), f.Name())
-	s.Error(err, "File should have been deleted so we should get an error")
+	suite.Error(err, "File should have been deleted so we should get an error")
 }
 
-func (s *ClientTestSuite) TestAllTheThings_FileWithPath() {
+func (suite *ClientIntegrationTestSuite) TestAllTheThings_FileWithPath() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/foo/bar/test.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	// create a new file
 	err = client.Upload(f, strings.NewReader("Hello world!"))
-	s.NoError(err, "The file should be successfully uploaded to azure")
+	suite.NoError(err, "The file should be successfully uploaded to azure")
 
-	// check to see if it exists
+	//check to see if it exists
 	_, err = client.Properties(f.Location().(*Location).ContainerURL(), f.Path())
-	s.NoError(err, "If the file exists no error should be returned")
+	suite.NoError(err, "If the file exists no error should be returned")
 
 	// download it
 	reader, err := client.Download(f)
-	s.NoError(err)
+	suite.NoError(err)
 	dlContent, err := ioutil.ReadAll(reader)
 	err = reader.Close()
-	s.NoError(err)
-	s.NoError(err, "there should be no error reading the downloaded file")
-	s.Equal("Hello world!", string(dlContent))
+	suite.NoError(err)
+	suite.NoError(err, "there should be no error reading the downloaded file")
+	suite.Equal("Hello world!", string(dlContent))
 
 	// list the location
 	list, err := client.List(f.Location())
-	s.NoError(err)
-	s.Len(list, 1)
-	s.Equal("foo/bar/test.txt", list[0])
+	suite.NoError(err)
+	suite.Len(list, 1)
+	suite.Equal("foo/bar/test.txt", list[0])
 }
 
-func (s *ClientTestSuite) TestProperties() {
+func (suite *ClientIntegrationTestSuite) TestProperties() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/foo/bar/test.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	err = client.Upload(f, strings.NewReader("Hello world!"))
-	s.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error")
+	suite.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error")
 	props, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path())
-	s.NoError(err, "Tne file exists so we shouldn't get an error")
-	s.NotNil(props, "We should get a non-nil BlobProperties pointer back")
-	s.Greater(props.Size, uint64(0), "The size should be greater than zero")
-	s.NotNil(props.LastModified, "Should have a non-nil LastModified")
+	suite.NoError(err, "Tne file exists so we shouldn't get an error")
+	suite.NotNil(props, "We should get a non-nil BlobProperties pointer back")
+	suite.Greater(props.Size, uint64(0), "The size should be greater than zero")
+	suite.NotNil(props.LastModified, "Should have a non-nil LastModified")
 }
 
-func (s *ClientTestSuite) TestProperties_Location() {
+func (suite *ClientIntegrationTestSuite) TestProperties_Location() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/foo/bar/test.txt")
 	l, _ := fs.NewLocation("test-container", "/")
 	client, _ := fs.Client()
 
 	err = client.Upload(f, strings.NewReader("Hello world!"))
-	s.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error")
+	suite.NoError(err, "The file should be successfully uploaded to azure so we shouldn't get an error")
 
 	props, err := client.Properties(l.URI(), "")
-	s.NoError(err)
-	s.Nil(props, "no props returned when calling properties on a location")
+	suite.NoError(err)
+	suite.Nil(props, "no props returned when calling properties on a location")
 }
 
-func (s *ClientTestSuite) TestProperties_NonExistentFile() {
+func (suite *ClientIntegrationTestSuite) TestProperties_NonExistentFile() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/nosuchfile.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	_, err = client.Properties(f.Location().URI(), f.Path())
-	s.Error(err, "Tne file does not exist so we expect an error")
-	s.Equal(404, err.(azblob.ResponseError).Response().StatusCode)
+	suite.Error(err, "Tne file does not exist so we expect an error")
+	suite.Equal(404, err.(azblob.ResponseError).Response().StatusCode)
 }
 
-func (s *ClientTestSuite) TestDelete_NonExistentFile() {
+func (suite *ClientIntegrationTestSuite) TestDelete_NonExistentFile() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/nosuchfile.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	err = client.Delete(f)
-	s.Error(err, "Tne file does not exist so we expect an error")
+	suite.Error(err, "Tne file does not exist so we expect an error")
 }
 
-func (s *ClientTestSuite) TestTouch_NonexistantContainer() {
+func (suite *ClientIntegrationTestSuite) TestTouch_NonexistantContainer() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("nosuchcontainer", "/file.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
+	suite.NoError(err, "Env variables (AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_ACCESS_KEY) should contain valid azure account credentials")
 
 	err = client.Upload(f, strings.NewReader(""))
-	s.Error(err, "The container doesn't exist so we should get an error")
+	suite.Error(err, "The container doesn't exist so we should get an error")
 }
 
-func (s *ClientTestSuite) TestTouch_FileAlreadyExists() {
+func (suite *ClientIntegrationTestSuite) TestTouch_FileAlreadyExists() {
 	fs := NewFileSystem()
 	f, err := fs.NewFile("test-container", "/touch-test.txt")
-	s.NoError(err)
+	suite.NoError(err)
 	client, err := fs.Client()
-	s.NoError(err)
+	suite.NoError(err)
 
 	err = client.Upload(f, strings.NewReader("One fish, two fish, red fish, blue fish."))
-	s.NoError(err)
+	suite.NoError(err)
 	originalProps, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path())
-	s.NoError(err, "Should get properties back from azure with no error")
+	suite.NoError(err, "Should get properties back from azure with no error")
 
 	err = f.Touch()
-	s.NoError(err, "Should not receive an error when touching an existing file")
+	suite.NoError(err, "Should not receive an error when touching an existing file")
 	newProps, err := client.Properties(f.Location().(*Location).ContainerURL(), f.Path())
-	s.NoError(err)
-	s.NotNil(newProps, "New props should be non-nil")
-	s.True(newProps.LastModified.After(*originalProps.LastModified), "newProps.LastModified should be after originalProps.LastModified")
+	suite.NoError(err)
+	suite.NotNil(newProps, "New props should be non-nil")
+	suite.True(newProps.LastModified.After(*originalProps.LastModified), "newProps.LastModified should be after originalProps.LastModified")
 }
 
 func TestAzureClient(t *testing.T) {
-	suite.Run(t, new(ClientTestSuite))
+	suite.Run(t, new(ClientIntegrationTestSuite))
 }
diff --git a/backend/azure/fileSystem_test.go b/backend/azure/fileSystem_test.go
index 706c8f9..e9decbc 100644
--- a/backend/azure/fileSystem_test.go
+++ b/backend/azure/fileSystem_test.go
@@ -133,13 +133,13 @@ func (s *FileSystemTestSuite) TestRetry() {
 		return errors.New("i always error")
 	}
 
-	fs := FileSystem{}
+	fs := NewFileSystem()
 	retryFn := fs.Retry()
 	s.NotNil(retryFn)
 	err := retryFn(doNothing)
 	s.NoError(err, "The default retry function just calls the passed func")
 
-	fs = *NewFileSystem().WithOptions(Options{RetryFunc: errorRetry})
+	fs = NewFileSystem().WithOptions(Options{RetryFunc: errorRetry})
 	retryFn = fs.Retry()
 	err = retryFn(doNothing)
 	s.Error(err, "This implementation should use the retry function from the options which always errors")
diff --git a/backend/azure/options_test.go b/backend/azure/options_test.go
new file mode 100644
index 0000000..a3e7bcc
--- /dev/null
+++ b/backend/azure/options_test.go
@@ -0,0 +1,81 @@
+package azure
+
+import (
+	"encoding/base64"
+	"os"
+	"testing"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/stretchr/testify/suite"
+)
+
+type OptionsTestSuite struct {
+	suite.Suite
+}
+
+func (s *OptionsTestSuite) TearDownTest() {
+	clearEnv()
+}
+
+func (s *OptionsTestSuite) TestNewOptions() {
+	o := NewOptions()
+	s.NotNil(o, "when NewOptions returns an error we expect to get a nill options struct")
+}
+
+func (s *OptionsTestSuite) TestCredentials_ServiceAccount() {
+	options := Options{
+		AccountName:            "foo",
+		TenantID:               "foo",
+		ClientID:               "foo",
+		ClientSecret:           "foo",
+		tokenCredentialFactory: &MockTokenCredentialFactory{},
+	}
+
+	credential, err := options.Credential()
+	s.NoError(err, "service account vars are present so no error")
+	s.NotNil(credential, "expect a non-nil credential when service account vars are set")
+	_, ok := credential.(azblob.TokenCredential)
+	s.True(ok, "credentials type should be TokenCredential")
+}
+
+func (s *OptionsTestSuite) TestCredentials_StorageAccount() {
+	options := Options{
+		AccountName:            "foo",
+		AccountKey:             base64.StdEncoding.EncodeToString([]byte("bar")),
+		tokenCredentialFactory: &MockTokenCredentialFactory{},
+	}
+
+	credential, err := options.Credential()
+	s.NoError(err, "service account vars are present so no error")
+	s.NotNil(credential, "expect a non-nil credential when service account vars are set")
+	_, ok := credential.(*azblob.SharedKeyCredential)
+	s.True(ok, "credential type should be SharedKeyCredential")
+}
+
+func (s *OptionsTestSuite) TestCredentials_Anon() {
+	options := Options{
+		AccountName:            "foo",
+		tokenCredentialFactory: &MockTokenCredentialFactory{},
+	}
+
+	credential, err := options.Credential()
+	s.NoError(err, "anon vars are present so no error")
+	s.NotNil(credential, "expect a non-nil credential when service account vars are set")
+	s.NotNil(credential, "when no env vars are set we should get a non-nil credential")
+	_, ok := credential.(azblob.TokenCredential)
+	s.False(ok)
+	_, ok = credential.(*azblob.SharedKeyCredential)
+	s.False(ok)
+}
+
+func TestOptions(t *testing.T) {
+	suite.Run(t, new(OptionsTestSuite))
+}
+
+func clearEnv() {
+	os.Unsetenv("VFS_AZURE_TENANT_ID")
+	os.Unsetenv("VFS_AZURE_CLIENT_ID")
+	os.Unsetenv("VFS_AZURE_CLIENT_SECRET")
+	os.Unsetenv("VFS_AZURE_STORAGE_ACCOUNT")
+	os.Unsetenv("VFS_AZURE_STORAGE_ACCESS_KEY")
+}