Repository: vmware/govmomi
Pull request #1770 (2020-01-03 23:31:42)

Add Content Library synchronization support
- vAPI: add sync client APIs

- vAPI: add helpers to sync subscribed library OVF items to local library VMTX items

- vcsim: support sync APIs

- govc: add library.sync command


Patch:
diff --git a/govc/library/sync.go b/govc/library/sync.go
new file mode 100644
index 00000000..6560f793
--- /dev/null
+++ b/govc/library/sync.go
@@ -0,0 +1,175 @@
+/*
+Copyright (c) 2019 VMware, Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package library
+
+import (
+	"context"
+	"flag"
+	"fmt"
+
+	"github.com/vmware/govmomi/govc/cli"
+	"github.com/vmware/govmomi/govc/flags"
+	"github.com/vmware/govmomi/vapi/library"
+	"github.com/vmware/govmomi/vapi/library/finder"
+	"github.com/vmware/govmomi/vapi/rest"
+	"github.com/vmware/govmomi/vapi/vcenter"
+)
+
+type sync struct {
+	*flags.FolderFlag
+	*flags.ResourcePoolFlag
+
+	vmtx string
+}
+
+func init() {
+	cli.Register("library.sync", &sync{})
+}
+
+func (cmd *sync) Register(ctx context.Context, f *flag.FlagSet) {
+	cmd.FolderFlag, ctx = flags.NewFolderFlag(ctx)
+	cmd.FolderFlag.Register(ctx, f)
+
+	cmd.ResourcePoolFlag, ctx = flags.NewResourcePoolFlag(ctx)
+	cmd.ResourcePoolFlag.Register(ctx, f)
+
+	f.StringVar(&cmd.vmtx, "vmtx", "", "Sync subscribed library to local library as VM Templates")
+}
+
+func (cmd *sync) Process(ctx context.Context) error {
+	if err := cmd.FolderFlag.Process(ctx); err != nil {
+		return err
+	}
+	return cmd.ResourcePoolFlag.Process(ctx)
+}
+
+func (cmd *sync) Description() string {
+	return `Sync library NAME or ITEM.
+
+Examples:
+  govc library.sync subscribed-library
+  govc library.sync subscribed-library/item
+  govc library.sync -vmtx local-library subscribed-library # convert subscribed OVFs to local VMTX`
+}
+
+func (cmd *sync) Usage() string {
+	return "NAME|ITEM"
+}
+
+func (cmd *sync) syncVMTX(ctx context.Context, m *library.Manager, src library.Library, dst library.Library, items ...library.Item) error {
+	if cmd.vmtx == "" {
+		return nil
+	}
+
+	pool, err := cmd.ResourcePool()
+	if err != nil {
+		return err
+	}
+
+	folder, err := cmd.Folder()
+	if err != nil {
+		return err
+	}
+
+	l := vcenter.TemplateLibrary{
+		Source:      src,
+		Destination: dst,
+		Placement: vcenter.Target{
+			FolderID:       folder.Reference().Value,
+			ResourcePoolID: pool.Reference().Value,
+		},
+		Include: func(item library.Item, current *library.Item) bool {
+			fmt.Printf("Syncing /%s/%s to /%s/%s...", src.Name, item.Name, dst.Name, item.Name)
+			if current == nil {
+				fmt.Println()
+				return true
+			}
+			fmt.Println("already exists.")
+			return false
+		},
+	}
+
+	return vcenter.NewManager(m.Client).SyncTemplateLibrary(ctx, l, items...)
+}
+
+func (cmd *sync) shouldSync(l library.Library) bool {
+	if cmd.vmtx == "" {
+		return true
+
+	}
+	// Allow library.sync -vmtx of LOCAL or SUBSCRIBED library
+	return l.Type == "SUBSCRIBED"
+}
+
+func (cmd *sync) Run(ctx context.Context, f *flag.FlagSet) error {
+	if f.NArg() != 1 {
+		return flag.ErrHelp
+	}
+	path := f.Arg(0)
+
+	return cmd.FolderFlag.WithRestClient(ctx, func(c *rest.Client) error {
+		m := library.NewManager(c)
+		finder := finder.NewFinder(m)
+
+		var local library.Library
+		if cmd.vmtx != "" {
+			res, err := finder.Find(ctx, cmd.vmtx)
+			if err != nil {
+				return err
+			}
+			if len(res) != 1 {
+				return ErrMultiMatch{Type: "library", Key: "name", Val: cmd.vmtx, Count: len(res)}
+			}
+			l, ok := res[0].GetResult().(library.Library)
+			if !ok {
+				return fmt.Errorf("%q is a %T", res[0].GetPath(), l)
+			}
+			local = l
+		}
+
+		res, err := finder.Find(ctx, path)
+		if err != nil {
+			return err
+		}
+
+		if len(res) != 1 {
+			return ErrMultiMatch{Type: "library", Key: "name", Val: path, Count: len(res)}
+		}
+
+		fmt.Printf("Syncing %s...\n", res[0].GetPath())
+
+		switch t := res[0].GetResult().(type) {
+		case library.Library:
+			if cmd.shouldSync(t) {
+				if err = m.SyncLibrary(ctx, &t); err != nil {
+					return err
+				}
+			}
+			return cmd.syncVMTX(ctx, m, t, local)
+		case library.Item:
+			lib := res[0].GetParent().GetResult().(library.Library)
+			if cmd.shouldSync(lib) {
+				if err = m.SyncLibraryItem(ctx, &t, false); err != nil {
+					return err
+				}
+			}
+			return cmd.syncVMTX(ctx, m, lib, local, t)
+		default:
+			return fmt.Errorf("%q is a %T", res[0].GetPath(), t)
+		}
+	})
+}
diff --git a/vapi/internal/internal.go b/vapi/internal/internal.go
index cf8bb5d3..b6ba063e 100644
--- a/vapi/internal/internal.go
+++ b/vapi/internal/internal.go
@@ -37,8 +37,10 @@ const (
 	LibraryItemDownloadSessionFile = "/com/vmware/content/library/item/downloadsession/file"
 	LocalLibraryPath               = "/com/vmware/content/local-library"
 	SubscribedLibraryPath          = "/com/vmware/content/subscribed-library"
+	SubscribedLibraryItem          = "/com/vmware/content/library/subscribed-item"
 	VCenterOVFLibraryItem          = "/com/vmware/vcenter/ovf/library-item"
 	VCenterVMTXLibraryItem         = "/vcenter/vm-template/library-items"
+	VCenterVM                      = "/vcenter/vm"
 	SessionCookieName              = "vmware-api-session-id"
 )
 
diff --git a/vapi/library/library.go b/vapi/library/library.go
index 4efd98e9..a848d0a0 100644
--- a/vapi/library/library.go
+++ b/vapi/library/library.go
@@ -40,6 +40,7 @@ type Library struct {
 	Description      string            `json:"description,omitempty"`
 	ID               string            `json:"id,omitempty"`
 	LastModifiedTime *time.Time        `json:"last_modified_time,omitempty"`
+	LastSyncTime     *time.Time        `json:"last_sync_time,omitempty"`
 	Name             string            `json:"name,omitempty"`
 	Storage          []StorageBackings `json:"storage_backings,omitempty"`
 	Type             string            `json:"type,omitempty"`
@@ -138,6 +139,13 @@ func (c *Manager) CreateLibrary(ctx context.Context, library Library) (string, e
 	return res, c.Do(ctx, url.Request(http.MethodPost, spec), &res)
 }
 
+// SyncLibrary syncs a subscribed library.
+func (c *Manager) SyncLibrary(ctx context.Context, library *Library) error {
+	path := internal.SubscribedLibraryPath
+	url := c.Resource(path).WithID(library.ID).WithAction("sync")
+	return c.Do(ctx, url.Request(http.MethodPost), nil)
+}
+
 // DeleteLibrary deletes an existing library.
 func (c *Manager) DeleteLibrary(ctx context.Context, library *Library) error {
 	path := internal.LocalLibraryPath
diff --git a/vapi/library/library_item.go b/vapi/library/library_item.go
index c5364177..d8137171 100644
--- a/vapi/library/library_item.go
+++ b/vapi/library/library_item.go
@@ -88,6 +88,15 @@ func (c *Manager) CreateLibraryItem(ctx context.Context, item Item) (string, err
 	return res, c.Do(ctx, url.Request(http.MethodPost, spec), &res)
 }
 
+// SyncLibraryItem syncs a subscribed library item
+func (c *Manager) SyncLibraryItem(ctx context.Context, item *Item, force bool) error {
+	body := struct {
+		Force bool `json:"force_sync_content"`
+	}{force}
+	url := c.Resource(internal.SubscribedLibraryItem).WithID(item.ID).WithAction("sync")
+	return c.Do(ctx, url.Request(http.MethodPost, body), nil)
+}
+
 // DeleteLibraryItem deletes an existing library item.
 func (c *Manager) DeleteLibraryItem(ctx context.Context, item *Item) error {
 	url := c.Resource(internal.LibraryItemPath).WithID(item.ID)
diff --git a/vapi/simulator/simulator.go b/vapi/simulator/simulator.go
index ce89784a..f87619f3 100644
--- a/vapi/simulator/simulator.go
+++ b/vapi/simulator/simulator.go
@@ -129,8 +129,10 @@ func New(u *url.URL, settings []vim.BaseOptionValue) (string, http.Handler) {
 		{internal.SubscribedLibraryPath, s.library},
 		{internal.LibraryPath + "/", s.libraryID},
 		{internal.LocalLibraryPath + "/", s.libraryID},
+		{internal.SubscribedLibraryPath + "/", s.libraryID},
 		{internal.LibraryItemPath, s.libraryItem},
 		{internal.LibraryItemPath + "/", s.libraryItemID},
+		{internal.SubscribedLibraryItem + "/", s.libraryItemID},
 		{internal.LibraryItemUpdateSession, s.libraryItemUpdateSession},
 		{internal.LibraryItemUpdateSession + "/", s.libraryItemUpdateSessionID},
 		{internal.LibraryItemUpdateSessionFile, s.libraryItemUpdateSessionFile},
@@ -145,6 +147,7 @@ func New(u *url.URL, settings []vim.BaseOptionValue) (string, http.Handler) {
 		{internal.VCenterOVFLibraryItem + "/", s.libraryItemDeployID},
 		{internal.VCenterVMTXLibraryItem, s.libraryItemCreateTemplate},
 		{internal.VCenterVMTXLibraryItem + "/", s.libraryItemTemplateID},
+		{internal.VCenterVM + "/", s.vmID},
 	}
 
 	for i := range handlers {
@@ -711,6 +714,14 @@ func (s *handler) libraryID(w http.ResponseWriter, r *http.Request) {
 			l.Patch(&spec.Library)
 			OK(w)
 		}
+	case http.MethodPost:
+	case "sync":
+		if l.Type == "SUBSCRIBED" {
+			l.LastSyncTime = types.NewTime(time.Now())
+			OK(w)
+		} else {
+			http.NotFound(w, r)
+		}
 	case http.MethodGet:
 		OK(w, l)
 	}
@@ -828,6 +839,16 @@ func (s *handler) libraryItemID(w http.ResponseWriter, r *http.Request) {
 			item.Patch(&spec.Item)
 			OK(w)
 		}
+	case http.MethodPost:
+		switch s.action(r) {
+		case "sync":
+			if l.Type == "SUBSCRIBED" {
+				item.LastSyncTime = types.NewTime(time.Now())
+				OK(w)
+			} else {
+				http.NotFound(w, r)
+			}
+		}
 	case http.MethodGet:
 		OK(w, item)
 	}
@@ -1724,3 +1745,14 @@ func (s *handler) libraryItemCheckOuts(item *item, w http.ResponseWriter, r *htt
 		http.NotFound(w, r)
 	}
 }
+
+func (s *handler) vmID(w http.ResponseWriter, r *http.Request) {
+	id := path.Base(r.URL.Path)
+
+	switch r.Method {
+	case http.MethodDelete:
+		s.deleteVM(&types.ManagedObjectReference{Type: "VirtualMachine", Value: id})
+	default:
+		http.NotFound(w, r)
+	}
+}
diff --git a/vapi/vcenter/vcenter_vmtx.go b/vapi/vcenter/vcenter_vmtx.go
index 4b797b30..46c4b1ea 100644
--- a/vapi/vcenter/vcenter_vmtx.go
+++ b/vapi/vcenter/vcenter_vmtx.go
@@ -18,10 +18,14 @@ package vcenter
 
 import (
 	"context"
+	"crypto/sha1"
+	"fmt"
+	"log"
 	"net/http"
 	"path"
 
 	"github.com/vmware/govmomi/vapi/internal"
+	"github.com/vmware/govmomi/vapi/library"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/types"
 )
@@ -157,3 +161,130 @@ func (c *Manager) CheckIn(ctx context.Context, libraryItemID string, vm mo.Refer
 	}{checkin}
 	return res, c.Do(ctx, url.Request(http.MethodPost, spec), &res)
 }
+
+// TemplateLibrary params for synchronizing subscription library OVF items to VM Template items
+type TemplateLibrary struct {
+	Source      library.Library
+	Destination library.Library
+	Placement   Target
+	Include     func(library.Item, *library.Item) bool
+	SyncItem    func(context.Context, library.Item, *Deploy, *Template) error
+}
+
+func (c *Manager) includeTemplateLibraryItem(src library.Item, dst *library.Item) bool {
+	return dst == nil
+}
+
+// SyncTemplateLibraryItem deploys an Library OVF item from which a VM template (vmtx) Library item is created.
+// The deployed VM is deleted after being converted to a Library vmtx item.
+func (c *Manager) SyncTemplateLibraryItem(ctx context.Context, item library.Item, deploy *Deploy, spec *Template) error {
+	destroy := false
+	if spec.SourceVM == "" {
+		ref, err := c.DeployLibraryItem(ctx, item.ID, *deploy)
+		if err != nil {
+			return err
+		}
+
+		destroy = true
+		spec.SourceVM = ref.Value
+	}
+
+	_, err := c.CreateTemplate(ctx, *spec)
+
+	if destroy {
+		// Delete source VM regardless of CreateTemplate result
+		url := c.Resource("/vcenter/vm/" + spec.SourceVM)
+		derr := c.Do(ctx, url.Request(http.MethodDelete), nil)
+		if derr != nil {
+			if err == nil {
+				// Return Delete error if CreateTemplate was successful
+				return derr
+			}
+			// Return CreateTemplate error and just log Delete error
+			log.Printf("destroy %s: %s", spec.SourceVM, derr)
+		}
+	}
+
+	return err
+}
+
+func vmtxSourceName(l library.Library, item library.Item) string {
+	sum := sha1.Sum([]byte(path.Join(l.Name, item.Name)))
+	return fmt.Sprintf("vmtx-src-%x", sum)
+}
+
+// SyncTemplateLibrary converts TemplateLibrary.Source OVF items to VM Template items within TemplateLibrary.Destination
+// The optional TemplateLibrary.Include func can be used to filter which items are synced.
+// By default all items that don't exist in the Destination library are synced.
+// The optional TemplateLibrary.SyncItem func can be used to change how the item is synced, by default SyncTemplateLibraryItem is used.
+func (c *Manager) SyncTemplateLibrary(ctx context.Context, l TemplateLibrary, items ...library.Item) error {
+	m := library.NewManager(c.Client)
+	var err error
+	if len(items) == 0 {
+		items, err = m.GetLibraryItems(ctx, l.Source.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	templates, err := m.GetLibraryItems(ctx, l.Destination.ID)
+	if err != nil {
+		return err
+	}
+
+	existing := make(map[string]*library.Item)
+	for i := range templates {
+		existing[templates[i].Name] = &templates[i]
+	}
+
+	include := l.Include
+	if include == nil {
+		include = c.includeTemplateLibraryItem
+	}
+
+	sync := l.SyncItem
+	if sync == nil {
+		sync = c.SyncTemplateLibraryItem
+	}
+
+	for _, item := range items {
+		if item.Type != library.ItemTypeOVF {
+			continue
+		}
+
+		// Deploy source VM from library ovf item
+		deploy := Deploy{
+			DeploymentSpec: DeploymentSpec{
+				Name:               vmtxSourceName(l.Destination, item),
+				DefaultDatastoreID: l.Destination.Storage[0].DatastoreID,
+				AcceptAllEULA:      true,
+			},
+			Target: l.Placement,
+		}
+
+		// Create library vmtx item from source VM
+		storage := &DiskStorage{
+			Datastore: deploy.DeploymentSpec.DefaultDatastoreID,
+		}
+		spec := Template{
+			Name:          item.Name,
+			Library:       l.Destination.ID,
+			DiskStorage:   storage,
+			VMHomeStorage: storage,
+			Placement: &Placement{
+				Folder:       deploy.Target.FolderID,
+				ResourcePool: deploy.Target.ResourcePoolID,
+			},
+		}
+
+		if !l.Include(item, existing[item.Name]) {
+			continue
+		}
+
+		if err = sync(ctx, item, &deploy, &spec); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}


Test patch:
diff --git a/govc/test/library.bats b/govc/test/library.bats
index f89ea4d9..88a65603 100755
--- a/govc/test/library.bats
+++ b/govc/test/library.bats
@@ -293,6 +293,18 @@ EOF
   assert_success
   assert_matches "Subscription:"
   assert_matches "$url"
+
+  run govc library.import my-content "$GOVC_IMAGES/ttylinux-latest.ova"
+  assert_success # TODO: this should fail, but allow until vcsim supports publishing
+
+  run govc library.sync my-content
+  assert_success
+
+  run govc library.create my-content-vmtx
+  assert_success
+
+  run govc library.sync -vmtx my-content-vmtx my-content
+  assert_success
 }
 
 @test "library.findbyid" {