Repository: redhat-certification/chart-verifier
Pull request #265 (2022-06-02 19:05:53)

allow for pyxis responses spanning more than one page
Updated code to allow for pyxis returning data in multiple pages. Also added some logging to image certification check. 
For testing added test for ```registry.access.redhat.com/rhscl/postgresql-10-rhel7:latest``` which is only found on the second pyxis page. 

see: https://github.com/redhat-certification/chart-verifier/issues/259
see: https://issues.redhat.com/browse/HELM-361

Added a second commit or a second fix needed by IBM. Basically now get the image sha from  the ```image_id```  attribute instead of ```parsed_data.docker_image_id``` which seemed to disappear. The new attribute was recommended by the Pyxis teaam.
see: https://issues.redhat.com/browse/HELM-362


Patch:
diff --git a/pkg/chartverifier/pyxis/pyxis.go b/pkg/chartverifier/pyxis/pyxis.go
index f181ccf..676335f 100644
--- a/pkg/chartverifier/pyxis/pyxis.go
+++ b/pkg/chartverifier/pyxis/pyxis.go
@@ -22,12 +22,17 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+
+	"github.com/redhat-certification/chart-verifier/pkg/chartverifier/utils"
 )
 
 var pyxisBaseUrl = "https://catalog.redhat.com/api/containers/v1/repositories"
 
 type RepositoriesBody struct {
 	PyxisRepositories []PyxisRepository `json:"data"`
+	Page              int               `json:"page"`
+	PageSize          int               `json:"page_size"`
+	Total             int               `json:"total"`
 }
 
 type PyxisRepository struct {
@@ -39,18 +44,17 @@ type PyxisRepository struct {
 
 type RegistriesBody struct {
 	PyxisRegistries []PyxisRegistry `json:"data"`
+	Page            int             `json:"page"`
+	PageSize        int             `json:"page_size"`
+	Total           int             `json:"total"`
 }
 
 type PyxisRegistry struct {
 	Id           string               `json:"_id"`
-	ParsedData   ImageData            `json:"parsed_data"`
+	ImageId      string               `json:"image_id"`
 	Repositories []RegistryRepository `json:"repositories"`
 }
 
-type ImageData struct {
-	Digest string `json:"docker_image_digest"`
-}
-
 type RegistryRepository struct {
 	Registry   string          `json:"registry"`
 	Repository string          `json:"repository"`
@@ -73,34 +77,59 @@ func GetImageRegistries(repository string) ([]string, error) {
 	var err error
 	var registries []string
 
-	req, _ := http.NewRequest("GET", pyxisBaseUrl, nil)
-	queryString := req.URL.Query()
-	queryString.Add("filter", fmt.Sprintf("repository==%s", repository))
-	req.URL.RawQuery = queryString.Encode()
-	req.Header.Set("X-API-KEY", "RedHatChartVerifier")
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		err = errors.New(fmt.Sprintf("Error getting repository %s : %v\n", repository, err))
-	} else {
-		if resp.StatusCode == 200 {
-			defer resp.Body.Close()
-			body, _ := ioutil.ReadAll(resp.Body)
-			var repositoriesBody RepositoriesBody
-			json.Unmarshal(body, &repositoriesBody)
-
-			if len(repositoriesBody.PyxisRepositories) > 0 {
-				for _, repo := range repositoriesBody.PyxisRepositories {
-					registries = append(registries, repo.Registry)
+	read := 0
+	total := 0
+	nextPage := 0
+	allDataRead := false
+
+	for !allDataRead {
+
+		utils.LogInfo(fmt.Sprintf("Look for repository %s at %s, page %d", repository, pyxisBaseUrl, nextPage))
+		req, _ := http.NewRequest("GET", pyxisBaseUrl, nil)
+		queryString := req.URL.Query()
+		queryString.Add("filter", fmt.Sprintf("repository==%s", repository))
+		queryString.Add("page_size", "100")
+		queryString.Add("page", fmt.Sprintf("%d", nextPage))
+		req.URL.RawQuery = queryString.Encode()
+		req.Header.Set("X-API-KEY", "RedHatChartVerifier")
+		client := &http.Client{}
+		resp, reqErr := client.Do(req)
+		if reqErr != nil {
+			err = errors.New(fmt.Sprintf("Error getting repository %s : %v\n", repository, err))
+		} else {
+			if resp.StatusCode == 200 {
+				defer resp.Body.Close()
+				body, _ := ioutil.ReadAll(resp.Body)
+				var repositoriesBody RepositoriesBody
+				json.Unmarshal(body, &repositoriesBody)
+
+				if total == 0 {
+					total = repositoriesBody.Total
+				}
+				read += repositoriesBody.PageSize
+				if read >= total {
+					allDataRead = true
+				} else {
+					nextPage += 1
+				}
+				utils.LogInfo(fmt.Sprintf("page: %d, page_size: %d, total: %d", repositoriesBody.Page, repositoriesBody.PageSize, total))
+				if len(repositoriesBody.PyxisRepositories) > 0 {
+					for _, repo := range repositoriesBody.PyxisRepositories {
+						registries = append(registries, repo.Registry)
+						utils.LogInfo(fmt.Sprintf("Found repository in registry: %s", repo.Registry))
+						fmt.Println(fmt.Sprintf("Found repository in registry: %s", repo.Registry))
+					}
+				} else {
+					err = errors.New(fmt.Sprintf("Respository not found: %s", repository))
 				}
 			} else {
-				err = errors.New(fmt.Sprintf("Respository not found: %s", repository))
+				err = errors.New(fmt.Sprintf("Bad response code from Pyxis: %d : %s", resp.StatusCode, req.URL))
 			}
-		} else {
-			err = errors.New(fmt.Sprintf("Bad response code from Pyxis: %d : %s", resp.StatusCode, req.URL))
 		}
 	}
-
+	if err != nil {
+		utils.LogError(err.Error())
+	}
 	return registries, err
 }
 
@@ -115,59 +144,84 @@ func IsImageInRegistry(imageRef ImageReference) (bool, error) {
 Loops:
 	for _, registry := range imageRef.Registries {
 
-		requestUrl := fmt.Sprintf("%s/registry/%s/repository/%s/images", pyxisBaseUrl, registry, imageRef.Repository)
-		req, _ := http.NewRequest("GET", requestUrl, nil)
-		queryString := req.URL.Query()
-		queryString.Add("filter", fmt.Sprintf("repositories=em=(repository==%s;registry==%s)", imageRef.Repository, registry))
-		req.URL.RawQuery = queryString.Encode()
-		req.Header.Set("X-API-KEY", "RedHatChartVerifier")
-		client := &http.Client{}
-		resp, reqErr := client.Do(req)
+		read := 0
+		total := 0
+		nextPage := 0
+		allDataRead := false
 
-		if reqErr == nil {
-			if resp.StatusCode == 200 {
-				defer resp.Body.Close()
-				body, _ := ioutil.ReadAll(resp.Body)
-				var registriesBody RegistriesBody
-				json.Unmarshal(body, &registriesBody)
-
-				if len(registriesBody.PyxisRegistries) > 0 {
-					found = false
-					for _, reg := range registriesBody.PyxisRegistries {
-						if len(imageRef.Sha) > 0 {
-							if imageRef.Sha == reg.ParsedData.Digest {
-								found = true
-								err = nil
-								continue Loops
+		requestUrl := fmt.Sprintf("%s/registry/%s/repository/%s/images", pyxisBaseUrl, registry, imageRef.Repository)
+		utils.LogInfo(fmt.Sprintf("Search url: %s, tag: %s, sha: %s ", requestUrl, imageRef.Tag, imageRef.Sha))
+
+		for !allDataRead && err == nil && !found {
+
+			req, _ := http.NewRequest("GET", requestUrl, nil)
+			queryString := req.URL.Query()
+			queryString.Add("filter", fmt.Sprintf("repositories=em=(repository==%s;registry==%s)", imageRef.Repository, registry))
+			queryString.Add("page_size", "100")
+			queryString.Add("page", fmt.Sprintf("%d", nextPage))
+			req.URL.RawQuery = queryString.Encode()
+			req.Header.Set("X-API-KEY", "RedHatChartVerifier")
+			client := &http.Client{}
+			resp, reqErr := client.Do(req)
+
+			if reqErr == nil {
+				if resp.StatusCode == 200 {
+					defer resp.Body.Close()
+					body, _ := ioutil.ReadAll(resp.Body)
+					var registriesBody RegistriesBody
+					json.Unmarshal(body, &registriesBody)
+
+					if total == 0 {
+						total = registriesBody.Total
+					}
+					read += registriesBody.PageSize
+					if read >= total {
+						allDataRead = true
+					} else {
+						nextPage += 1
+					}
+					utils.LogInfo(fmt.Sprintf("page: %d, page_size: %d, total: %d", registriesBody.Page, registriesBody.PageSize, registriesBody.Total))
+
+					if len(registriesBody.PyxisRegistries) > 0 {
+						found = false
+						for _, reg := range registriesBody.PyxisRegistries {
+							if len(imageRef.Sha) > 0 {
+								if imageRef.Sha == reg.ImageId {
+									utils.LogInfo(fmt.Sprintf("sha found: %s", imageRef.Sha))
+									found = true
+									err = nil
+									continue Loops
+								} else {
+									shas = append(shas, reg.ImageId)
+								}
 							} else {
-								shas = append(shas, reg.ParsedData.Digest)
-							}
-						} else {
-							for _, repo := range reg.Repositories {
-								if repo.Repository == imageRef.Repository && repo.Registry == registry {
-									if len(imageRef.Sha) == 0 {
-										for _, tag := range repo.Tags {
-											if tag.Name == imageRef.Tag {
-												found = true
-												err = nil
-												continue Loops
-											} else {
-												tags = append(tags, tag.Name)
+								for _, repo := range reg.Repositories {
+									if repo.Repository == imageRef.Repository && repo.Registry == registry {
+										if len(imageRef.Sha) == 0 {
+											for _, tag := range repo.Tags {
+												if tag.Name == imageRef.Tag {
+													utils.LogInfo(fmt.Sprintf("tag found: %s", imageRef.Tag))
+													found = true
+													err = nil
+													continue Loops
+												} else {
+													tags = append(tags, tag.Name)
+												}
 											}
 										}
 									}
 								}
 							}
 						}
+					} else {
+						err = errors.New(fmt.Sprintf("No images found for Registry/Repository: %s/%s", registry, imageRef.Repository))
 					}
 				} else {
-					err = errors.New(fmt.Sprintf("No images found for Registry/Repository: %s/%s", registry, imageRef.Repository))
+					err = errors.New(fmt.Sprintf("Bad response code %d from pyxis request : %s", resp.StatusCode, requestUrl))
 				}
 			} else {
-				err = errors.New(fmt.Sprintf("Bad response code %d from pyxis request : %s", resp.StatusCode, requestUrl))
+				err = reqErr
 			}
-		} else {
-			err = reqErr
 		}
 	}
 	if !found {
@@ -179,5 +233,8 @@ Loops:
 			}
 		}
 	}
+	if err != nil {
+		utils.LogError(err.Error())
+	}
 	return found, err
 }


Test patch:
diff --git a/pkg/chartverifier/pyxis/pyxis_test.go b/pkg/chartverifier/pyxis/pyxis_test.go
index fd33520..41421ec 100644
--- a/pkg/chartverifier/pyxis/pyxis_test.go
+++ b/pkg/chartverifier/pyxis/pyxis_test.go
@@ -35,7 +35,8 @@ func Test_getImageRegistries(t *testing.T) {
 		{description: "Test rhel6.7 respository", repository: "rhel6.7", registry: "registry.access.redhat.com", message: ""},
 		{description: "Test rhel8/nginx-116 respository", repository: "rhel8/nginx-116", registry: "registry.access.redhat.com", message: ""},
 		{description: "Test ibm/nginx respository", repository: "ibm/nginx", registry: "non_registry", message: ""},
-		{description: "Test turbonomic/zookeeper respository", repository: "turbonomic/zookeeper", registry: "registry.connect.redhat.com", message: ""}}
+		{description: "Test turbonomic/zookeeper respository", repository: "turbonomic/zookeeper", registry: "registry.connect.redhat.com", message: ""},
+		{description: "Test cpopen/ibmcloud-object-storage-driver respository", repository: "cpopen/ibmcloud-object-storage-driver", registry: "icr.io", message: ""}}
 
 	for _, tc := range PassTestCases {
 		t.Run(tc.description, func(t *testing.T) {
@@ -75,8 +76,10 @@ func Test_checkImageInRegistry(t *testing.T) {
 		{description: "Test turbonomic/zookeeper respository and version found.", imageRef: ImageReference{Repository: "turbonomic/zookeeper", Registries: []string{"registry.connect.redhat.com"}, Tag: "8.1.2", Sha: ""}, message: ""},
 		{description: "Test cpopen/ibmcloud-object-storage-driver respository and sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-driver", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:fc17bb3e89d00b3eb0f50b3ea83aa75c52e43d8e56cf2e0f17475e934eeeeb5f"}, message: ""},
 		{description: "Test cpopen/ibmcloud-object-storage-plugin respository and sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-plugin", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:cf654987c38d048bc9e654f3928e9ce9a2a4fd47ce0283bb5f339c1b99298e6e"}, message: ""},
+		{description: "Test postgresql-10-rhel7 respository and tag found", imageRef: ImageReference{Repository: "rhscl/postgresql-10-rhel7", Registries: []string{"registry.access.redhat.com"}, Tag: "latest", Sha: ""}, message: ""},
+		{description: "Test cpopen/ibmcloud-object-storage-plugin respository sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-plugin", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:7c00bc76f91d456164f98375cd8932a0ec500c9dca1728368f3c1ccdbfd96e91"}, message: ""},
+		{description: "Test cpopen/ibmcloud-object-storage-driver respository sha found.", imageRef: ImageReference{Repository: "cpopen/ibmcloud-object-storage-driver", Registries: []string{"icr.io"}, Tag: "", Sha: "sha256:667667c5907d0ad145e8518ca0f8cf013ca778d6738b028d1cd08103b1b64667"}, message: ""},
 	}
-
 	for _, tc := range PassTestCases {
 		t.Run(tc.description, func(t *testing.T) {
 			found, err := IsImageInRegistry(tc.imageRef)