Repository: apigee/registry
Pull request #552 (2022-04-26 23:28:22)

Add support for compute score command
Summary of changes:
* Added support for command `registry compute score`.
* TODOs:
   - timestamp based check for artifact dependencies in score calculation
   - support for dry_run flag
   - support for RollUp score
   - Support for json artifacts

codecov[bot]: # [Codecov](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee) Report
> Merging [#552](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee) (df32832) into [main](https://codecov.io/gh/apigee/registry/commit/d11444289d6e9f244d1f2b18be2529ad850cc326?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee) (d114442) will **decrease** coverage by `2.15%`.
> The diff coverage is `0.71%`.

> :exclamation: Current head df32832 differs from pull request most recent head 67b8cd2. Consider uploading reports for the commit 67b8cd2 to get more accurate results

```diff
@@            Coverage Diff             @@
##             main     #552      +/-   ##
==========================================
- Coverage   41.26%   39.11%   -2.16%     
==========================================
  Files          69       71       +2     
  Lines        8191     8534     +343     
==========================================
- Hits         3380     3338      -42     
- Misses       4404     4798     +394     
+ Partials      407      398       -9     
```


| [Impacted Files](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee) | Coverage Δ | |
|---|---|---|
| [cmd/registry/cmd/compute/compute.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L2NtZC9jb21wdXRlL2NvbXB1dGUuZ28=) | `0.00% <0.00%> (ø)` | |
| [cmd/registry/cmd/compute/score.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L2NtZC9jb21wdXRlL3Njb3JlLmdv) | `0.00% <0.00%> (ø)` | |
| [cmd/registry/patterns/resources.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L3BhdHRlcm5zL3Jlc291cmNlcy5nbw==) | `19.50% <0.00%> (-7.07%)` | :arrow_down: |
| [cmd/registry/scoring/expression.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L3Njb3JpbmcvZXhwcmVzc2lvbi5nbw==) | `0.00% <0.00%> (ø)` | |
| [cmd/registry/scoring/score.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L3Njb3Jpbmcvc2NvcmUuZ28=) | `0.00% <0.00%> (ø)` | |
| [cmd/registry/controller/controller.go](https://codecov.io/gh/apigee/registry/pull/552/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee#diff-Y21kL3JlZ2lzdHJ5L2NvbnRyb2xsZXIvY29udHJvbGxlci5nbw==) | `76.95% <100.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee). Last update [d114442...67b8cd2](https://codecov.io/gh/apigee/registry/pull/552?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apigee).

shrutiparabgoogle: > I still have some open comments about improving test descriptions, but I won't block the PR for that. LGTM overall.

The test descriptions were fixed in [this](https://github.com/apigee/registry/pull/552/commits/a3a7aa0640a48f59c8d3fde1e2791449e65604ef#diff-4d809861e18c54411199dc95b93bc7afdc8abf9b50c1c48939ec99f9fec05694) commit. I missed resolving the comments.

Patch:
diff --git a/cmd/registry/cmd/compute/compute.go b/cmd/registry/cmd/compute/compute.go
index 06d661a..b4ab248 100644
--- a/cmd/registry/cmd/compute/compute.go
+++ b/cmd/registry/cmd/compute/compute.go
@@ -29,6 +29,7 @@ func Command() *cobra.Command {
 	cmd.AddCommand(lintCommand())
 	cmd.AddCommand(lintStatsCommand())
 	cmd.AddCommand(referencesCommand())
+	cmd.AddCommand(scoreCommand())
 	cmd.AddCommand(vocabularyCommand())
 
 	cmd.PersistentFlags().String("filter", "", "Filter selected resources")
diff --git a/cmd/registry/cmd/compute/score.go b/cmd/registry/cmd/compute/score.go
new file mode 100644
index 0000000..eadd83e
--- /dev/null
+++ b/cmd/registry/cmd/compute/score.go
@@ -0,0 +1,75 @@
+package compute
+
+import (
+	"context"
+
+	"github.com/apigee/registry/cmd/registry/core"
+	"github.com/apigee/registry/cmd/registry/patterns"
+	"github.com/apigee/registry/cmd/registry/scoring"
+	"github.com/apigee/registry/connection"
+	"github.com/apigee/registry/log"
+	"github.com/apigee/registry/rpc"
+	"github.com/spf13/cobra"
+)
+
+func scoreCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "score (wip)",
+		Short: "Compute scores for APIs and API specs",
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			ctx := cmd.Context()
+			filter, err := cmd.Flags().GetString("filter")
+			if err != nil {
+				log.FromContext(ctx).WithError(err).Fatal("Failed to get filter from flags")
+			}
+
+			client, err := connection.NewClient(ctx)
+			if err != nil {
+				log.FromContext(ctx).WithError(err).Fatal("Failed to get client")
+			}
+			// Initialize task queue.
+			taskQueue, wait := core.WorkerPool(ctx, 64)
+			defer wait()
+
+			resources, err := patterns.ListResources(ctx, client, args[0], filter)
+			if err != nil {
+				log.FromContext(ctx).WithError(err).Fatal("Failed to list resources")
+			}
+
+			for _, r := range resources {
+				// Fetch the ScoreDefinitions which can be applied to this resource
+				scoreDefinitions, err := scoring.FetchScoreDefinitions(ctx, client, r.ResourceName())
+				if err != nil {
+					log.FromContext(ctx).WithError(err).Errorf("Skipping resource %q", r.ResourceName())
+					continue
+				}
+				for _, d := range scoreDefinitions {
+					taskQueue <- &computeScoreTask{
+						ctx:        ctx,
+						client:     client,
+						definition: d,
+						resource:   r,
+					}
+				}
+			}
+
+			return
+		},
+	}
+}
+
+type computeScoreTask struct {
+	ctx        context.Context
+	client     connection.Client
+	definition *rpc.ScoreDefinition
+	resource   patterns.ResourceInstance
+}
+
+func (task *computeScoreTask) String() string {
+	return "compute score " + task.resource.ResourceName().String()
+}
+
+func (task *computeScoreTask) Run(ctx context.Context) error {
+	return scoring.CalculateScore(task.ctx, task.client, task.definition, task.resource)
+}
diff --git a/cmd/registry/controller/controller.go b/cmd/registry/controller/controller.go
index 1996d58..65b647c 100644
--- a/cmd/registry/controller/controller.go
+++ b/cmd/registry/controller/controller.go
@@ -118,7 +118,7 @@ func generateDependencyMap(
 	}
 
 	// Fetch resources using the extDependencyQuery
-	sourceList, err := listResources(ctx, client, extDependencyName.String(), dependency.Filter)
+	sourceList, err := patterns.ListResources(ctx, client, extDependencyName.String(), dependency.Filter)
 	if err != nil {
 		return nil, err
 	}
@@ -183,7 +183,7 @@ func generateUpdateActions(
 	actions := make([]*Action, 0)
 
 	// Generate resource list
-	resourceList, err := listResources(ctx, client, resourcePattern, filter)
+	resourceList, err := patterns.ListResources(ctx, client, resourcePattern, filter)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -284,7 +284,7 @@ func generateCreateActions(
 
 	default:
 		filter := excludeVisitedParents(visited)
-		parentList, err = listResources(ctx, client, parentName.String(), filter)
+		parentList, err = patterns.ListResources(ctx, client, parentName.String(), filter)
 		if err != nil {
 			return nil, err
 		}
diff --git a/cmd/registry/controller/list.go b/cmd/registry/controller/list.go
deleted file mode 100644
index 57b21df..0000000
--- a/cmd/registry/controller/list.go
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2021 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package controller
-
-import (
-	"context"
-
-	"github.com/apigee/registry/cmd/registry/core"
-	"github.com/apigee/registry/cmd/registry/patterns"
-	"github.com/apigee/registry/connection"
-	"github.com/apigee/registry/rpc"
-	"github.com/apigee/registry/server/registry/names"
-)
-
-func listResources(ctx context.Context, client connection.Client, pattern, filter string) ([]patterns.ResourceInstance, error) {
-	var result []patterns.ResourceInstance
-	var err2 error
-
-	// First try to match collection names.
-	if api, err := names.ParseApiCollection(pattern); err == nil {
-		err2 = core.ListAPIs(ctx, client, api, filter, generateApiHandler(&result))
-	} else if version, err := names.ParseVersionCollection(pattern); err == nil {
-		err2 = core.ListVersions(ctx, client, version, filter, generateVersionHandler(&result))
-	} else if spec, err := names.ParseSpecCollection(pattern); err == nil {
-		err2 = core.ListSpecs(ctx, client, spec, filter, generateSpecHandler(&result))
-	} else if artifact, err := names.ParseArtifactCollection(pattern); err == nil {
-		err2 = core.ListArtifacts(ctx, client, artifact, filter, false, generateArtifactHandler(&result))
-	}
-
-	// Then try to match resource names.
-	if api, err := names.ParseApi(pattern); err == nil {
-		err2 = core.ListAPIs(ctx, client, api, filter, generateApiHandler(&result))
-	} else if version, err := names.ParseVersion(pattern); err == nil {
-		err2 = core.ListVersions(ctx, client, version, filter, generateVersionHandler(&result))
-	} else if spec, err := names.ParseSpec(pattern); err == nil {
-		err2 = core.ListSpecs(ctx, client, spec, filter, generateSpecHandler(&result))
-	} else if artifact, err := names.ParseArtifact(pattern); err == nil {
-		err2 = core.ListArtifacts(ctx, client, artifact, filter, false, generateArtifactHandler(&result))
-	}
-
-	if err2 != nil {
-		return nil, err2
-	}
-
-	return result, nil
-}
-
-func generateApiHandler(result *[]patterns.ResourceInstance) func(*rpc.Api) error {
-	return func(api *rpc.Api) error {
-		name, err := names.ParseApi(api.GetName())
-		if err != nil {
-			return err
-		}
-
-		(*result) = append((*result), patterns.ApiResource{
-			ApiName:   patterns.ApiName{Name: name},
-			Timestamp: api.UpdateTime.AsTime(),
-		})
-
-		return nil
-	}
-}
-
-func generateVersionHandler(result *[]patterns.ResourceInstance) func(*rpc.ApiVersion) error {
-	return func(version *rpc.ApiVersion) error {
-		name, err := names.ParseVersion(version.GetName())
-		if err != nil {
-			return err
-		}
-
-		(*result) = append((*result), patterns.VersionResource{
-			VersionName: patterns.VersionName{Name: name},
-			Timestamp:   version.UpdateTime.AsTime(),
-		})
-
-		return nil
-	}
-}
-
-func generateSpecHandler(result *[]patterns.ResourceInstance) func(*rpc.ApiSpec) error {
-	return func(spec *rpc.ApiSpec) error {
-		name, err := names.ParseSpec(spec.GetName())
-		if err != nil {
-			return err
-		}
-
-		(*result) = append((*result), patterns.SpecResource{
-			SpecName:  patterns.SpecName{Name: name},
-			Timestamp: spec.RevisionUpdateTime.AsTime(),
-		})
-
-		return nil
-	}
-}
-
-func generateArtifactHandler(result *[]patterns.ResourceInstance) func(*rpc.Artifact) error {
-	return func(artifact *rpc.Artifact) error {
-		name, err := names.ParseArtifact(artifact.GetName())
-		if err != nil {
-			return err
-		}
-
-		(*result) = append((*result), patterns.ArtifactResource{
-			ArtifactName: patterns.ArtifactName{Name: name},
-			Timestamp:    artifact.UpdateTime.AsTime(),
-		})
-
-		return nil
-	}
-}
diff --git a/cmd/registry/patterns/resources.go b/cmd/registry/patterns/resources.go
index ae11111..e2cffa8 100644
--- a/cmd/registry/patterns/resources.go
+++ b/cmd/registry/patterns/resources.go
@@ -15,8 +15,12 @@
 package patterns
 
 import (
+	"context"
 	"time"
 
+	"github.com/apigee/registry/cmd/registry/core"
+	"github.com/apigee/registry/connection"
+	"github.com/apigee/registry/rpc"
 	"github.com/apigee/registry/server/registry/names"
 )
 
@@ -391,3 +395,100 @@ func (ar ArtifactResource) UpdateTimestamp() time.Time {
 func (ar ArtifactResource) ResourceName() ResourceName {
 	return ar.ArtifactName
 }
+
+func ListResources(ctx context.Context, client connection.Client, pattern, filter string) ([]ResourceInstance, error) {
+	var result []ResourceInstance
+	var err2 error
+
+	// First try to match collection names.
+	if api, err := names.ParseApiCollection(pattern); err == nil {
+		err2 = core.ListAPIs(ctx, client, api, filter, generateApiHandler(&result))
+	} else if version, err := names.ParseVersionCollection(pattern); err == nil {
+		err2 = core.ListVersions(ctx, client, version, filter, generateVersionHandler(&result))
+	} else if spec, err := names.ParseSpecCollection(pattern); err == nil {
+		err2 = core.ListSpecs(ctx, client, spec, filter, generateSpecHandler(&result))
+	} else if artifact, err := names.ParseArtifactCollection(pattern); err == nil {
+		err2 = core.ListArtifacts(ctx, client, artifact, filter, false, generateArtifactHandler(&result))
+	}
+
+	// Then try to match resource names.
+	if api, err := names.ParseApi(pattern); err == nil {
+		err2 = core.ListAPIs(ctx, client, api, filter, generateApiHandler(&result))
+	} else if version, err := names.ParseVersion(pattern); err == nil {
+		err2 = core.ListVersions(ctx, client, version, filter, generateVersionHandler(&result))
+	} else if spec, err := names.ParseSpec(pattern); err == nil {
+		err2 = core.ListSpecs(ctx, client, spec, filter, generateSpecHandler(&result))
+	} else if artifact, err := names.ParseArtifact(pattern); err == nil {
+		err2 = core.ListArtifacts(ctx, client, artifact, filter, false, generateArtifactHandler(&result))
+	}
+
+	if err2 != nil {
+		return nil, err2
+	}
+
+	return result, nil
+}
+
+func generateApiHandler(result *[]ResourceInstance) func(*rpc.Api) error {
+	return func(api *rpc.Api) error {
+		name, err := names.ParseApi(api.GetName())
+		if err != nil {
+			return err
+		}
+
+		(*result) = append((*result), ApiResource{
+			ApiName:   ApiName{Name: name},
+			Timestamp: api.UpdateTime.AsTime(),
+		})
+
+		return nil
+	}
+}
+
+func generateVersionHandler(result *[]ResourceInstance) func(*rpc.ApiVersion) error {
+	return func(version *rpc.ApiVersion) error {
+		name, err := names.ParseVersion(version.GetName())
+		if err != nil {
+			return err
+		}
+
+		(*result) = append((*result), VersionResource{
+			VersionName: VersionName{Name: name},
+			Timestamp:   version.UpdateTime.AsTime(),
+		})
+
+		return nil
+	}
+}
+
+func generateSpecHandler(result *[]ResourceInstance) func(*rpc.ApiSpec) error {
+	return func(spec *rpc.ApiSpec) error {
+		name, err := names.ParseSpec(spec.GetName())
+		if err != nil {
+			return err
+		}
+
+		(*result) = append((*result), SpecResource{
+			SpecName:  SpecName{Name: name},
+			Timestamp: spec.RevisionUpdateTime.AsTime(),
+		})
+
+		return nil
+	}
+}
+
+func generateArtifactHandler(result *[]ResourceInstance) func(*rpc.Artifact) error {
+	return func(artifact *rpc.Artifact) error {
+		name, err := names.ParseArtifact(artifact.GetName())
+		if err != nil {
+			return err
+		}
+
+		(*result) = append((*result), ArtifactResource{
+			ArtifactName: ArtifactName{Name: name},
+			Timestamp:    artifact.UpdateTime.AsTime(),
+		})
+
+		return nil
+	}
+}
diff --git a/cmd/registry/scoring/expression.go b/cmd/registry/scoring/expression.go
new file mode 100644
index 0000000..a63e0c4
--- /dev/null
+++ b/cmd/registry/scoring/expression.go
@@ -0,0 +1,117 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scoring
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/apigee/registry/cmd/registry/core"
+	"github.com/apigee/registry/rpc"
+	"github.com/google/cel-go/cel"
+	metrics "github.com/google/gnostic/metrics"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+)
+
+func evaluateScoreExpression(expression, mimeType string, contents []byte) (interface{}, error) {
+	// https://github.com/google/cel-spec/blob/master/doc/langdef.md#dynamic-values
+	structProto, err := getMap(contents, mimeType)
+	if err != nil {
+		return nil, err
+	}
+
+	env, err := cel.NewEnv()
+	if err != nil {
+		return nil, fmt.Errorf("error creating CEL environment: %s", err)
+	}
+
+	ast, issues := env.Parse(expression)
+	if issues != nil && issues.Err() != nil {
+		return nil, fmt.Errorf("error parsing score_expression, %q: %s", expression, issues)
+	}
+	prg, err := env.Program(ast)
+	if err != nil {
+		return nil, fmt.Errorf("program construction error, %q: %s", expression, err)
+	}
+
+	out, _, err := prg.Eval(structProto)
+	if err != nil {
+		return nil, fmt.Errorf("error in evaluating expression %q: %s", expression, err)
+	}
+
+	switch value := out.Value().(type) {
+	case int64, float64, bool:
+		return value, nil
+	default:
+		return nil, fmt.Errorf("evaluating expression %q generated an unexpected output type %T: should be one of [int, double, bool]", expression, value)
+	}
+}
+
+func getMap(contents []byte, mimeType string) (map[string]interface{}, error) {
+	messageType, err := core.MessageTypeForMimeType(mimeType)
+	if err != nil {
+		return nil, fmt.Errorf("failed extracting message type from %q", mimeType)
+	}
+
+	switch messageType {
+	case "gnostic.metrics.Complexity":
+		return unmarshalAndMap(contents, &metrics.Complexity{})
+	case "gnostic.metrics.Vocabulary":
+		return unmarshalAndMap(contents, &metrics.Vocabulary{})
+	case "google.cloud.apigeeregistry.applications.v1alpha1.ConformanceReport":
+		return unmarshalAndMap(contents, &rpc.ConformanceReport{})
+	case "google.cloud.apigeeregistry.applications.v1alpha1.Index":
+		return unmarshalAndMap(contents, &rpc.Index{})
+	case "google.cloud.apigeeregistry.applications.v1alpha1.Lint":
+		return unmarshalAndMap(contents, &rpc.Lint{})
+	case "google.cloud.apigeeregistry.v1.apihub.ReferenceList":
+		return unmarshalAndMap(contents, &rpc.ReferenceList{})
+	case "google.cloud.apigeeregistry.v1.controller.Receipt":
+		return unmarshalAndMap(contents, &rpc.Receipt{})
+	case "google.cloud.apigeeregistry.applications.v1alpha1.References":
+		return unmarshalAndMap(contents, &rpc.References{})
+	case "google.cloud.apigeeregistry.v1.scoring.Score":
+		return unmarshalAndMap(contents, &rpc.Score{})
+	case "google.cloud.apigeeregistry.v1.scoring.ScoreCard":
+		return unmarshalAndMap(contents, &rpc.ScoreCard{})
+	// TODO: Add support for JSON artifacts
+	default:
+		return nil, fmt.Errorf("unsupported artifact type: %s", messageType)
+	}
+}
+
+func unmarshalAndMap(contents []byte, message proto.Message) (map[string]interface{}, error) {
+	// Convert to proto
+	err := proto.Unmarshal(contents, message)
+	if err != nil {
+		return nil, fmt.Errorf("failed unmarshling: %s", err)
+	}
+
+	// Convert proto to json
+	jsonData, err := protojson.Marshal(message)
+	if err != nil {
+		return nil, fmt.Errorf("failed converting to json: %s", err)
+	}
+
+	var mapValue map[string]interface{}
+	// Convert json to map
+	err = json.Unmarshal(jsonData, &mapValue)
+	if err != nil {
+		return nil, fmt.Errorf("failed converting to map: %s", err)
+	}
+
+	return mapValue, nil
+}
diff --git a/cmd/registry/scoring/parser.go b/cmd/registry/scoring/parser.go
index ef69bbd..0d22569 100644
--- a/cmd/registry/scoring/parser.go
+++ b/cmd/registry/scoring/parser.go
@@ -1,3 +1,17 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package scoring
 
 import (
diff --git a/cmd/registry/scoring/score.go b/cmd/registry/scoring/score.go
new file mode 100644
index 0000000..eb60de7
--- /dev/null
+++ b/cmd/registry/scoring/score.go
@@ -0,0 +1,357 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scoring
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/apigee/registry/cmd/registry/core"
+	"github.com/apigee/registry/cmd/registry/patch"
+	"github.com/apigee/registry/cmd/registry/patterns"
+	"github.com/apigee/registry/connection"
+	"github.com/apigee/registry/log"
+	"github.com/apigee/registry/rpc"
+	"github.com/apigee/registry/server/registry/names"
+	"google.golang.org/protobuf/proto"
+)
+
+func FetchScoreDefinitions(
+	ctx context.Context,
+	client connection.Client,
+	resource patterns.ResourceName) ([]*rpc.ScoreDefinition, error) {
+	definitions := make([]*rpc.ScoreDefinition, 0)
+
+	project := fmt.Sprintf("%s/locations/global", resource.Project())
+	artifact, err := names.ParseArtifact(fmt.Sprintf("%s/artifacts/-", project))
+	if err != nil {
+		return nil, err
+	}
+	listFilter := fmt.Sprintf("mime_type == %q", patch.ScoreDefinitionMimeType)
+	err = core.ListArtifacts(ctx, client, artifact, listFilter, true,
+		func(artifact *rpc.Artifact) error {
+			definition := &rpc.ScoreDefinition{}
+			if err := proto.Unmarshal(artifact.GetContents(), definition); err != nil {
+				return err
+			}
+
+			// Check if ScoreDefinition.TargetResource matches with the supplied resource
+			err := matchResourceWithTarget(definition.GetTargetResource(), resource, project)
+			if err != nil {
+				return err
+			}
+
+			definitions = append(definitions, definition)
+			return nil
+		})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return definitions, nil
+}
+
+func matchResourceWithTarget(targetPattern *rpc.ResourcePattern, resourceName patterns.ResourceName, project string) error {
+	targetPatternName, err := patterns.ParseResourcePattern(fmt.Sprintf("%s/%s", project, targetPattern.GetPattern()))
+	if err != nil {
+		return err
+	}
+
+	switch tp := targetPatternName.(type) {
+	case patterns.SpecName:
+		// Check if targetPattern and resourceName match in type
+		r, ok := resourceName.(patterns.SpecName)
+		if !ok {
+			return fmt.Errorf("resource %s doesn't match target pattern %s", r, tp)
+		}
+
+		// Check if the individual entities match
+		if tp.Name.ApiID != "-" && tp.Name.ApiID != r.Name.ApiID {
+			return fmt.Errorf("api mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+		if tp.Name.VersionID != "-" && tp.Name.VersionID != r.Name.VersionID {
+			return fmt.Errorf("version mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+		if tp.Name.SpecID != "-" && tp.Name.SpecID != r.Name.SpecID {
+			return fmt.Errorf("spec mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+	case patterns.VersionName:
+		// Check if targetPattern and resourceName match in type
+		r, ok := resourceName.(patterns.VersionName)
+		if !ok {
+			return fmt.Errorf("resource %s doesn't match target pattern %s", r, tp)
+		}
+
+		// Check if the individual entities match
+		if tp.Name.ApiID != "-" && tp.Name.ApiID != r.Name.ApiID {
+			return fmt.Errorf("api mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+		if tp.Name.VersionID != "-" && tp.Name.VersionID != r.Name.VersionID {
+			return fmt.Errorf("version mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+	case patterns.ApiName:
+		// Check if targetPattern and resourceName match in type
+		r, ok := resourceName.(patterns.ApiName)
+		if !ok {
+			return fmt.Errorf("resource %s doesn't match target pattern %s", r, tp)
+		}
+
+		// Check if the individual entities match
+		if tp.Name.ApiID != "-" && tp.Name.ApiID != r.Name.ApiID {
+			return fmt.Errorf("api mismatch in resource %s and target pattern %v", resourceName.String(), targetPattern)
+		}
+	default:
+		return fmt.Errorf("unsupported resource type %T", targetPatternName)
+	}
+
+	// TODO: Filter check
+	return nil
+}
+
+func CalculateScore(
+	ctx context.Context,
+	client connection.Client,
+	definition *rpc.ScoreDefinition,
+	resource patterns.ResourceInstance) error {
+	project := fmt.Sprintf("%s/locations/global", resource.ResourceName().Project())
+
+	// evaluate the expression and return a scoreValue
+	scoreValue, err := processFormula(ctx, client, definition, resource)
+	if err != nil {
+		return err
+	}
+
+	// generate a score proto from the scoreValue
+	score, err := processScoreType(definition, scoreValue, project)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Add dry_run flag
+
+	err = uploadScore(ctx, client, resource, score)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func processFormula(
+	ctx context.Context,
+	client connection.Client,
+	definition *rpc.ScoreDefinition,
+	resource patterns.ResourceInstance) (interface{}, error) {
+	// Apply score formula
+	switch formula := definition.GetFormula().(type) {
+	case *rpc.ScoreDefinition_ScoreFormula:
+		scoreValue, err := processScoreFormula(ctx, client, formula.ScoreFormula, resource)
+		if err != nil {
+			return nil, err
+		}
+		return scoreValue, nil
+	// TODO: Add support for rollup_formula
+	default:
+		return nil, fmt.Errorf("invalid formula in ScoreDefinition: {%v} ", formula)
+	}
+}
+
+func processScoreFormula(
+	ctx context.Context,
+	client connection.Client,
+	formula *rpc.ScoreFormula,
+	resource patterns.ResourceInstance) (interface{}, error) {
+	// Fetch the artifact
+	extendedArtifact, err := patterns.SubstituteReferenceEntity(formula.GetArtifact().GetPattern(), resource.ResourceName())
+	if err != nil {
+		return nil, fmt.Errorf("invalid score_formula.artifact.pattern: %s, %s", formula.GetArtifact().GetPattern(), err)
+	}
+	artifactName, err := names.ParseArtifact(extendedArtifact.String())
+	if err != nil {
+		return nil, fmt.Errorf("invalid score_formula.artifact.pattern: %s, %s", formula.GetArtifact().GetPattern(), err)
+	}
+
+	var contents []byte
+	var mimeType string
+	err = core.GetArtifact(ctx, client, artifactName, true, func(artifact *rpc.Artifact) error {
+		contents = artifact.GetContents()
+		mimeType = artifact.GetMimeType()
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch artifact %s: %s", artifactName, err)
+	}
+
+	// TODO: Add timestamp check. Compute the score only if the artifact has an update since the last score calculation
+
+	// Apply the score_expression
+	value, err := evaluateScoreExpression(formula.GetScoreExpression(), mimeType, contents)
+	if err != nil {
+		return nil, err
+	}
+
+	return value, nil
+}
+
+func processScoreType(definition *rpc.ScoreDefinition, scoreValue interface{}, project string) (*rpc.Score, error) {
+	// Initialize Score proto
+	score := &rpc.Score{
+		Id:             fmt.Sprintf("score-%s", definition.GetId()),
+		Kind:           "Score",
+		DisplayName:    definition.GetDisplayName(),
+		Description:    definition.GetDescription(),
+		Uri:            definition.GetUri(),
+		UriDisplayName: definition.GetUriDisplayName(),
+		DefinitionName: fmt.Sprintf("%s/artifacts/%s", project, definition.GetId()),
+	}
+
+	// Set the Value field according to the type
+	switch definition.GetType().(type) {
+	case *rpc.ScoreDefinition_Integer:
+		// Score proto expects int32 type
+		var value int32
+
+		// Convert scoreValue to appropriate type
+		// evaluateScoreExpression can return either a float or int value.
+		// Both are valid for an integer.
+		switch v := scoreValue.(type) {
+		case int64:
+			value = int32(v)
+		case float64:
+			value = int32(v)
+		default:
+			return nil, fmt.Errorf("failed typecheck for output: expected either int64 or float64 got %s (type: %T)", v, v)
+		}
+
+		// Check that the scoreValue is within min/max limits
+		configuredMin := definition.GetInteger().GetMinValue() // 0 if not set
+		configuredMax := definition.GetInteger().GetMaxValue() // 0 if not set
+		if value < configuredMin {
+			return nil, fmt.Errorf("evaluated score value(%d) cannot be less than the configured min_value (%d)", value, configuredMin)
+		}
+		if value > configuredMax {
+			return nil, fmt.Errorf("evaluated score value(%d) cannot be greater than the configured max_value (%d)", value, configuredMax)
+		}
+
+		// Populate Value field in Score proto
+		score.Value = &rpc.Score_IntegerValue{
+			IntegerValue: &rpc.IntegerValue{
+				Value:    value,
+				MinValue: configuredMin,
+				MaxValue: configuredMax,
+			},
+		}
+
+		// Populate the severity field according to Thresholds
+		for _, t := range definition.GetInteger().GetThresholds() {
+			if value >= t.GetRange().GetMin() && value <= t.GetRange().GetMax() {
+				score.Severity = t.GetSeverity()
+				break
+			}
+		}
+
+	case *rpc.ScoreDefinition_Percent:
+		// Score proto expects float32 type
+		var value float32
+
+		// Convert scoreValue to appropriate type
+		// evaluateScoreExpression can return either a float or int value.
+		// Both are valid for an integer.
+		switch v := scoreValue.(type) {
+		case int64:
+			value = float32(v)
+		case float64:
+			value = float32(v)
+		default:
+			return nil, fmt.Errorf("failed typecheck for output: expected either int64 or float64 got %s (type: %T)", v, v)
+		}
+
+		// Check that the scoreValue is within min/max limits
+		if value < 0 {
+			return nil, fmt.Errorf("evaluated score value(%f) cannot be less than 0", value)
+		}
+		if value > 100 {
+			return nil, fmt.Errorf("evaluated score value(%f) cannot be greater than 100", value)
+		}
+
+		// Populate Value field in Score proto
+		score.Value = &rpc.Score_PercentValue{
+			PercentValue: &rpc.PercentValue{
+				Value: value,
+			},
+		}
+
+		// Populate the severity field according to Thresholds
+		for _, t := range definition.GetPercent().GetThresholds() {
+			if value >= float32(t.GetRange().GetMin()) && value <= float32(t.GetRange().GetMax()) {
+				score.Severity = t.GetSeverity()
+				break
+			}
+		}
+
+	case *rpc.ScoreDefinition_Boolean:
+		// Convert scoreValue to appropriate type
+		boolVal, ok := scoreValue.(bool)
+		if !ok {
+			return nil, fmt.Errorf("failed typecheck for output: expected bool")
+		}
+
+		var displayValue string
+		if t := definition.GetBoolean().GetDisplayTrue(); boolVal && t != "" {
+			displayValue = t
+		} else if f := definition.GetBoolean().GetDisplayFalse(); !boolVal && f != "" {
+			displayValue = f
+		} else {
+			displayValue = strconv.FormatBool(boolVal)
+		}
+
+		// Populate Value field in Score proto
+		score.Value = &rpc.Score_BooleanValue{
+			BooleanValue: &rpc.BooleanValue{
+				Value:        boolVal,
+				DisplayValue: displayValue,
+			},
+		}
+
+		// Populate the severity field according to Thresholds
+		for _, t := range definition.GetBoolean().GetThresholds() {
+			if t.Value == boolVal {
+				score.Severity = t.Severity
+			}
+		}
+	}
+
+	return score, nil
+}
+
+func uploadScore(ctx context.Context, client connection.Client, resource patterns.ResourceInstance, score *rpc.Score) error {
+	artifactBytes, err := proto.Marshal(score)
+	if err != nil {
+		return err
+	}
+	artifact := &rpc.Artifact{
+		Name:     fmt.Sprintf("%s/artifacts/%s", resource.ResourceName().String(), score.GetId()),
+		Contents: artifactBytes,
+		MimeType: patch.ScoreMimeType,
+	}
+	log.Debugf(ctx, "Uploading %s", artifact.GetName())
+	if err = core.SetArtifact(ctx, client, artifact); err != nil {
+		return fmt.Errorf("failed to save artifact %s: %s", artifact.GetName(), err)
+	}
+
+	return nil
+}


Test patch:
diff --git a/cmd/registry/scoring/expression_test.go b/cmd/registry/scoring/expression_test.go
new file mode 100644
index 0000000..52305de
--- /dev/null
+++ b/cmd/registry/scoring/expression_test.go
@@ -0,0 +1,267 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scoring
+
+import (
+	"testing"
+
+	"github.com/apigee/registry/rpc"
+	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/testing/protocmp"
+)
+
+func TestGetMap(t *testing.T) {
+	tests := []struct {
+		desc          string
+		contentsProto proto.Message
+		mimeType      string
+		wantMap       map[string]interface{}
+	}{
+		{
+			desc: "happy path rpc.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+			mimeType: "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			wantMap: map[string]interface{}{
+				"name": "openapi.yaml",
+				"files": []interface{}{
+					map[string]interface{}{
+						"filePath": "openapi.yaml",
+						"problems": []interface{}{
+							map[string]interface{}{
+								"message": "lint-error",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			contents, _ := proto.Marshal(test.contentsProto)
+
+			gotMap, gotErr := getMap(contents, test.mimeType)
+			if gotErr != nil {
+				t.Errorf("getMap() returned unexpected error: %s", gotErr)
+			}
+			opts := protocmp.Transform()
+			if !cmp.Equal(test.wantMap, gotMap, opts) {
+				t.Errorf("getMap returned unexpected response (-want +got):\n%s", cmp.Diff(test.wantMap, gotMap, opts))
+			}
+		})
+	}
+}
+
+func TestGetMapError(t *testing.T) {
+	tests := []struct {
+		desc          string
+		contentsProto proto.Message
+		mimeType      string
+	}{
+		{
+			desc: "unsupported artifact type",
+			contentsProto: &rpc.ScoreDefinition{
+				TargetResource: &rpc.ResourcePattern{},
+				Formula: &rpc.ScoreDefinition_ScoreFormula{
+					ScoreFormula: &rpc.ScoreFormula{
+						Artifact: &rpc.ResourcePattern{
+							Pattern: "$resource.spec/artifacts/lint-error",
+						},
+						ScoreExpression: "size(files[0].problems)",
+					},
+				},
+				Type: &rpc.ScoreDefinition_Integer{
+					Integer: &rpc.IntegerType{
+						MinValue: 0,
+						MaxValue: 10,
+					},
+				},
+			},
+			mimeType: "application/octet-stream;type=google.cloud.apigeeregistry.v1.scoring.ScoreDefinition",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			contents, _ := proto.Marshal(test.contentsProto)
+
+			_, gotErr := getMap(contents, test.mimeType)
+			if gotErr == nil {
+				t.Errorf("getMap(%v, %s) did not return an error", test.contentsProto, test.mimeType)
+			}
+		})
+	}
+}
+
+func TestEvaluateScoreExpression(t *testing.T) {
+	tests := []struct {
+		desc          string
+		expression    string
+		mimeType      string
+		contentsProto proto.Message
+		wantValue     interface{}
+	}{
+		{
+			desc:       "int happy path",
+			expression: "size(files[0].problems)",
+			mimeType:   "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+			wantValue: int64(1),
+		},
+		{
+			desc:       "double happy path",
+			expression: "double(size(files[0].problems))",
+			mimeType:   "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+			wantValue: float64(1),
+		},
+		{
+			desc:       "bool happy path",
+			expression: "size(files[0].problems)>0",
+			mimeType:   "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+			wantValue: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			contents, _ := proto.Marshal(test.contentsProto)
+
+			gotValue, gotErr := evaluateScoreExpression(test.expression, test.mimeType, contents)
+			if gotErr != nil {
+				t.Errorf("evaluateScoreExpression() returned unexpected error: %s", gotErr)
+			}
+
+			if test.wantValue != gotValue {
+				t.Errorf("evaluateScoreExpression() returned unexpected value, want: %v, got: %v", test.wantValue, gotValue)
+			}
+		})
+	}
+}
+
+func TestEvaluateScoreExpressionError(t *testing.T) {
+	tests := []struct {
+		desc          string
+		expression    string
+		mimeType      string
+		contentsProto proto.Message
+	}{
+		{
+			desc:       "error in getMap()",
+			expression: "",
+			// getMap() will return an error since ScoreDefinition is an unsupported type
+			mimeType:      "application/octet-stream;type=google.cloud.apigeeregistry.v1.scoring.ScoreDefinition",
+			contentsProto: &rpc.ScoreDefinition{},
+		},
+		{
+			desc:       "invalid field reference",
+			expression: "size(files.problems)", // correct expression should be "size(files[0].problems)"
+			mimeType:   "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:       "unsupported type (list)",
+			expression: "files[0].problems", //this will return a list value
+			mimeType:   "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint",
+			contentsProto: &rpc.Lint{
+				Name: "openapi.yaml",
+				Files: []*rpc.LintFile{
+					{
+						FilePath: "openapi.yaml",
+						Problems: []*rpc.LintProblem{
+							{
+								Message: "lint-error",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			contents, _ := proto.Marshal(test.contentsProto)
+
+			_, gotErr := evaluateScoreExpression(test.expression, test.mimeType, contents)
+			if gotErr == nil {
+				t.Errorf("evaluateScoreExpression(%s, %s, %v) did not return an error", test.expression, test.mimeType, test.contentsProto)
+			}
+		})
+	}
+}
diff --git a/cmd/registry/scoring/parser_test.go b/cmd/registry/scoring/parser_test.go
index d87cbfc..7b47949 100644
--- a/cmd/registry/scoring/parser_test.go
+++ b/cmd/registry/scoring/parser_test.go
@@ -1,3 +1,17 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package scoring
 
 import (
diff --git a/cmd/registry/scoring/score_test.go b/cmd/registry/scoring/score_test.go
new file mode 100644
index 0000000..335ebe7
--- /dev/null
+++ b/cmd/registry/scoring/score_test.go
@@ -0,0 +1,625 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scoring
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apigee/registry/cmd/registry/patterns"
+	"github.com/apigee/registry/connection"
+	"github.com/apigee/registry/rpc"
+	"github.com/apigee/registry/server/registry/names"
+	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/testing/protocmp"
+)
+
+var (
+	integerDefinition = &rpc.ScoreDefinition{
+		Id:             "lint-error",
+		Kind:           "ScoreDefinition",
+		DisplayName:    "Lint Error",
+		Description:    "Number of errors found by linter",
+		Uri:            "http://some/test/uri",
+		UriDisplayName: "Test URI",
+		Type: &rpc.ScoreDefinition_Integer{
+			Integer: &rpc.IntegerType{
+				MinValue: 0,
+				MaxValue: 10,
+				Thresholds: []*rpc.NumberThreshold{
+					{
+						Severity: rpc.Severity_OK,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 0,
+							Max: 3,
+						},
+					},
+					{
+						Severity: rpc.Severity_WARNING,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 4,
+							Max: 6,
+						},
+					},
+					{
+						Severity: rpc.Severity_ALERT,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 6,
+							Max: 10,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	percentDefinition = &rpc.ScoreDefinition{
+		Id:             "lint-error-percent",
+		Kind:           "ScoreDefinition",
+		DisplayName:    "Lint Error Percentage",
+		Description:    "Percentage errors found by linter",
+		Uri:            "http://some/test/uri",
+		UriDisplayName: "Test URI",
+		Type: &rpc.ScoreDefinition_Percent{
+			Percent: &rpc.PercentType{
+				Thresholds: []*rpc.NumberThreshold{
+					{
+						Severity: rpc.Severity_OK,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 0,
+							Max: 30,
+						},
+					},
+					{
+						Severity: rpc.Severity_WARNING,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 31,
+							Max: 60,
+						},
+					},
+					{
+						Severity: rpc.Severity_ALERT,
+						Range: &rpc.NumberThreshold_NumberRange{
+							Min: 61,
+							Max: 100,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	booleanDefinition = &rpc.ScoreDefinition{
+		Id:             "lint-approval",
+		Kind:           "ScoreDefinition",
+		DisplayName:    "Lint Approval",
+		Description:    "Approval by linter",
+		Uri:            "http://some/test/uri",
+		UriDisplayName: "Test URI",
+		Type: &rpc.ScoreDefinition_Boolean{
+			Boolean: &rpc.BooleanType{
+				DisplayTrue:  "Approved",
+				DisplayFalse: "Denied",
+				Thresholds: []*rpc.BooleanThreshold{
+					{
+						Severity: rpc.Severity_WARNING,
+						Value:    false,
+					},
+					{
+						Severity: rpc.Severity_OK,
+						Value:    true,
+					},
+				},
+			},
+		},
+	}
+)
+
+func TestMatchResourceWithTarget(t *testing.T) {
+	tests := []struct {
+		desc          string
+		targetPattern *rpc.ResourcePattern
+		resourceName  string
+		wantErr       bool
+	}{
+		{
+			desc: "spec pattern",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/-/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+		},
+		{
+			desc: "specific api match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/petstore/versions/-/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+		},
+		{
+			desc: "specific api no match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/test/versions/-/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+			wantErr:      true,
+		},
+		{
+			desc: "specific version match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/1.0.0/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+		},
+		{
+			desc: "specific version no match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/2.0.0/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+			wantErr:      true,
+		},
+		{
+			desc: "specific spec match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/-/specs/openapi.yaml",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+		},
+		{
+			desc: "specific spec no match",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/-/specs/swagger.yaml",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml",
+			wantErr:      true,
+		},
+		{
+			desc: "artifact pattern error",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/-/specs/-/artifacts/lint-spectral",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0/specs/openapi.yaml/artifacts/lint-spectral",
+			wantErr:      true,
+		},
+		{
+			desc: "target and resource mismatch",
+			targetPattern: &rpc.ResourcePattern{
+				Pattern: "apis/-/versions/-/specs/-",
+			},
+			resourceName: "projects/pattern-test/locations/global/apis/petstore/versions/1.0.0",
+			wantErr:      true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			resourceName, _ := patterns.ParseResourcePattern(test.resourceName)
+			gotErr := matchResourceWithTarget(test.targetPattern, resourceName, "projects/pattern-test/locations/global")
+			if test.wantErr && gotErr == nil {
+				t.Errorf("expected matchResourceWithTarget() to return errors")
+			}
+
+			if !test.wantErr && gotErr != nil {
+				t.Errorf("matchResourceWithTarget() returned unexpected error: %s", gotErr)
+			}
+		})
+	}
+}
+
+func TestProcessScoreFormula(t *testing.T) {
+	ctx := context.Background()
+	registryClient, err := connection.NewClient(ctx)
+	if err != nil {
+		t.Logf("Failed to create client: %+v", err)
+		t.FailNow()
+	}
+	defer registryClient.Close()
+	adminClient, err := connection.NewAdminClient(ctx)
+	if err != nil {
+		t.Logf("Failed to create client: %+v", err)
+		t.FailNow()
+	}
+	defer adminClient.Close()
+
+	//setup
+	deleteProject(ctx, adminClient, t, "score-formula-test")
+	createProject(ctx, adminClient, t, "score-formula-test")
+	createApi(ctx, registryClient, t, "projects/score-formula-test/locations/global", "petstore")
+	createVersion(ctx, registryClient, t, "projects/score-formula-test/locations/global/apis/petstore", "1.0.0")
+	createSpec(ctx, registryClient, t, "projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0", "openapi.yaml", gzipOpenAPIv3)
+	artifactBytes, _ := proto.Marshal(&rpc.Lint{
+		Name: "openapi.yaml",
+		Files: []*rpc.LintFile{
+			{
+				FilePath: "openapi.yaml",
+				Problems: []*rpc.LintProblem{
+					{
+						Message: "lint-error",
+					},
+				},
+			},
+		},
+	})
+	createUpdateArtifact(
+		ctx, registryClient, t,
+		"projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0/specs//openapi.yaml/artifacts/lint-spectral",
+		artifactBytes, "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint")
+
+	// arguments
+	formula := &rpc.ScoreFormula{
+		Artifact: &rpc.ResourcePattern{
+			Pattern: "$resource.spec/artifacts/lint-spectral",
+		},
+		ScoreExpression: "size(files[0].problems)",
+	}
+	resource := patterns.SpecResource{
+		SpecName: patterns.SpecName{
+			Name: names.Spec{
+				ProjectID: "score-formula-test",
+				ApiID:     "petstore",
+				VersionID: "1.0.0",
+				SpecID:    "openapi.yaml",
+			},
+		},
+	}
+	gotValue, gotErr := processScoreFormula(ctx, registryClient, formula, resource)
+	if gotErr != nil {
+		t.Errorf("processScoreFormula() returned unexpected error: %s", gotErr)
+	}
+
+	wantValue := int64(1)
+
+	if gotValue != nil && wantValue != gotValue {
+		t.Errorf("processScoreFormula() returned unexpected value, want: %v, got: %v", wantValue, gotValue)
+	}
+}
+
+func TestProcessScoreFormulaError(t *testing.T) {
+	tests := []struct {
+		desc     string
+		setup    func(context.Context, connection.Client, connection.AdminClient)
+		formula  *rpc.ScoreFormula
+		resource patterns.ResourceInstance
+	}{
+		{
+			desc: "invalid reference",
+			setup: func(ctx context.Context, client connection.Client, adminClient connection.AdminClient) {
+				deleteProject(ctx, adminClient, t, "score-formula-test")
+				createProject(ctx, adminClient, t, "score-formula-test")
+				createApi(ctx, client, t, "projects/score-formula-test/locations/global", "petstore")
+				createVersion(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore", "1.0.0")
+				createSpec(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0", "openapi.yaml", gzipOpenAPIv3)
+			},
+			formula: &rpc.ScoreFormula{
+				Artifact: &rpc.ResourcePattern{
+					Pattern: "$resource.specs/artifacts/lint-spectral", //error
+				},
+				ScoreExpression: "size(files[0].problems)",
+			},
+			resource: patterns.SpecResource{
+				SpecName: patterns.SpecName{
+					Name: names.Spec{
+						ProjectID: "score-formula-test",
+						ApiID:     "petstore",
+						VersionID: "1.0.0",
+						SpecID:    "openapi.yaml",
+					},
+				},
+			},
+		},
+		{
+			desc: "invalid extended pattern",
+			setup: func(ctx context.Context, client connection.Client, adminClient connection.AdminClient) {
+				deleteProject(ctx, adminClient, t, "score-formula-test")
+				createProject(ctx, adminClient, t, "score-formula-test")
+				createApi(ctx, client, t, "projects/score-formula-test/locations/global", "petstore")
+				createVersion(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore", "1.0.0")
+				createSpec(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0", "openapi.yaml", gzipOpenAPIv3)
+			},
+			formula: &rpc.ScoreFormula{
+				Artifact: &rpc.ResourcePattern{
+					Pattern: "$resource.spec/artifact/lint-spectral", // error
+				},
+				ScoreExpression: "size(files[0].problems)",
+			},
+			resource: patterns.SpecResource{
+				SpecName: patterns.SpecName{
+					Name: names.Spec{
+						ProjectID: "score-formula-test",
+						ApiID:     "petstore",
+						VersionID: "1.0.0",
+						SpecID:    "openapi.yaml",
+					},
+				},
+			},
+		},
+		{
+			desc: "missing artifact",
+			setup: func(ctx context.Context, client connection.Client, adminClient connection.AdminClient) {
+				deleteProject(ctx, adminClient, t, "score-formula-test")
+				createProject(ctx, adminClient, t, "score-formula-test")
+				createApi(ctx, client, t, "projects/score-formula-test/locations/global", "petstore")
+				createVersion(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore", "1.0.0")
+				createSpec(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0", "openapi.yaml", gzipOpenAPIv3)
+			},
+			formula: &rpc.ScoreFormula{
+				Artifact: &rpc.ResourcePattern{
+					Pattern: "$resource.spec/artifacts/lint-spectral",
+				},
+				ScoreExpression: "size(files[0].problems)",
+			},
+			resource: patterns.SpecResource{
+				SpecName: patterns.SpecName{
+					Name: names.Spec{
+						ProjectID: "score-formula-test",
+						ApiID:     "petstore",
+						VersionID: "1.0.0",
+						SpecID:    "openapi.yaml",
+					},
+				},
+			},
+		},
+		{
+			desc: "invalid expression",
+			setup: func(ctx context.Context, client connection.Client, adminClient connection.AdminClient) {
+				deleteProject(ctx, adminClient, t, "score-formula-test")
+				createProject(ctx, adminClient, t, "score-formula-test")
+				createApi(ctx, client, t, "projects/score-formula-test/locations/global", "petstore")
+				createVersion(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore", "1.0.0")
+				createSpec(ctx, client, t, "projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0", "openapi.yaml", gzipOpenAPIv3)
+				artifactBytes, _ := proto.Marshal(&rpc.Lint{
+					Name: "openapi.yaml",
+					Files: []*rpc.LintFile{
+						{
+							FilePath: "openapi.yaml",
+							Problems: []*rpc.LintProblem{
+								{
+									Message: "lint-error",
+								},
+							},
+						},
+					},
+				})
+				createUpdateArtifact(
+					ctx, client, t,
+					"projects/score-formula-test/locations/global/apis/petstore/versions/1.0.0/specs//openapi.yaml/artifacts/lint-spectral",
+					artifactBytes, "application/octet-stream;type=google.cloud.apigeeregistry.applications.v1alpha1.Lint")
+			},
+			formula: &rpc.ScoreFormula{
+				Artifact: &rpc.ResourcePattern{
+					Pattern: "$resource.spec/artifacts/lint-spectral",
+				},
+				ScoreExpression: "size(files[0].problem)", // invalid expression
+			},
+			resource: patterns.SpecResource{
+				SpecName: patterns.SpecName{
+					Name: names.Spec{
+						ProjectID: "score-formula-test",
+						ApiID:     "petstore",
+						VersionID: "1.0.0",
+						SpecID:    "openapi.yaml",
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ctx := context.Background()
+			registryClient, err := connection.NewClient(ctx)
+			if err != nil {
+				t.Logf("Failed to create client: %+v", err)
+				t.FailNow()
+			}
+			defer registryClient.Close()
+			adminClient, err := connection.NewAdminClient(ctx)
+			if err != nil {
+				t.Logf("Failed to create client: %+v", err)
+				t.FailNow()
+			}
+			defer adminClient.Close()
+
+			test.setup(ctx, registryClient, adminClient)
+
+			_, gotErr := processScoreFormula(ctx, registryClient, test.formula, test.resource)
+			if gotErr == nil {
+				t.Errorf("expected processScoreFormula() to return an error")
+			}
+		})
+	}
+}
+
+func TestProcessScoreType(t *testing.T) {
+	tests := []struct {
+		desc       string
+		definition *rpc.ScoreDefinition
+		scoreValue interface{}
+		wantScore  *rpc.Score
+	}{
+		{
+			desc:       "happy path integer",
+			definition: integerDefinition,
+			scoreValue: int64(1),
+			wantScore: &rpc.Score{
+				Id:             "score-lint-error",
+				Kind:           "Score",
+				DisplayName:    "Lint Error",
+				Description:    "Number of errors found by linter",
+				Uri:            "http://some/test/uri",
+				UriDisplayName: "Test URI",
+				DefinitionName: "projects/score-type-test/locations/global/artifacts/lint-error",
+				Severity:       rpc.Severity_OK,
+				Value: &rpc.Score_IntegerValue{
+					IntegerValue: &rpc.IntegerValue{
+						Value:    1,
+						MinValue: 0,
+						MaxValue: 10,
+					},
+				},
+			},
+		},
+		{
+			desc:       "happy path integer with float value",
+			definition: integerDefinition,
+			scoreValue: float64(1),
+			wantScore: &rpc.Score{
+				Id:             "score-lint-error",
+				Kind:           "Score",
+				DisplayName:    "Lint Error",
+				Description:    "Number of errors found by linter",
+				Uri:            "http://some/test/uri",
+				UriDisplayName: "Test URI",
+				DefinitionName: "projects/score-type-test/locations/global/artifacts/lint-error",
+				Severity:       rpc.Severity_OK,
+				Value: &rpc.Score_IntegerValue{
+					IntegerValue: &rpc.IntegerValue{
+						Value:    1,
+						MinValue: 0,
+						MaxValue: 10,
+					},
+				},
+			},
+		},
+		{
+			desc:       "happy path percent",
+			definition: percentDefinition,
+			scoreValue: float64(50),
+			wantScore: &rpc.Score{
+				Id:             "score-lint-error-percent",
+				Kind:           "Score",
+				DisplayName:    "Lint Error Percentage",
+				Description:    "Percentage errors found by linter",
+				Uri:            "http://some/test/uri",
+				UriDisplayName: "Test URI",
+				DefinitionName: "projects/score-type-test/locations/global/artifacts/lint-error-percent",
+				Severity:       rpc.Severity_WARNING,
+				Value: &rpc.Score_PercentValue{
+					PercentValue: &rpc.PercentValue{
+						Value: 50,
+					},
+				},
+			},
+		},
+		{
+			desc:       "happy path percent with integer value",
+			definition: percentDefinition,
+			scoreValue: int64(50),
+			wantScore: &rpc.Score{
+				Id:             "score-lint-error-percent",
+				Kind:           "Score",
+				DisplayName:    "Lint Error Percentage",
+				Description:    "Percentage errors found by linter",
+				Uri:            "http://some/test/uri",
+				UriDisplayName: "Test URI",
+				DefinitionName: "projects/score-type-test/locations/global/artifacts/lint-error-percent",
+				Severity:       rpc.Severity_WARNING,
+				Value: &rpc.Score_PercentValue{
+					PercentValue: &rpc.PercentValue{
+						Value: 50,
+					},
+				},
+			},
+		},
+		{
+			desc:       "happy path boolean",
+			definition: booleanDefinition,
+			scoreValue: true,
+			wantScore: &rpc.Score{
+				Id:             "score-lint-approval",
+				Kind:           "Score",
+				DisplayName:    "Lint Approval",
+				Description:    "Approval by linter",
+				Uri:            "http://some/test/uri",
+				UriDisplayName: "Test URI",
+				DefinitionName: "projects/score-type-test/locations/global/artifacts/lint-approval",
+				Severity:       rpc.Severity_OK,
+				Value: &rpc.Score_BooleanValue{
+					BooleanValue: &rpc.BooleanValue{
+						Value:        true,
+						DisplayValue: "Approved",
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotScore, gotErr := processScoreType(test.definition, test.scoreValue, "projects/score-type-test/locations/global")
+			if gotErr != nil {
+				t.Errorf("processScoreType() returned unexpected error: %s", gotErr)
+			}
+
+			opts := cmp.Options{protocmp.Transform()}
+			if !cmp.Equal(test.wantScore, gotScore, opts) {
+				t.Errorf("processScoreType() returned unexpected response (-want +got):\n%s", cmp.Diff(test.wantScore, gotScore, opts))
+			}
+		})
+	}
+}
+
+func TestProcessScoreTypeError(t *testing.T) {
+	tests := []struct {
+		desc       string
+		definition *rpc.ScoreDefinition
+		scoreValue interface{}
+	}{
+		{
+			desc:       "type mismatch integer",
+			definition: integerDefinition,
+			scoreValue: true,
+		},
+		{
+			desc:       "greater than max integer",
+			definition: integerDefinition,
+			scoreValue: 11,
+		},
+		{
+			desc:       "less than min integer",
+			definition: integerDefinition,
+			scoreValue: -1,
+		},
+		{
+			desc:       "type mismatch percent",
+			definition: percentDefinition,
+			scoreValue: false,
+		},
+		{
+			desc:       "greater than max percent",
+			definition: integerDefinition,
+			scoreValue: 101,
+		},
+		{
+			desc:       "less than min percent",
+			definition: integerDefinition,
+			scoreValue: -1,
+		},
+		{
+			desc:       "type mismatch boolean",
+			definition: booleanDefinition,
+			scoreValue: 1,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			_, gotErr := processScoreType(test.definition, test.scoreValue, "projects/score-type-test/locations/global")
+			if gotErr == nil {
+				t.Errorf("expected processScoreType() to return an error")
+			}
+		})
+	}
+}
diff --git a/cmd/registry/scoring/test_helpers.go b/cmd/registry/scoring/test_helpers.go
new file mode 100644
index 0000000..a93b86f
--- /dev/null
+++ b/cmd/registry/scoring/test_helpers.go
@@ -0,0 +1,148 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scoring
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/apigee/registry/cmd/registry/core"
+	"github.com/apigee/registry/connection"
+	"github.com/apigee/registry/rpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const gzipOpenAPIv3 = "application/x.openapi+gzip;version=3.0.0"
+
+func deleteProject(
+	ctx context.Context,
+	client connection.AdminClient,
+	t *testing.T,
+	projectID string) {
+	t.Helper()
+	req := &rpc.DeleteProjectRequest{
+		Name:  "projects/" + projectID,
+		Force: true,
+	}
+	err := client.DeleteProject(ctx, req)
+	if err != nil && status.Code(err) != codes.NotFound {
+		t.Fatalf("Failed DeleteProject(%v): %s", req, err.Error())
+	}
+}
+
+func createProject(
+	ctx context.Context,
+	client connection.AdminClient,
+	t *testing.T,
+	projectID string) {
+	t.Helper()
+	req := &rpc.CreateProjectRequest{
+		ProjectId: projectID,
+		Project: &rpc.Project{
+			DisplayName: "Demo",
+			Description: "A demo catalog",
+		},
+	}
+	project, err := client.CreateProject(ctx, req)
+	if err != nil {
+		t.Fatalf("Failed CreateProject(%v): %s", req, err.Error())
+	}
+	if project.GetName() != fmt.Sprintf("projects/%s", projectID) {
+		t.Fatalf("Invalid project name %s", project.GetName())
+	}
+}
+
+func createApi(
+	ctx context.Context,
+	client connection.Client,
+	t *testing.T,
+	parent, apiID string) {
+	t.Helper()
+	req := &rpc.CreateApiRequest{
+		Parent: parent,
+		ApiId:  apiID,
+		Api: &rpc.Api{
+			DisplayName:  apiID,
+			Description:  fmt.Sprintf("Sample Test API: %s", apiID),
+			Availability: "GENERAL",
+		},
+	}
+	_, err := client.CreateApi(ctx, req)
+	if err != nil {
+		t.Fatalf("Failed CreateApi(%v): %s", req, err.Error())
+	}
+}
+
+func createVersion(
+	ctx context.Context,
+	client connection.Client,
+	t *testing.T,
+	parent, versionID string) {
+	t.Helper()
+	req := &rpc.CreateApiVersionRequest{
+		Parent:       parent,
+		ApiVersionId: versionID,
+		ApiVersion:   &rpc.ApiVersion{},
+	}
+	_, err := client.CreateApiVersion(ctx, req)
+	if err != nil {
+		t.Fatalf("Failed CreateApiVersion(%v): %s", req, err.Error())
+	}
+}
+
+func createSpec(
+	ctx context.Context,
+	client connection.Client,
+	t *testing.T,
+	parent string,
+	specId string,
+	mimeType string,
+) {
+	t.Helper()
+	// Create a spec entry with empty data
+	req := &rpc.CreateApiSpecRequest{
+		Parent:    parent,
+		ApiSpecId: specId,
+		ApiSpec: &rpc.ApiSpec{
+			MimeType: mimeType,
+		},
+	}
+	_, err := client.CreateApiSpec(ctx, req)
+	if err != nil {
+		t.Fatalf("Failed CreateApiSpec(%v): %s", req, err.Error())
+	}
+}
+
+func createUpdateArtifact(
+	ctx context.Context,
+	client connection.Client,
+	t *testing.T,
+	artifactName string,
+	data []byte,
+	mimeType string) {
+	t.Helper()
+	// Creates an artifact entry with empty data
+	artifact := &rpc.Artifact{
+		Name:     artifactName,
+		Contents: data,
+		MimeType: mimeType,
+	}
+	err := core.SetArtifact(ctx, client, artifact)
+	if err != nil {
+		t.Fatalf("Failed SetArtifact(%v): %s", artifact, err.Error())
+	}
+}