Repository: ossf/package-feeds (go)

Issue:
Consume PyPI updates from the XML-RPC mirroring api instead of RSS
PyPI updates can sometimes be missed.

To avoid this from happening the PyPI Mirroring API allows all changes to the PyPI package repository to be listed.

See: https://warehouse.pypa.io/api-reference/xml-rpc.html#mirroring-support

Pull request description:
Add PyPI XML-RPC feed
Fixes #172 

This PR adds a new feed called `pypi-artifacts` that collects data from the [PyPI XML-RPC feed](https://warehouse.pypa.io/api-reference/xml-rpc.html#mirroring-support) rather than RSS. 

While this feed is officially deprecated, the current RSS feed has some problems with missing data when package archives are uploaded after the initial release, and the XML-RPC remains the only other fully supported way to reliably get this data.

Furthermore, a new optional field named `artifact_id` is added to the JSON schema; this allows identifying packages at the level of specific artifacts rather than just by the ecosystem/name/version triple. 

Patch:
diff --git a/go.mod b/go.mod
index 8d3ced6..6654f3e 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/ossf/package-feeds
 go 1.15
 
 require (
+	github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/robfig/cron v1.2.0
 	github.com/sirupsen/logrus v1.8.1
diff --git a/go.sum b/go.sum
index 788660a..a889455 100644
--- a/go.sum
+++ b/go.sum
@@ -238,6 +238,8 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
 github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
diff --git a/package.schema.json b/package.schema.json
index 7b84398..a88e24d 100644
--- a/package.schema.json
+++ b/package.schema.json
@@ -26,6 +26,11 @@
         "description": "The type of package, this being the `FeedName` of the given package feed",
         "examples": ["pypi", "npm", "crates", "goproxy"]
       },
+      "artifact_id": {
+        "type": "string",
+        "description": "Identifies a particular downloadable artifact for this package version. No particular format is guaranteed; e.g. names may indicate platform-specific variants or include commit hashes.",
+        "examples": ["factor_reader-1.0.3.tar.gz", "micrograd2023-0.0.3-py3-none-any.whl", "odoo14_addon_l10n_es_aeat-14.0.3.0.2.dev1-py3-none-any.whl"]
+      },
       "schema_ver": {
         "type": "string",
         "pattern":  "^[1-9][0-9]*\\.[0-9]+",
diff --git a/pkg/feeds/feed.go b/pkg/feeds/feed.go
index 3719b42..3018e1c 100644
--- a/pkg/feeds/feed.go
+++ b/pkg/feeds/feed.go
@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-const schemaVer = "1.0"
+const schemaVer = "1.1"
 
 var ErrNoPackagesPolled = errors.New("no packages were successfully polled")
 
@@ -37,6 +37,7 @@ type Package struct {
 	Version     string    `json:"version"`
 	CreatedDate time.Time `json:"created_date"`
 	Type        string    `json:"type"`
+	ArtifactID  string    `json:"artifact_id"`
 	SchemaVer   string    `json:"schema_ver"`
 }
 
@@ -49,12 +50,19 @@ func (err PackagePollError) Error() string {
 	return fmt.Sprintf("Polling for package %s returned error: %v", err.Name, err.Err)
 }
 
+// NewPackage creates a Package object without the artifact ID field populated.
 func NewPackage(created time.Time, name, version, feed string) *Package {
+	return NewArtifact(created, name, version, "", feed)
+}
+
+// NewArtifact creates a Package object with the artifact ID field populated.
+func NewArtifact(created time.Time, name, version, artifactID, feed string) *Package {
 	return &Package{
 		Name:        name,
 		Version:     version,
 		CreatedDate: created,
 		Type:        feed,
+		ArtifactID:  artifactID,
 		SchemaVer:   schemaVer,
 	}
 }
diff --git a/pkg/feeds/pypi/README.md b/pkg/feeds/pypi/README.md
index 6ffba5a..ba08a44 100644
--- a/pkg/feeds/pypi/README.md
+++ b/pkg/feeds/pypi/README.md
@@ -1,11 +1,12 @@
-# Pypi Feed
+# PyPI Feed
 
-This feed allows polling of package updates from the pypi package repository.
+This feed allows polling of package updates from the PyPI package repository.
 
 ## Configuration options
 
-The `packages` Field can be supplied to the pypi feed options to enable polling of package specific apis. This is less effective
-with large lists of packages as it polls the RSS feed for each package individually but it is much less likely to miss package updates between polling.
+The `packages` Field can be supplied to the PyPI feed options to enable polling of package specific apis.
+This is less effective with large lists of packages as it polls the RSS feed for each package individually,
+but it is much less likely to miss package updates between polling.
 
 
 ```
@@ -15,4 +16,22 @@ feeds:
     packages:
     - numpy
     - scipy
-```
\ No newline at end of file
+```
+
+# PyPI Artifacts Feed
+
+This feed allows polling of PyPI package updates using the
+[XML-RPC feed](https://warehouse.pypa.io/api-reference/xml-rpc.html#mirroring-support).
+This feed contains extra information compared to the other PyPI feed in this project.
+In particular, this avoids missing upstream notifications when platform-specific archives are
+uploaded for a package some time after the release was made. Furthermore, the `artifact_id`
+field in the output JSON schema is populated with the filename of the release tarball.
+
+
+## Configuration
+
+No configuration; all package updates are monitored. In particular, `FeedOptions.Packages` is not supported
+```
+feeds:
+- type: pypi-v2
+```
diff --git a/pkg/feeds/pypi/pypi_artifacts.go b/pkg/feeds/pypi/pypi_artifacts.go
new file mode 100644
index 0000000..79ea050
--- /dev/null
+++ b/pkg/feeds/pypi/pypi_artifacts.go
@@ -0,0 +1,144 @@
+package pypi
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/kolo/xmlrpc"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+)
+
+const (
+	ArtifactFeedName = "pypi-artifacts"
+)
+
+// We care about changelog entries where the action is 'add X file <filename>'.
+var archiveUploadAction = regexp.MustCompile("add (.*) file (.*)")
+
+type ArtifactFeed struct {
+	baseURL string
+	options feeds.FeedOptions
+}
+
+func NewArtifactFeed(feedOptions feeds.FeedOptions) (*ArtifactFeed, error) {
+	return &ArtifactFeed{
+		baseURL: "https://pypi.org/pypi",
+		options: feedOptions,
+	}, nil
+}
+
+func (feed ArtifactFeed) Latest(cutoff time.Time) ([]*feeds.Package, []error) {
+	client, err := xmlrpc.NewClient(feed.baseURL, nil)
+	if err != nil {
+		return nil, []error{err}
+	}
+
+	changelogEntries, err := getPyPIChangeLog(client, cutoff)
+	if err != nil {
+		return nil, []error{err}
+	}
+
+	return getUploadedArtifacts(changelogEntries), nil
+}
+
+func (feed ArtifactFeed) GetFeedOptions() feeds.FeedOptions {
+	return feed.options
+}
+
+func (feed ArtifactFeed) GetName() string {
+	return ArtifactFeedName
+}
+
+type pypiChangelogEntry struct {
+	Name        string
+	Version     string
+	Timestamp   time.Time
+	Action      string
+	ArchiveName string
+}
+
+func (e *pypiChangelogEntry) isArchiveUpload() bool {
+	return e.ArchiveName != ""
+}
+
+func (e *pypiChangelogEntry) String() string {
+	return fmt.Sprintf("%s (%s): %s ts=%s", e.Name, e.Version, e.Action, e.Timestamp)
+}
+
+// getPyPIChangeLog returns a list of PyPI changelog entries since the given timestamp
+// defined by https://warehouse.pypa.io/api-reference/xml-rpc.html#changelog-since-with-ids-false
+func getPyPIChangeLog(client *xmlrpc.Client, since time.Time) ([]pypiChangelogEntry, error) {
+	// Raw result structure is array[array[string, string|nil, int64, string (, int64 if with_ids=true) ]]
+	// which cannot be represented in Go (struct mapping is not supported by library)
+	var result [][]interface{}
+	if err := client.Call("changelog", []interface{}{since.Unix(), false}, &result); err != nil {
+		return nil, err
+	}
+
+	return processRawChangelog(result), nil
+}
+
+func processRawChangelog(apiResult [][]interface{}) []pypiChangelogEntry {
+	changelogEntries := make([]pypiChangelogEntry, len(apiResult))
+	for i, r := range apiResult {
+		changelogEntries[i] = processRawChangelogItem(r)
+	}
+
+	return changelogEntries
+}
+
+func processRawChangelogItem(data []interface{}) pypiChangelogEntry {
+	/*
+		Each item of the changelog contains the following fields:
+		name: string
+		version: string (nullable)
+		timestamp: int64
+		action: string
+	*/
+	name, ok := data[0].(string)
+	if !ok {
+		name = ""
+	}
+	version, ok := data[1].(string)
+	if !ok {
+		version = ""
+	}
+	unixTimestamp, ok := data[2].(int64)
+	if !ok {
+		unixTimestamp = 0
+	}
+	action, ok := data[3].(string)
+	if !ok {
+		action = ""
+	}
+
+	archiveName := ""
+	// Changelog entries corresponding to new archives being added have an action string
+	// that looks like 'add <archive type> file <archive name>'. This code is generated by
+	// github.com/pypi/warehouse/blob/3bfd3e0d32e7396582d9635a316f1a47e407304d/warehouse/forklift/legacy.py#L1355
+	if match := archiveUploadAction.FindStringSubmatch(action); match != nil {
+		// it's a new archive!
+		archiveName = match[2]
+	}
+
+	return pypiChangelogEntry{
+		Name:        name,
+		Version:     version,
+		Timestamp:   time.Unix(unixTimestamp, 0),
+		Action:      action,
+		ArchiveName: archiveName,
+	}
+}
+
+func getUploadedArtifacts(changelogEntries []pypiChangelogEntry) []*feeds.Package {
+	var pkgs []*feeds.Package
+	for _, e := range changelogEntries {
+		if e.isArchiveUpload() {
+			pkgs = append(pkgs, feeds.NewArtifact(e.Timestamp, e.Name, e.Version, e.ArchiveName, ArtifactFeedName))
+		}
+	}
+
+	return pkgs
+}


Test patch:
diff --git a/pkg/feeds/pypi/pypi_artifacts_test.go b/pkg/feeds/pypi/pypi_artifacts_test.go
new file mode 100644
index 0000000..3ff8a48
--- /dev/null
+++ b/pkg/feeds/pypi/pypi_artifacts_test.go
@@ -0,0 +1,619 @@
+package pypi
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/ossf/package-feeds/pkg/feeds"
+)
+
+// testAPIData is a sample of PyPI changelog raw data (as a Go struct).
+var testAPIData = [][]interface{}{
+	{
+		"supertemplater",
+		"1.4.0",
+		int64(1678414652),
+		"new release",
+	},
+	{
+		"supertemplater",
+		"1.4.0",
+		int64(1678414652),
+		"add py3 file supertemplater-1.4.0-py3-none-any.whl",
+	},
+	{
+		"supertemplater",
+		"1.4.0",
+		int64(1678414654),
+		"add source file supertemplater-1.4.0.tar.gz",
+	},
+	{
+		"OpenVisus",
+		"2.2.96",
+		int64(1678414663),
+		"add cp310 file OpenVisus-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+	},
+	{
+		"OpenVisusNoGui",
+		"2.2.96",
+		int64(1678414694),
+		"add cp310 file OpenVisusNoGui-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+	},
+	{
+		"benchling-api-client",
+		"2.0.118",
+		int64(1678414736),
+		"new release",
+	},
+	{
+		"benchling-api-client",
+		"2.0.118",
+		int64(1678414736),
+		"add py3 file benchling_api_client-2.0.118-py3-none-any.whl",
+	},
+	{
+		"benchling-api-client",
+		"2.0.118",
+		int64(1678414738),
+		"add source file benchling_api_client-2.0.118.tar.gz",
+	},
+	{
+		"adbutils",
+		"1.2.9",
+		int64(1678415159),
+		"new release",
+	},
+	{
+		"adbutils",
+		"1.2.9",
+		int64(1678415161),
+		"add py3 file adbutils-1.2.9-py3-none-manylinux1_x86_64.whl",
+	},
+	{
+		"adbutils",
+		"1.2.9",
+		int64(1678415164),
+		"add py3 file adbutils-1.2.9-py3-none-win32.whl",
+	},
+	{
+		"adbutils",
+		"1.2.9",
+		int64(1678415166),
+		"add py3 file adbutils-1.2.9-py3-none-win_amd64.whl",
+	},
+	{
+		"adbutils",
+		"1.2.9",
+		int64(1678415167),
+		"add source file adbutils-1.2.9.tar.gz",
+	},
+	{
+		"qiskit-qasm2",
+		"0.5.1",
+		int64(1678415182),
+		"new release",
+	},
+	{
+		"qiskit-qasm2",
+		"0.5.1",
+		int64(1678415182),
+		"add source file qiskit_qasm2-0.5.1.tar.gz",
+	},
+	{
+		"dsp-py",
+		nil,
+		int64(1678415258),
+		"remove project",
+	},
+	{
+		"genai",
+		"0.12.0a0",
+		int64(1678415278),
+		"new release",
+	},
+	{
+		"genai",
+		"0.12.0a0",
+		int64(1678415278),
+		"add py3 file genai-0.12.0a0-py3-none-any.whl",
+	},
+	{
+		"genai",
+		"0.12.0a0",
+		int64(1678415281),
+		"add source file genai-0.12.0a0.tar.gz",
+	},
+	{
+		"chia-blockchain",
+		"1.7.1rc1",
+		int64(1678415319),
+		"new release",
+	},
+	{
+		"chia-blockchain",
+		"1.7.1rc1",
+		int64(1678415319),
+		"add source file chia-blockchain-1.7.1rc1.tar.gz",
+	},
+	{
+		"ScraperFC",
+		"2.6.3",
+		int64(1678415386),
+		"new release",
+	},
+	{
+		"ScraperFC",
+		"2.6.3",
+		int64(1678415386),
+		"add py3 file ScraperFC-2.6.3-py3-none-any.whl",
+	},
+	{
+		"ScraperFC",
+		"2.6.3",
+		int64(1678415389),
+		"add source file ScraperFC-2.6.3.tar.gz",
+	},
+	{
+		"callpyfile",
+		nil,
+		int64(1678415402),
+		"create",
+	},
+	{
+		"callpyfile",
+		nil,
+		int64(1678415402),
+		"add Owner hansalemao",
+	},
+	{
+		"callpyfile",
+		"0.10",
+		int64(1678415402),
+		"new release",
+	},
+	{
+		"callpyfile",
+		"0.10",
+		int64(1678415402),
+		"add py3 file callpyfile-0.10-py3-none-any.whl",
+	},
+	{
+		"callpyfile",
+		"0.10",
+		int64(1678415403),
+		"add source file callpyfile-0.10.tar.gz",
+	},
+}
+
+// testChangelogEntries is a sample of PyPI changelog data after processing, corresponding to the above raw data.
+var testChangelogEntries = []pypiChangelogEntry{
+	{
+		Name:        "supertemplater",
+		Version:     "1.4.0",
+		Timestamp:   time.Unix(1678414652, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "supertemplater",
+		Version:     "1.4.0",
+		Timestamp:   time.Unix(1678414652, 0),
+		Action:      "add py3 file supertemplater-1.4.0-py3-none-any.whl",
+		ArchiveName: "supertemplater-1.4.0-py3-none-any.whl",
+	},
+	{
+		Name:        "supertemplater",
+		Version:     "1.4.0",
+		Timestamp:   time.Unix(1678414654, 0),
+		Action:      "add source file supertemplater-1.4.0.tar.gz",
+		ArchiveName: "supertemplater-1.4.0.tar.gz",
+	},
+	{
+		Name:        "OpenVisus",
+		Version:     "2.2.96",
+		Timestamp:   time.Unix(1678414663, 0),
+		Action:      "add cp310 file OpenVisus-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+		ArchiveName: "OpenVisus-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+	},
+	{
+		Name:        "OpenVisusNoGui",
+		Version:     "2.2.96",
+		Timestamp:   time.Unix(1678414694, 0),
+		Action:      "add cp310 file OpenVisusNoGui-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+		ArchiveName: "OpenVisusNoGui-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+	},
+	{
+		Name:        "benchling-api-client",
+		Version:     "2.0.118",
+		Timestamp:   time.Unix(1678414736, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "benchling-api-client",
+		Version:     "2.0.118",
+		Timestamp:   time.Unix(1678414736, 0),
+		Action:      "add py3 file benchling_api_client-2.0.118-py3-none-any.whl",
+		ArchiveName: "benchling_api_client-2.0.118-py3-none-any.whl",
+	},
+	{
+		Name:        "benchling-api-client",
+		Version:     "2.0.118",
+		Timestamp:   time.Unix(1678414738, 0),
+		Action:      "add source file benchling_api_client-2.0.118.tar.gz",
+		ArchiveName: "benchling_api_client-2.0.118.tar.gz",
+	},
+	{
+		Name:        "adbutils",
+		Version:     "1.2.9",
+		Timestamp:   time.Unix(1678415159, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "adbutils",
+		Version:     "1.2.9",
+		Timestamp:   time.Unix(1678415161, 0),
+		Action:      "add py3 file adbutils-1.2.9-py3-none-manylinux1_x86_64.whl",
+		ArchiveName: "adbutils-1.2.9-py3-none-manylinux1_x86_64.whl",
+	},
+	{
+		Name:        "adbutils",
+		Version:     "1.2.9",
+		Timestamp:   time.Unix(1678415164, 0),
+		Action:      "add py3 file adbutils-1.2.9-py3-none-win32.whl",
+		ArchiveName: "adbutils-1.2.9-py3-none-win32.whl",
+	},
+	{
+		Name:        "adbutils",
+		Version:     "1.2.9",
+		Timestamp:   time.Unix(1678415166, 0),
+		Action:      "add py3 file adbutils-1.2.9-py3-none-win_amd64.whl",
+		ArchiveName: "adbutils-1.2.9-py3-none-win_amd64.whl",
+	},
+	{
+		Name:        "adbutils",
+		Version:     "1.2.9",
+		Timestamp:   time.Unix(1678415167, 0),
+		Action:      "add source file adbutils-1.2.9.tar.gz",
+		ArchiveName: "adbutils-1.2.9.tar.gz",
+	},
+	{
+		Name:        "qiskit-qasm2",
+		Version:     "0.5.1",
+		Timestamp:   time.Unix(1678415182, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "qiskit-qasm2",
+		Version:     "0.5.1",
+		Timestamp:   time.Unix(1678415182, 0),
+		Action:      "add source file qiskit_qasm2-0.5.1.tar.gz",
+		ArchiveName: "qiskit_qasm2-0.5.1.tar.gz",
+	},
+	{
+		Name:        "dsp-py",
+		Version:     "",
+		Timestamp:   time.Unix(1678415258, 0),
+		Action:      "remove project",
+		ArchiveName: "",
+	},
+	{
+		Name:        "genai",
+		Version:     "0.12.0a0",
+		Timestamp:   time.Unix(1678415278, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "genai",
+		Version:     "0.12.0a0",
+		Timestamp:   time.Unix(1678415278, 0),
+		Action:      "add py3 file genai-0.12.0a0-py3-none-any.whl",
+		ArchiveName: "genai-0.12.0a0-py3-none-any.whl",
+	},
+	{
+		Name:        "genai",
+		Version:     "0.12.0a0",
+		Timestamp:   time.Unix(1678415281, 0),
+		Action:      "add source file genai-0.12.0a0.tar.gz",
+		ArchiveName: "genai-0.12.0a0.tar.gz",
+	},
+	{
+		Name:        "chia-blockchain",
+		Version:     "1.7.1rc1",
+		Timestamp:   time.Unix(1678415319, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "chia-blockchain",
+		Version:     "1.7.1rc1",
+		Timestamp:   time.Unix(1678415319, 0),
+		Action:      "add source file chia-blockchain-1.7.1rc1.tar.gz",
+		ArchiveName: "chia-blockchain-1.7.1rc1.tar.gz",
+	},
+	{
+		Name:        "ScraperFC",
+		Version:     "2.6.3",
+		Timestamp:   time.Unix(1678415386, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "ScraperFC",
+		Version:     "2.6.3",
+		Timestamp:   time.Unix(1678415386, 0),
+		Action:      "add py3 file ScraperFC-2.6.3-py3-none-any.whl",
+		ArchiveName: "ScraperFC-2.6.3-py3-none-any.whl",
+	},
+	{
+		Name:        "ScraperFC",
+		Version:     "2.6.3",
+		Timestamp:   time.Unix(1678415389, 0),
+		Action:      "add source file ScraperFC-2.6.3.tar.gz",
+		ArchiveName: "ScraperFC-2.6.3.tar.gz",
+	},
+	{
+		Name:        "callpyfile",
+		Version:     "",
+		Timestamp:   time.Unix(1678415402, 0),
+		Action:      "create",
+		ArchiveName: "",
+	},
+	{
+		Name:        "callpyfile",
+		Version:     "",
+		Timestamp:   time.Unix(1678415402, 0),
+		Action:      "add Owner hansalemao",
+		ArchiveName: "",
+	},
+	{
+		Name:        "callpyfile",
+		Version:     "0.10",
+		Timestamp:   time.Unix(1678415402, 0),
+		Action:      "new release",
+		ArchiveName: "",
+	},
+	{
+		Name:        "callpyfile",
+		Version:     "0.10",
+		Timestamp:   time.Unix(1678415402, 0),
+		Action:      "add py3 file callpyfile-0.10-py3-none-any.whl",
+		ArchiveName: "callpyfile-0.10-py3-none-any.whl",
+	},
+	{
+		Name:        "callpyfile",
+		Version:     "0.10",
+		Timestamp:   time.Unix(1678415403, 0),
+		Action:      "add source file callpyfile-0.10.tar.gz",
+		ArchiveName: "callpyfile-0.10.tar.gz",
+	},
+}
+
+// TestPyPIArtifactsLive is a live test of the XML-RPC call with basic sanity checks on the final output data.
+func TestPyPIArtifactsLive(t *testing.T) {
+	t.Parallel()
+
+	feed, err := NewArtifactFeed(feeds.FeedOptions{})
+	if err != nil {
+		t.Fatalf("Failed to create new pypi-artifacts feed: %v", err)
+	}
+
+	cutoff := time.Now().AddDate(0, 0, -1)
+	pkgs, errs := feed.Latest(cutoff)
+	if len(errs) != 0 {
+		t.Fatalf("feed.Latest returned error: %v", err)
+	}
+
+	for _, p := range pkgs {
+		if p.Name == "" {
+			t.Errorf("Package has no name: %s", p)
+		}
+		if p.Version == "" {
+			t.Errorf("Package has no version: %s", p)
+		}
+		if p.ArtifactID == "" {
+			t.Errorf("Package has no artifact ID: %s", p)
+		}
+		if p.CreatedDate.Unix() < cutoff.Unix() {
+			t.Errorf("Package create date (%s) is before cutoff (%s)", p.CreatedDate, cutoff)
+		}
+
+		if p.Type != ArtifactFeedName {
+			t.Errorf("Feed type not set correctly in pypi package following Latest()")
+		}
+	}
+}
+
+func TestProcessRawChangelog(t *testing.T) {
+	t.Parallel()
+
+	expectedResults := testChangelogEntries
+	actualResults := processRawChangelog(testAPIData)
+
+	if len(actualResults) != len(expectedResults) {
+		t.Errorf("expected %d changelog entries, got %d", len(expectedResults), len(actualResults))
+	}
+
+	for i, e := range expectedResults {
+		if i < len(actualResults) {
+			a := actualResults[i]
+			if a != e {
+				t.Errorf("Mismatch between expectedResult[%d] and actualResult[%d]: want %s, got %s", i, i, e, a)
+			}
+		} else {
+			t.Errorf("expectedResult[%d] with value %s but got no actual result with this index", i, e)
+		}
+	}
+}
+
+func TestCachedLatest(t *testing.T) {
+	t.Parallel()
+
+	expectedResults := []*feeds.Package{
+		{
+			Name:        "supertemplater",
+			Version:     "1.4.0",
+			CreatedDate: time.Unix(1678414652, 0),
+			ArtifactID:  "supertemplater-1.4.0-py3-none-any.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "supertemplater",
+			Version:     "1.4.0",
+			CreatedDate: time.Unix(1678414654, 0),
+			ArtifactID:  "supertemplater-1.4.0.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "OpenVisus",
+			Version:     "2.2.96",
+			CreatedDate: time.Unix(1678414663, 0),
+			ArtifactID:  "OpenVisus-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "OpenVisusNoGui",
+			Version:     "2.2.96",
+			CreatedDate: time.Unix(1678414694, 0),
+			ArtifactID:  "OpenVisusNoGui-2.2.96-cp310-none-macosx_10_9_x86_64.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "benchling-api-client",
+			Version:     "2.0.118",
+			CreatedDate: time.Unix(1678414736, 0),
+			ArtifactID:  "benchling_api_client-2.0.118-py3-none-any.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "benchling-api-client",
+			Version:     "2.0.118",
+			CreatedDate: time.Unix(1678414738, 0),
+			ArtifactID:  "benchling_api_client-2.0.118.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "adbutils",
+			Version:     "1.2.9",
+			CreatedDate: time.Unix(1678415161, 0),
+			ArtifactID:  "adbutils-1.2.9-py3-none-manylinux1_x86_64.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "adbutils",
+			Version:     "1.2.9",
+			CreatedDate: time.Unix(1678415164, 0),
+			ArtifactID:  "adbutils-1.2.9-py3-none-win32.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "adbutils",
+			Version:     "1.2.9",
+			CreatedDate: time.Unix(1678415166, 0),
+			ArtifactID:  "adbutils-1.2.9-py3-none-win_amd64.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "adbutils",
+			Version:     "1.2.9",
+			CreatedDate: time.Unix(1678415167, 0),
+			ArtifactID:  "adbutils-1.2.9.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "qiskit-qasm2",
+			Version:     "0.5.1",
+			CreatedDate: time.Unix(1678415182, 0),
+			ArtifactID:  "qiskit_qasm2-0.5.1.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "genai",
+			Version:     "0.12.0a0",
+			CreatedDate: time.Unix(1678415278, 0),
+			ArtifactID:  "genai-0.12.0a0-py3-none-any.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "genai",
+			Version:     "0.12.0a0",
+			CreatedDate: time.Unix(1678415281, 0),
+			ArtifactID:  "genai-0.12.0a0.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "chia-blockchain",
+			Version:     "1.7.1rc1",
+			CreatedDate: time.Unix(1678415319, 0),
+			ArtifactID:  "chia-blockchain-1.7.1rc1.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "ScraperFC",
+			Version:     "2.6.3",
+			CreatedDate: time.Unix(1678415386, 0),
+			ArtifactID:  "ScraperFC-2.6.3-py3-none-any.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "ScraperFC",
+			Version:     "2.6.3",
+			CreatedDate: time.Unix(1678415389, 0),
+			ArtifactID:  "ScraperFC-2.6.3.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "callpyfile",
+			Version:     "0.10",
+			CreatedDate: time.Unix(1678415402, 0),
+			ArtifactID:  "callpyfile-0.10-py3-none-any.whl",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+		{
+			Name:        "callpyfile",
+			Version:     "0.10",
+			CreatedDate: time.Unix(1678415403, 0),
+			ArtifactID:  "callpyfile-0.10.tar.gz",
+			Type:        ArtifactFeedName,
+			SchemaVer:   "1.1",
+		},
+	}
+
+	actualResults := getUploadedArtifacts(testChangelogEntries)
+
+	if len(actualResults) != len(expectedResults) {
+		t.Errorf("expected %d changelog entries, got %d", len(expectedResults), len(actualResults))
+	}
+
+	for i, e := range expectedResults {
+		if i < len(actualResults) {
+			a := actualResults[i]
+			if !reflect.DeepEqual(a, e) {
+				t.Errorf("Mismatch between expectedResult[%d] and actualResult[%d]: want %s, got %s", i, i, e, a)
+			}
+		} else {
+			t.Errorf("expectedResult[%d] with value %s but got no actual result with this index", i, e)
+		}
+	}
+}