func (s *StarManager) SaveStarredRepository(
	star *github.StarredRepository, wg *sync.WaitGroup,
) error {
	defer wg.Done()
	err := s.db.Save(&Star{
		PushedAt:    star.GetRepository().GetPushedAt().Time,
		StarredAt:   star.StarredAt.Time,
		URL:         star.GetRepository().GetHTMLURL(),
		Language:    strings.ToLower(star.GetRepository().GetLanguage()),
		Stargazers:  star.GetRepository().GetStargazersCount(),
		Description: star.GetRepository().GetDescription(),
		Topics:      star.GetRepository().Topics,
		Archived:    star.GetRepository().GetArchived(),
	})
	if err != nil {
		return err
	}
	log.Infof(
		"Saved %s (with topics %s)\n",
		star.GetRepository().GetHTMLURL(),
		star.GetRepository().Topics,
	)
	return nil
}
SaveStarredRepository saves a single starred repository to the local cache.
package starmanager
import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"
	"github.com/hashicorp/go-multierror"
	"github.com/asdine/storm"
	"github.com/asdine/storm/q"
	"github.com/gkze/stars/auth"
	"github.com/gkze/stars/utils"
	"github.com/google/go-github/v25/github"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/afero"
	"go.uber.org/multierr"
	"golang.org/x/oauth2"
)
const (
	GitHubHost string = "github.com"
	GitHubAPIHost string = "api." + GitHubHost
	CachePath string = ".cache"
	CacheFile string = "stars.db"
	PageSize int = 100
	DefaultConcurrency int = 10
)
type Star struct {
	Archived bool `storm:"index"`
	Description string `storm:"index"`
	Language string `storm:"index"`
	PushedAt time.Time `storm:"index"`
	Stargazers int
	StarredAt time.Time `storm:"index"`
	Topics []string `storm:"index"`
	URL string `storm:"id,index,unique"`
}
type StarManager struct {
	username string
	password string
	context  context.Context
	client   *github.Client
	db       *storm.DB
}
func New(logLevel log.Level) (*StarManager, error) {
	log.Tracef("Setting log level to %+v\n", logLevel)
	log.SetLevel(logLevel)
	log.Debug("Initializing auth")
	cfg, err := auth.NewConfig()
	if err != nil {
		log.Errorf("Could not parse netrc config: %v", err)
		return nil, err
	}
	log.Debug("Parsing auth credentials from .netrc")
	netrcAuth, err := auth.NewNetrc(cfg)
	username, password, err := netrcAuth.GetAuth(GitHubAPIHost)
	if err != nil {
		log.Errorf(
			"Could not find authentication credentials for %s in netrc configuration",
			GitHubAPIHost,
		)
		return nil, err
	}
	log.Trace("Initializing context")
	ctx := context.Background()
	client := github.NewClient(oauth2.NewClient(
		ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: password}),
	))
	log.Debug("Determining current user")
	currentUser, err := user.Current()
	if err != nil {
		log.Errorf("Could not determine the current user! %v\n", err)
		return nil, err
	}
	log.Debugf("Current user: %s\n", currentUser.Username)
	log.Debug("Ensuring local cache")
	cacheDir := filepath.Join(currentUser.HomeDir, CachePath)
	cacheFullPath := filepath.Join(cacheDir, CacheFile)
	for _, p := range []struct {
		path string
		mode os.FileMode
	}{
		{cacheDir, os.ModeDir},
		{cacheFullPath, 0},
	} {
		err := utils.CreateIfNotExists(p.path, p.mode, afero.NewOsFs())
		if err != nil {
			log.Infof("An error occurred while attempting to create %s: %v\n",
				p.path, err,
			)
		}
	}
	log.Debug("Initializing Storm/Bolt")
	db, err := storm.Open(cacheFullPath, storm.Batch())
	if err != nil {
		log.Errorf("An error occurred opening the db! %v", err)
		return nil, err
	}
	return &StarManager{
		username: username,
		password: password,
		context:  ctx,
		client:   client,
		db:       db,
	}, nil
}
func (s *StarManager) ClearCache() error {
	log.Debug("Clearing out cache")
	return os.Remove(s.db.Bolt.Path())
}
func (s *StarManager) StarRepository(owner, repo string) error {
	log.Debugf("Starring %s/%s\n", owner, repo)
	resp, err := s.client.Activity.Star(s.context, owner, repo)
	if err != nil {
		log.Errorf("An error occurred starring a repository! %+v", resp)
		return err
	}
	if resp.StatusCode == (http.StatusOK | http.StatusNoContent) {
		log.Infof("Successfully starred %s/%s", owner, repo)
	} else {
		log.Warningf("Got non-200 response for %s/%s: %d",
			owner, repo, resp.StatusCode,
		)
	}
	return nil
}
func (s *StarManager) starReposFromURLsChan(
	g *sync.WaitGroup,
	urlsChan chan *url.URL,
	successfulStars chan int,
	notOlderThanMonths int,
) {
	g.Add(1)
	defer g.Done()
	wg := sync.WaitGroup{}
	defer wg.Wait()
	then := time.Now().AddDate(0, -notOlderThanMonths, 0)
	for u := range urlsChan {
		log.Debugf("Got URL: %+v\n", u)
		parts := strings.Split(strings.Trim((*u).EscapedPath(), "/"), "/")
		if len(parts) != 2 {
			log.Errorf("%+v invalid", u)
			continue
		}
		log.Infof("Evaluating %s/%s\n", parts[0], parts[1])
		repo, _, err := s.client.Repositories.Get(s.context, parts[0], parts[1])
		if err != nil {
			log.Errorf("encountered error: %+v", err)
			continue
		}
		owner := repo.GetOwner().GetLogin()
		name := repo.GetName()
		log.Debugf("Checking whether %s/%s is starred\n", owner, name)
		starred, _, err := s.client.Activity.IsStarred(s.context, owner, name)
		if err != nil {
			log.Errorf("Encountered error: %+v", err)
			continue
		}
		if starred {
			log.Infof("%s/%s already starred - skipping\n", owner, name)
			continue
		}
		if !(repo.GetPushedAt().Before(then) || repo.GetArchived()) {
			if err := s.StarRepository(owner, name); err != nil {
				log.Errorf(
					"failed to star %s/%s\n",
					repo.GetOwner().GetLogin(),
					repo.GetName(),
				)
				continue
			}
			wg.Add(1)
			go func() { defer wg.Done(); successfulStars <- 1 }()
		} else {
			log.Infof(
				"%s/%s does not qualify - archived: %t, pushed: %s\n",
				repo.GetOwner().GetLogin(),
				repo.GetName(),
				repo.GetArchived(),
				repo.GetPushedAt(),
			)
		}
	}
}
func (s *StarManager) StarRepositoriesFromURLs(
	urls []*url.URL, notOlderThanMonths, maxConcurrency int,
) (int, error) {
	log.Debugf("Preparing to star %d repositories\n", len(urls))
	urlsCh := make(chan *url.URL)
	starCountCh := make(chan int)
	wg := sync.WaitGroup{}
	log.Debugf("Spawning %d goroutines\n", maxConcurrency)
	for i := 0; i < maxConcurrency; i++ {
		go s.starReposFromURLsChan(&wg, urlsCh, starCountCh, notOlderThanMonths)
	}
	for _, u := range urls {
		urlsCh <- u
	}
	close(urlsCh)
	go func() {
		wg.Wait()
		close(starCountCh)
	}()
	total := 0
	for count := range starCountCh {
		total += count
	}
	if total == 0 {
		log.Warn("Added 0 repos")
		return 0, nil
	}
	log.Infof("Successfully starred %d repos\n", total)
	return total, nil
}
func (s *StarManager) getReposFromOrgPageChan(
	org string, orgReposPageCh chan int, reposCh chan *github.Repository,
) {
	for pageNo := range orgReposPageCh {
		log.Printf("Fetching repos from page %d of %s org\n", pageNo, org)
		repos, resp, err := s.client.Repositories.ListByOrg(
			s.context,
			org,
			&github.RepositoryListByOrgOptions{
				Type:        "sources",
				ListOptions: github.ListOptions{PerPage: PageSize, Page: pageNo},
			},
		)
		if err != nil {
			log.Errorf("encountered error fetching page %d for org %s\n", pageNo, org)
			log.Errorf("response: %+v\n", resp)
			log.Errorf("error: %+v\n", err)
			continue
		}
		log.Infof("Successfully fetched %d repos from page %d of %s org",
			len(repos), pageNo, org,
		)
		for _, repo := range repos {
			reposCh <- repo
		}
	}
}
func (s *StarManager) StarRepositoriesFromOrg(
	org string, notOlderThanMonths, maxConcurrency int,
) error {
	repoURLs := []*url.URL{}
	log.Infof("Listing first %d repos for %s\n", PageSize, org)
	orgReposPage1, resp, err := s.client.Repositories.ListByOrg(
		s.context,
		org,
		&github.RepositoryListByOrgOptions{
			Type:        "sources",
			ListOptions: github.ListOptions{PerPage: PageSize, Page: 1},
		},
	)
	if err != nil {
		return fmt.Errorf("Got error: %+v (%w)", resp, err)
	}
	log.Infof("Parsing %d repos into URLs\n", len(orgReposPage1))
	for _, p1Repo := range orgReposPage1 {
		repoURL, err := url.Parse(p1Repo.GetHTMLURL())
		if err != nil {
			log.Errorf(
				"encountered error parsing repo url for %s/%s: %+v",
				org, p1Repo.GetName(), err,
			)
			continue
		}
		log.Debugf("Parsed %+v\n", repoURL)
		repoURLs = append(repoURLs, repoURL)
	}
	if resp.LastPage > 1 {
		log.Infof("Last page is %d. Fetching the rest of the pages\n", resp.LastPage)
		orgPagesCh := make(chan int)
		reposCh := make(chan *github.Repository)
		urlsCh := make(chan *url.URL)
		defer func() { close(orgPagesCh); close(reposCh); close(urlsCh) }()
		startIdx := 2
		numGoroutines := resp.LastPage
		if maxConcurrency > 0 {
			startIdx = 0
			numGoroutines = maxConcurrency
		}
		for i := startIdx; i < numGoroutines; i++ {
			go s.getReposFromOrgPageChan(org, orgPagesCh, reposCh)
		}
		go func(opc chan int, lastPage int) {
			for i := 2; i < lastPage+1; i++ {
				opc <- i
			}
		}(orgPagesCh, resp.LastPage)
		go func(rCh chan *github.Repository, uCh chan *url.URL) {
			for repo := range reposCh {
				repoURL, err := url.Parse(repo.GetHTMLURL())
				if err != nil {
					log.Errorf(
						"encountered error parsing repo url for %s/%s: %+v",
						org, repo.GetName(), err,
					)
					continue
				}
				urlsCh <- repoURL
			}
		}(reposCh, urlsCh)
		go func(uCh chan *url.URL, uSl []*url.URL) {
			for u := range uCh {
				repoURLs = append(uSl, u)
			}
		}(urlsCh, repoURLs)
	}
	_, starErr := s.StarRepositoriesFromURLs(
		repoURLs, notOlderThanMonths, maxConcurrency,
	)
	return starErr
}
func (s *StarManager) StarRepositoriesFromUser(
	username string, notOlderThanMonths, maxConcurrency int,
) error {
	repoURLs := []*url.URL{}
	repos, _, err := s.client.Repositories.List(
		s.context, username, &github.RepositoryListOptions{},
	)
	if err != nil {
		return fmt.Errorf(
			"error listing repositories for user %s: %w", username, err,
		)
	}
	log.Infof("Parsing %d repos into URLs\n", len(repos))
	for _, repo := range repos {
		repoURL, err := url.Parse(repo.GetHTMLURL())
		if err != nil {
			log.Errorf(
				"encountered error parsing repo url for %s/%s: %+v",
				username, repo.GetName(), err,
			)
		}
		log.Debugf("Parsed %+v\n", repoURL)
		repoURLs = append(repoURLs, repoURL)
	}
	count, err := s.StarRepositoriesFromURLs(
		repoURLs, notOlderThanMonths, maxConcurrency,
	)
	if err != nil {
		return fmt.Errorf(
			"encountered error starring repositories for user %s: %w",
			username, err,
		)
	}
	log.Infof("Successfully starred %d repos\n", count)
	return nil
}
func (s *StarManager) StarRepositoriesFromReader(
	r io.Reader, maxConcurrency, notOlderThanMonths int,
) (int, error) {
	urlsCh := make(chan *url.URL)
	starCountCh := make(chan int)
	wg := sync.WaitGroup{}
	var combinedErrors error
	log.Debugf("Spawning %d goroutines\n", maxConcurrency)
	for i := 0; i < maxConcurrency; i++ {
		go s.starReposFromURLsChan(&wg, urlsCh, starCountCh, notOlderThanMonths)
	}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		urlStr := scanner.Text()
		urlStrByDot := strings.Split(urlStr, ".")
		if urlStrByDot[len(urlStrByDot)-1] == "git" {
			urlStrByDot = urlStrByDot[:len(urlStrByDot)-1]
		}
		urlStr = strings.Join(urlStrByDot, ".")
		u, err := url.Parse(urlStr)
		if err != nil {
			multierror.Append(combinedErrors, err)
		}
		urlsCh <- u
	}
	close(urlsCh)
	go func() {
		wg.Wait()
		close(starCountCh)
	}()
	total := 0
	for count := range starCountCh {
		total += count
	}
	if total == 0 {
		log.Warn("Added 0 repos")
		return 0, nil
	}
	log.Infof("Successfully starred %d repos\n", total)
	return total, combinedErrors
}