func walkManContents(ctx context.Context, renderChan chan<- renderJob, dir string, mode renderingMode, gv globalView, newestModTime time.Time) (time.Time, error) {
	files, err := os.Open(dir)
	if err != nil {
		return newestModTime, err
	}
	defer files.Close()
	var predictedEOF bool
	for {
		if predictedEOF {
			break
		}
		names, err := files.Readdirnames(2048)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				if sce, ok := err.(*os.SyscallError); ok && sce.Err == syscall.ENOTDIR {
					return newestModTime, nil
				}
				return newestModTime, err
			}
		}
		predictedEOF = len(names) < 2048
		for _, fn := range names {
			if !strings.HasSuffix(fn, ".gz") ||
				strings.HasSuffix(fn, ".html.gz") {
				continue
			}
			full := filepath.Join(dir, fn)
			st, err := os.Lstat(full)
			if err != nil {
				continue
			}
			if st.ModTime().After(newestModTime) {
				newestModTime = st.ModTime()
			}
			symlink := st.Mode()&os.ModeSymlink != 0
			if !symlink {
				atomic.AddUint64(&gv.stats.ManpageBytes, uint64(st.Size()))
			}
			if mode == regularFiles && symlink ||
				mode == symlinks && !symlink {
				continue
			}
			n := strings.TrimSuffix(fn, ".gz") + ".html.gz"
			htmlst, err := os.Stat(filepath.Join(dir, n))
			if err == nil {
				atomic.AddUint64(&gv.stats.HTMLBytes, uint64(htmlst.Size()))
			}
			if err != nil || *forceRerender || htmlst.ModTime().Before(st.ModTime()) {
				m, err := manpage.FromServingPath(*servingDir, full)
				if err != nil {
					log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
					continue
				}
				versions := gv.xref[m.Name]
				for _, v := range versions {
					if v.ServingPath() == m.ServingPath() {
						m = v
						break
					}
				}
				for _, v := range versions {
					if v == m || *forceRerender {
						continue
					}
					vfull := filepath.Join(*servingDir, v.RawPath())
					vfn := filepath.Join(*servingDir, v.ServingPath()+".html.gz")
					vhtmlst, err := os.Stat(vfn)
					if err == nil && vhtmlst.ModTime().After(gv.start) {
						continue
					}
					vst, err := os.Stat(vfull)
					if err != nil {
						log.Printf("WARNING: stat %q: %v", vfull, err)
						continue
					}
					vreuse := ""
					if vhtmlst != nil && vhtmlst.ModTime().After(vst.ModTime()) {
						vreuse = vfn
					}
					log.Printf("%s invalidated by %s", vfn, full)
					select {
					case renderChan <- renderJob{
						dest:     vfn,
						src:      vfull,
						meta:     v,
						versions: versions,
						xref:     gv.xref,
						modTime:  vst.ModTime(),
						reuse:    vreuse,
					}:
					case <-ctx.Done():
						break
					}
				}
				var reuse string
				if symlink {
					link, err := os.Readlink(full)
					if err == nil {
						resolved := filepath.Join(dir, link)
						reuse = strings.TrimSuffix(resolved, ".gz") + ".html.gz"
					}
				}
				select {
				case renderChan <- renderJob{
					dest:     filepath.Join(dir, n),
					src:      full,
					meta:     m,
					versions: versions,
					xref:     gv.xref,
					modTime:  st.ModTime(),
					reuse:    reuse,
				}:
				case <-ctx.Done():
					break
				}
			}
		}
	}
	return newestModTime, nil
}
walkManContents walks over all entries in dir and, depending on mode, does:
1. send a renderJob for each regular file
2. send a renderJob for each symlink
package main
import (
	"compress/gzip"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
	"github.com/Debian/debiman/internal/commontmpl"
	"github.com/Debian/debiman/internal/convert"
	"github.com/Debian/debiman/internal/manpage"
	"github.com/Debian/debiman/internal/sitemap"
	"github.com/Debian/debiman/internal/write"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)
var (
	manwalkConcurrency = flag.Int("concurrency_manwalk",
		1000,  
		"Concurrency level for walking through binary package man directories (ulimit -n must be higher!)")
	renderConcurrency = flag.Int("concurrency_render",
		5,
		"Concurrency level for rendering manpages using mandoc")
	gzipLevel = flag.Int("gzip",
		9,
		"gzip compression level to use for compressing HTML versions of manpages. defaults to 9 to keep network traffic minimal, but useful to reduce for development/disaster recovery (level 1 results in a 2x speedup!)")
	baseURL = flag.String("base_url",
		"https://manpages.debian.org",
		"Base URL (without trailing slash) to the site. Used where absolute URLs are required, e.g. sitemaps.")
)
type breadcrumb struct {
	Link string
	Text string
}
type breadcrumbs []breadcrumb
func (b breadcrumbs) ToJSON() template.HTML {
	type item struct {
		Type string `json:"@type"`
		ID   string `json:"@id"`
		Name string `json:"name"`
	}
	type listItem struct {
		Type     string `json:"@type"`
		Position int    `json:"position"`
		Item     item   `json:"item"`
	}
	type breadcrumbList struct {
		Context  string     `json:"@context"`
		Type     string     `json:"@type"`
		Elements []listItem `json:"itemListElement"`
	}
	l := breadcrumbList{
		Context:  "http://schema.org",
		Type:     "BreadcrumbList",
		Elements: make([]listItem, len(b)),
	}
	for idx, br := range b {
		l.Elements[idx] = listItem{
			Type:     "ListItem",
			Position: idx + 1,
			Item: item{
				Type: "Thing",
				ID:   br.Link,
				Name: br.Text,
			},
		}
	}
	jsonb, err := json.Marshal(l)
	if err != nil {
		log.Fatal(err)
	}
	return template.HTML(jsonb)
}
var commonTmpls = commontmpl.MustParseCommonTmpls()
type renderingMode int
const (
	regularFiles renderingMode = iota
	symlinks
)
func listManpages(dir string) (map[string]*manpage.Meta, error) {
	manpageByName := make(map[string]*manpage.Meta)
	files, err := os.Open(dir)
	if err != nil {
		return nil, err
	}
	defer files.Close()
	var predictedEOF bool
	for {
		if predictedEOF {
			break
		}
		names, err := files.Readdirnames(2048)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				if sce, ok := err.(*os.SyscallError); ok && sce.Err == syscall.ENOTDIR {
					return nil, nil
				}
				return nil, err
			}
		}
		predictedEOF = len(names) < 2048
		for _, fn := range names {
			if !strings.HasSuffix(fn, ".gz") ||
				strings.HasSuffix(fn, ".html.gz") {
				continue
			}
			full := filepath.Join(dir, fn)
			m, err := manpage.FromServingPath(*servingDir, full)
			if err != nil {
				log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
				continue
			}
			manpageByName[fn] = m
		}
	}
	return manpageByName, nil
}
func renderDirectoryIndex(dir string, newestModTime time.Time) error {
	st, err := os.Stat(filepath.Join(dir, "index.html.gz"))
	if !*forceRerender && err == nil && st.ModTime().After(newestModTime) {
		return nil
	}
	manpageByName, err := listManpages(dir)
	if err != nil {
		return err
	}
	if len(manpageByName) == 0 {
		log.Printf("WARNING: empty directory %q, not generating package index", dir)
		return nil
	}
	return renderPkgindex(filepath.Join(dir, "index.html.gz"), manpageByName)
}