<commit_before>package tools

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/arduino/arduino-create-agent/utilities"
	"github.com/blang/semver"
	"github.com/xrash/smetrics"
)

type system struct {
	Host     string `json:"host"`
	URL      string `json:"url"`
	Name     string `json:"archiveFileName"`
	CheckSum string `json:"checksum"`
}

type tool struct {
	Name        string   `json:"name"`
	Version     string   `json:"version"`
	Systems     []system `json:"systems"`
	url         string
	destination string
}

type index struct {
	Packages []struct {
		Name  string `json:"name"`
		Tools []tool `json:"tools"`
	} `json:"packages"`
}

var systems = map[string]string{
	"linuxamd64":  "x86_64-linux-gnu",
	"linux386":    "i686-linux-gnu",
	"darwinamd64": "i386-apple-darwin11",
	"windows386":  "i686-mingw32",
}

func mimeType(data []byte) (string, error) {
	return http.DetectContentType(data[0:512]), nil
}

// Download will parse the index at the indexURL for the tool to download.
// It will extract it in a folder in .arduino-create, and it will update the
// Installed map.
//
// name contains the name of the tool.
// version contains the version of the tool.
// behaviour contains the strategy to use when there is already a tool installed
//
// If version is "latest" it will always download the latest version (regardless
// of the value of behaviour)
//
// If version is not "latest" and behaviour is "replace", it will download the
// version again. If instead behaviour is "keep" it will not download the version
// if it already exists.
func (t *Tools) Download(name, version, behaviour string) error {

	// Fetch the index
	resp, err := http.Get(t.IndexURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Read the body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var data index
	json.Unmarshal(body, &data)

	t.Logger.Println(string(body))

	// Find the tool by name
	correctTool := findTool(name, version, data)

	if correctTool.Name == "" {
		return errors.New("We couldn't find a tool with the name " + name + " and version " + version)
	}

	// Find the url based on system
	var correctSystem system
	max_similarity := 0.8

	for _, s := range correctTool.Systems {
		similarity := smetrics.Jaro(s.Host, systems[runtime.GOOS+runtime.GOARCH])
		if similarity > max_similarity {
			correctSystem = s
			max_similarity = similarity
		}
	}

	key := correctTool.Name + "-" + correctTool.Version

	// Check if it already exists
	if behaviour == "keep" {
		if _, ok := t.installed[key]; ok {
			t.Logger.Println("The tool is already present on the system")
			return nil
		}
	}

	// Download the tool
	t.Logger.Println("Downloading tool " + name + " from " + correctSystem.URL)
	resp, err = http.Get(correctSystem.URL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Read the body
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// Checksum
	checksum := sha256.Sum256(body)
	checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])

	if checkSumString != correctSystem.CheckSum {
		return errors.New("Checksum doesn't match")
	}

	// Decompress
	t.Logger.Println("Unpacking tool " + name)

	location := path.Join(dir(), correctTool.Name, correctTool.Version)
	err = os.RemoveAll(location)

	if err != nil {
		return err
	}

	srcType, err := mimeType(body)
	if err != nil {
		return err
	}

	switch srcType {
	case "application/zip":
		location, err = extractZip(body, location)
	case "application/x-bz2":
	case "application/octet-stream":
		location, err = extractBz2(body, location)
	case "application/x-gzip":
		location, err = extractTarGz(body, location)
	default:
		return errors.New("Unknown extension for file " + correctSystem.URL)
	}

	if err != nil {
		return err
	}

	t.installDrivers(location)

	// Ensure that the files are executable
	t.Logger.Println("Ensure that the files are executable")

	// Update the tool map
	t.Logger.Println("Updating map with location " + location)

	t.installed[name] = location
	t.installed[name+"-"+correctTool.Version] = location
	return t.writeMap()
}

func findTool(name, version string, data index) tool {
	var correctTool tool
	correctTool.Version = "0.0"

	for _, p := range data.Packages {
		for _, t := range p.Tools {
			if version != "latest" {
				if t.Name == name && t.Version == version {
					correctTool = t
				}
			} else {
				// Find latest
				v1, _ := semver.Make(t.Version)
				v2, _ := semver.Make(correctTool.Version)
				if t.Name == name && v1.Compare(v2) > 0 {
					correctTool = t
				}
			}
		}
	}
	return correctTool
}

func findBaseDir(dirList []string) string {
	baseDir := ""
	for index, _ := range dirList {
		candidateBaseDir := dirList[index]
		for i := index; i < len(dirList); i++ {
			if !strings.Contains(dirList[i], candidateBaseDir) {
				return baseDir
			}
		}
		// avoid setting the candidate if it is the last file
		if dirList[len(dirList)-1] != candidateBaseDir {
			baseDir = candidateBaseDir
		}
	}
	return baseDir
}

func extractZip(body []byte, location string) (string, error) {
	path, err := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body))
	r, err := zip.OpenReader(path)
	if err != nil {
		return location, err
	}

	var dirList []string

	for _, f := range r.File {
		dirList = append(dirList, f.Name)
	}

	basedir := findBaseDir(dirList)

	for _, f := range r.File {
		fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1))
		if f.FileInfo().IsDir() {
			os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
		} else {
			os.MkdirAll(filepath.Dir(fullname), 0755)
			perms := f.FileInfo().Mode().Perm()
			out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
			if err != nil {
				return location, err
			}
			rc, err := f.Open()
			if err != nil {
				return location, err
			}
			_, err = io.CopyN(out, rc, f.FileInfo().Size())
			if err != nil {
				return location, err
			}
			rc.Close()
			out.Close()

			mtime := f.FileInfo().ModTime()
			err = os.Chtimes(fullname, mtime, mtime)
			if err != nil {
				return location, err
			}
		}
	}
	return location, nil
}

func extractTarGz(body []byte, location string) (string, error) {
	bodyCopy := make([]byte, len(body))
	copy(bodyCopy, body)
	tarFile, _ := gzip.NewReader(bytes.NewReader(body))
	tarReader := tar.NewReader(tarFile)

	var dirList []string

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		dirList = append(dirList, header.Name)
	}

	basedir := findBaseDir(dirList)

	tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy))
	tarReader = tar.NewReader(tarFile)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			//return location, err
		}

		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
		info := header.FileInfo()

		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return location, err
			}
			continue
		}

		if header.Typeflag == tar.TypeSymlink {
			err = os.Symlink(header.Linkname, path)
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			//return location, err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			//return location, err
		}
	}
	return location, nil
}

func (t *Tools) installDrivers(location string) {
	if runtime.GOOS == "windows" {
		if _, err := os.Stat(filepath.Join(location, "post_install.bat")); err == nil {
			t.Logger.Println("Installing drivers")
			oscmd := exec.Command(filepath.Join(location, "post_install.bat"))
			TellCommandNotToSpawnShell(oscmd)
			oscmd.Run()
		}
	}
}

func extractBz2(body []byte, location string) (string, error) {
	bodyCopy := make([]byte, len(body))
	copy(bodyCopy, body)
	tarFile := bzip2.NewReader(bytes.NewReader(body))
	tarReader := tar.NewReader(tarFile)

	var dirList []string

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		dirList = append(dirList, header.Name)
	}

	basedir := findBaseDir(dirList)

	tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
	tarReader = tar.NewReader(tarFile)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			//return location, err
		}

		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
		info := header.FileInfo()

		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return location, err
			}
			continue
		}

		if header.Typeflag == tar.TypeSymlink {
			err = os.Symlink(header.Linkname, path)
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			//return location, err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			//return location, err
		}
	}
	return location, nil
}

func makeExecutable(location string) error {
	location = path.Join(location, "bin")
	files, err := ioutil.ReadDir(location)
	if err != nil {
		return err
	}

	for _, file := range files {
		err = os.Chmod(path.Join(location, file.Name()), 0755)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>add exclude list for extended header "files"<commit_after>package tools

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/arduino/arduino-create-agent/utilities"
	"github.com/blang/semver"
	"github.com/xrash/smetrics"
)

type system struct {
	Host     string `json:"host"`
	URL      string `json:"url"`
	Name     string `json:"archiveFileName"`
	CheckSum string `json:"checksum"`
}

type tool struct {
	Name        string   `json:"name"`
	Version     string   `json:"version"`
	Systems     []system `json:"systems"`
	url         string
	destination string
}

type index struct {
	Packages []struct {
		Name  string `json:"name"`
		Tools []tool `json:"tools"`
	} `json:"packages"`
}

var systems = map[string]string{
	"linuxamd64":  "x86_64-linux-gnu",
	"linux386":    "i686-linux-gnu",
	"darwinamd64": "i386-apple-darwin11",
	"windows386":  "i686-mingw32",
}

func mimeType(data []byte) (string, error) {
	return http.DetectContentType(data[0:512]), nil
}

// Download will parse the index at the indexURL for the tool to download.
// It will extract it in a folder in .arduino-create, and it will update the
// Installed map.
//
// name contains the name of the tool.
// version contains the version of the tool.
// behaviour contains the strategy to use when there is already a tool installed
//
// If version is "latest" it will always download the latest version (regardless
// of the value of behaviour)
//
// If version is not "latest" and behaviour is "replace", it will download the
// version again. If instead behaviour is "keep" it will not download the version
// if it already exists.
func (t *Tools) Download(name, version, behaviour string) error {

	// Fetch the index
	resp, err := http.Get(t.IndexURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Read the body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var data index
	json.Unmarshal(body, &data)

	t.Logger.Println(string(body))

	// Find the tool by name
	correctTool := findTool(name, version, data)

	if correctTool.Name == "" {
		return errors.New("We couldn't find a tool with the name " + name + " and version " + version)
	}

	// Find the url based on system
	var correctSystem system
	max_similarity := 0.8

	for _, s := range correctTool.Systems {
		similarity := smetrics.Jaro(s.Host, systems[runtime.GOOS+runtime.GOARCH])
		if similarity > max_similarity {
			correctSystem = s
			max_similarity = similarity
		}
	}

	key := correctTool.Name + "-" + correctTool.Version

	// Check if it already exists
	if behaviour == "keep" {
		if _, ok := t.installed[key]; ok {
			t.Logger.Println("The tool is already present on the system")
			return nil
		}
	}

	// Download the tool
	t.Logger.Println("Downloading tool " + name + " from " + correctSystem.URL)
	resp, err = http.Get(correctSystem.URL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Read the body
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	// Checksum
	checksum := sha256.Sum256(body)
	checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])

	if checkSumString != correctSystem.CheckSum {
		return errors.New("Checksum doesn't match")
	}

	// Decompress
	t.Logger.Println("Unpacking tool " + name)

	location := path.Join(dir(), correctTool.Name, correctTool.Version)
	err = os.RemoveAll(location)

	if err != nil {
		return err
	}

	srcType, err := mimeType(body)
	if err != nil {
		return err
	}

	switch srcType {
	case "application/zip":
		location, err = extractZip(body, location)
	case "application/x-bz2":
	case "application/octet-stream":
		location, err = extractBz2(body, location)
	case "application/x-gzip":
		location, err = extractTarGz(body, location)
	default:
		return errors.New("Unknown extension for file " + correctSystem.URL)
	}

	if err != nil {
		return err
	}

	t.installDrivers(location)

	// Ensure that the files are executable
	t.Logger.Println("Ensure that the files are executable")

	// Update the tool map
	t.Logger.Println("Updating map with location " + location)

	t.installed[name] = location
	t.installed[name+"-"+correctTool.Version] = location
	return t.writeMap()
}

func findTool(name, version string, data index) tool {
	var correctTool tool
	correctTool.Version = "0.0"

	for _, p := range data.Packages {
		for _, t := range p.Tools {
			if version != "latest" {
				if t.Name == name && t.Version == version {
					correctTool = t
				}
			} else {
				// Find latest
				v1, _ := semver.Make(t.Version)
				v2, _ := semver.Make(correctTool.Version)
				if t.Name == name && v1.Compare(v2) > 0 {
					correctTool = t
				}
			}
		}
	}
	return correctTool
}

func stringInSlice(str string, list []string) bool {
	for _, v := range list {
		if v == str {
			return true
		}
	}
	return false
}

func findBaseDir(dirList []string) string {
	baseDir := ""
	// https://github.com/backdrop-ops/contrib/issues/55#issuecomment-73814500
	dontdiff := []string{"pax_global_header"}
	for index, _ := range dirList {
		if stringInSlice(dirList[index], dontdiff) {
			continue
		}
		candidateBaseDir := dirList[index]
		for i := index; i < len(dirList); i++ {
			if !strings.Contains(dirList[i], candidateBaseDir) {
				return baseDir
			}
		}
		// avoid setting the candidate if it is the last file
		if dirList[len(dirList)-1] != candidateBaseDir {
			baseDir = candidateBaseDir
		}
	}
	return baseDir
}

func extractZip(body []byte, location string) (string, error) {
	path, err := utilities.SaveFileonTempDir("tooldownloaded.zip", bytes.NewReader(body))
	r, err := zip.OpenReader(path)
	if err != nil {
		return location, err
	}

	var dirList []string

	for _, f := range r.File {
		dirList = append(dirList, f.Name)
	}

	basedir := findBaseDir(dirList)

	for _, f := range r.File {
		fullname := filepath.Join(location, strings.Replace(f.Name, basedir, "", -1))
		if f.FileInfo().IsDir() {
			os.MkdirAll(fullname, f.FileInfo().Mode().Perm())
		} else {
			os.MkdirAll(filepath.Dir(fullname), 0755)
			perms := f.FileInfo().Mode().Perm()
			out, err := os.OpenFile(fullname, os.O_CREATE|os.O_RDWR, perms)
			if err != nil {
				return location, err
			}
			rc, err := f.Open()
			if err != nil {
				return location, err
			}
			_, err = io.CopyN(out, rc, f.FileInfo().Size())
			if err != nil {
				return location, err
			}
			rc.Close()
			out.Close()

			mtime := f.FileInfo().ModTime()
			err = os.Chtimes(fullname, mtime, mtime)
			if err != nil {
				return location, err
			}
		}
	}
	return location, nil
}

func extractTarGz(body []byte, location string) (string, error) {
	bodyCopy := make([]byte, len(body))
	copy(bodyCopy, body)
	tarFile, _ := gzip.NewReader(bytes.NewReader(body))
	tarReader := tar.NewReader(tarFile)

	var dirList []string

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		dirList = append(dirList, header.Name)
	}

	basedir := findBaseDir(dirList)

	tarFile, _ = gzip.NewReader(bytes.NewReader(bodyCopy))
	tarReader = tar.NewReader(tarFile)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			//return location, err
		}

		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
		info := header.FileInfo()

		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return location, err
			}
			continue
		}

		if header.Typeflag == tar.TypeSymlink {
			err = os.Symlink(header.Linkname, path)
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			//return location, err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			//return location, err
		}
	}
	return location, nil
}

func (t *Tools) installDrivers(location string) {
	if runtime.GOOS == "windows" {
		if _, err := os.Stat(filepath.Join(location, "post_install.bat")); err == nil {
			t.Logger.Println("Installing drivers")
			oscmd := exec.Command(filepath.Join(location, "post_install.bat"))
			TellCommandNotToSpawnShell(oscmd)
			oscmd.Run()
		}
	}
}

func extractBz2(body []byte, location string) (string, error) {
	bodyCopy := make([]byte, len(body))
	copy(bodyCopy, body)
	tarFile := bzip2.NewReader(bytes.NewReader(body))
	tarReader := tar.NewReader(tarFile)

	var dirList []string

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		}
		dirList = append(dirList, header.Name)
	}

	basedir := findBaseDir(dirList)

	tarFile = bzip2.NewReader(bytes.NewReader(bodyCopy))
	tarReader = tar.NewReader(tarFile)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			//return location, err
		}

		path := filepath.Join(location, strings.Replace(header.Name, basedir, "", -1))
		info := header.FileInfo()

		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return location, err
			}
			continue
		}

		if header.Typeflag == tar.TypeSymlink {
			err = os.Symlink(header.Linkname, path)
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			//return location, err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			//return location, err
		}
	}
	return location, nil
}

func makeExecutable(location string) error {
	location = path.Join(location, "bin")
	files, err := ioutil.ReadDir(location)
	if err != nil {
		return err
	}

	for _, file := range files {
		err = os.Chmod(path.Join(location, file.Name()), 0755)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package gofakeit

import (
	"math/rand"
	"strings"
)

// AddressInfo is a struct full of address information
type AddressInfo struct {
	Address   string
	Street    string
	City      string
	State     string
	Zip       string
	Country   string
	Latitude  float64
	Longitude float64
}

// Address will generate a struct of address information
func Address() *AddressInfo {
	street := Street()
	city := City()
	state := State()
	zip := Zip()

	return &AddressInfo{
		Address:   street + ", " + city + ", " + state + " " + zip,
		Street:    street,
		City:      city,
		State:     state,
		Zip:       zip,
		Country:   Country(),
		Latitude:  Latitude(),
		Longitude: Longitude(),
	}
}

// Street will generate a random address street string
func Street() (street string) {
	switch randInt := randIntRange(1, 2); randInt {
	case 1:
		street = StreetNumber() + " " + StreetPrefix() + " " + StreetName() + " " + StreetSuffix()
	case 2:
		street = StreetNumber() + " " + StreetName() + " " + StreetSuffix()
	}

	return
}

// StreetNumber will generate a random address street number string
func StreetNumber() string {
	return strings.TrimLeft(replaceWithNumbers(getRandValue([]string{"address", "number"})), "0")
}

// StreetPrefix will generate a random address street prefix string
func StreetPrefix() string {
	return getRandValue([]string{"address", "street_prefix"})
}

// StreetName will generate a random address street name string
func StreetName() string {
	return getRandValue([]string{"address", "street_name"})
}

// StreetSuffix will generate a random address street suffix string
func StreetSuffix() string {
	return getRandValue([]string{"address", "street_suffix"})
}

// City will generate a random city string
func City() (city string) {
	switch randInt := randIntRange(1, 3); randInt {
	case 1:
		city = FirstName() + StreetSuffix()
	case 2:
		city = LastName() + StreetSuffix()
	case 3:
		city = StreetPrefix() + " " + LastName()
	}

	return
}

// State will generate a random state string
func State() string {
	return getRandValue([]string{"address", "state"})
}

// StateAbr will generate a random abbreviated state string
func StateAbr() string {
	return getRandValue([]string{"address", "state_abr"})
}

// Zip will generate a random Zip code string
func Zip() string {
	return replaceWithNumbers(getRandValue([]string{"address", "zip"}))
}

// Country will generate a random country string
func Country() string {
	return getRandValue([]string{"address", "country"})
}

// Latitude will generate a random latitude float64
func Latitude() float64 { return (rand.Float64() * 180) - 90 }

// Longitude will generate a random longitude float64
func Longitude() float64 { return (rand.Float64() * 360) - 180 }
<commit_msg>Fixes extra space between street name and suffix<commit_after>package gofakeit

import (
	"math/rand"
	"strings"
)

// AddressInfo is a struct full of address information
type AddressInfo struct {
	Address   string
	Street    string
	City      string
	State     string
	Zip       string
	Country   string
	Latitude  float64
	Longitude float64
}

// Address will generate a struct of address information
func Address() *AddressInfo {
	street := Street()
	city := City()
	state := State()
	zip := Zip()

	return &AddressInfo{
		Address:   street + ", " + city + ", " + state + " " + zip,
		Street:    street,
		City:      city,
		State:     state,
		Zip:       zip,
		Country:   Country(),
		Latitude:  Latitude(),
		Longitude: Longitude(),
	}
}

// Street will generate a random address street string
func Street() (street string) {
	switch randInt := randIntRange(1, 2); randInt {
	case 1:
		street = StreetNumber() + " " + StreetPrefix() + " " + StreetName() + StreetSuffix()
	case 2:
		street = StreetNumber() + " " + StreetName() + StreetSuffix()
	}

	return
}

// StreetNumber will generate a random address street number string
func StreetNumber() string {
	return strings.TrimLeft(replaceWithNumbers(getRandValue([]string{"address", "number"})), "0")
}

// StreetPrefix will generate a random address street prefix string
func StreetPrefix() string {
	return getRandValue([]string{"address", "street_prefix"})
}

// StreetName will generate a random address street name string
func StreetName() string {
	return getRandValue([]string{"address", "street_name"})
}

// StreetSuffix will generate a random address street suffix string
func StreetSuffix() string {
	return getRandValue([]string{"address", "street_suffix"})
}

// City will generate a random city string
func City() (city string) {
	switch randInt := randIntRange(1, 3); randInt {
	case 1:
		city = FirstName() + StreetSuffix()
	case 2:
		city = LastName() + StreetSuffix()
	case 3:
		city = StreetPrefix() + " " + LastName()
	}

	return
}

// State will generate a random state string
func State() string {
	return getRandValue([]string{"address", "state"})
}

// StateAbr will generate a random abbreviated state string
func StateAbr() string {
	return getRandValue([]string{"address", "state_abr"})
}

// Zip will generate a random Zip code string
func Zip() string {
	return replaceWithNumbers(getRandValue([]string{"address", "zip"}))
}

// Country will generate a random country string
func Country() string {
	return getRandValue([]string{"address", "country"})
}

// Latitude will generate a random latitude float64
func Latitude() float64 { return (rand.Float64() * 180) - 90 }

// Longitude will generate a random longitude float64
func Longitude() float64 { return (rand.Float64() * 360) - 180 }
<|endoftext|>


<commit_before>package cli

var all_templates = map[string]string{
	"debug":          default_debug_template,
	"fields":         default_debug_template,
	"editmeta":       default_debug_template,
	"transmeta":      default_debug_template,
	"createmeta":     default_debug_template,
	"issuelinktypes": default_debug_template,
	"list":           default_list_template,
	"view":           default_view_template,
	"edit":           default_edit_template,
	"transitions":    default_transitions_template,
	"issuetypes":     default_issuetypes_template,
	"create":         default_create_template,
	"comment":        default_comment_template,
}

const default_debug_template = "{{ . | toJson}}\n"

const default_list_template = "{{ range .issues }}{{ .key | append \":\" | printf \"%-12s\"}} {{ .fields.summary }}\n{{ end }}"

const default_view_template = `issue: {{ .key }}
status: {{ .fields.status.name }}
summary: {{ .fields.summary }}
project: {{ .fields.project.key }}
components: {{ range .fields.components }}{{ .name }} {{end}}
issuetype: {{ .fields.issuetype.name }}
assignee: {{ .fields.assignee.name }}
reporter: {{ .fields.reporter.name }}
watchers: {{ range .fields.customfield_10110 }}{{ .name }} {{end}}
blockers: {{ range .fields.issuelinks }}{{if .outwardIssue}}{{ .outwardIssue.key }}[{{.outwardIssue.fields.status.name}}]{{end}}{{end}}
depends: {{ range .fields.issuelinks }}{{if .inwardIssue}}{{ .inwardIssue.key }}[{{.inwardIssue.fields.status.name}}]{{end}}{{end}}
priority: {{ .fields.priority.name }}
description: |
  {{ .fields.description | indent 2 }}

comments:
{{ range .fields.comment.comments }}  - | # {{.author.name}} at {{.created}}
    {{ .body | indent 4}}
{{end}}
`
const default_edit_template = `update:
  comment:
    - add:
        body: |
          {{ or .overrides.comment ""}}
fields:
  summary: {{ or .overrides.summary .fields.summary }}
  components: # {{ range .meta.fields.components.allowedValues }}{{.name}}, {{end}}{{if .overrides.components }}{{ range (split "," .overrides.components)}}
    - name: {{.}}{{end}}{{else}}{{ range .fields.components }}
    - name: {{ .name }}{{end}}{{end}}
  assignee:
    name: {{ if .overrides.assignee }}{{.overrides.assignee}}{{else}}{{if .fields.assignee }}{{ .fields.assignee.name }}{{end}}{{end}}
  reporter:
    name: {{ or .overrides.reporter .fields.reporter.name }}
  # watchers
  customfield_10110: {{ range .fields.customfield_10110 }}
    - name: {{ .name }}{{end}}{{if .overrides.watcher}}
    - name: {{ .overrides.watcher}}{{end}}
  priority: # {{ range .meta.fields.priority.allowedValues }}{{.name}}, {{end}}
    name: {{ or .overrides.priority .fields.priority.name }}
  description: |
    {{ or .overrides.description (or .fields.description "") | indent 4 }}
`
const default_transitions_template = `{{ range .transitions }}{{.id }}: {{.name}}
{{end}}`

const default_issuetypes_template = `{{ range .projects }}{{ range .issuetypes }}{{color "+bh"}}{{.name | append ":" | printf "%-13s" }}{{color "reset"}} {{.description}}
{{end}}{{end}}`

const default_create_template = `fields:
  project:
    key: {{ .overrides.project }}
  issuetype:
    name: {{ .overrides.issuetype }}
  summary: {{ or .overrides.summary "" }}
  priority: # {{ range .meta.fields.priority.allowedValues }}{{.name}}, {{end}}
    name: {{ or .overrides.priority "" }}
  components: # {{ range .meta.fields.components.allowedValues }}{{.name}}, {{end}}{{ range split "," (or .overrides.components "")}}
    - name: {{ . }}{{end}}
  description: |
    {{ or .overrides.description "" | indent 4 }}
  assignee:
    name: {{ or .overrides.assignee .overrides.user}}
  reporter:
    name: {{ or .overrides.reporter .overrides.user }}
  # watchers
  customfield_10110:
    - name:
`

const default_comment_template = `body: |

`
<commit_msg>default view template shouldn't fail when assignee is nil<commit_after>package cli

var all_templates = map[string]string{
	"debug":          default_debug_template,
	"fields":         default_debug_template,
	"editmeta":       default_debug_template,
	"transmeta":      default_debug_template,
	"createmeta":     default_debug_template,
	"issuelinktypes": default_debug_template,
	"list":           default_list_template,
	"view":           default_view_template,
	"edit":           default_edit_template,
	"transitions":    default_transitions_template,
	"issuetypes":     default_issuetypes_template,
	"create":         default_create_template,
	"comment":        default_comment_template,
}

const default_debug_template = "{{ . | toJson}}\n"

const default_list_template = "{{ range .issues }}{{ .key | append \":\" | printf \"%-12s\"}} {{ .fields.summary }}\n{{ end }}"

const default_view_template = `issue: {{ .key }}
status: {{ .fields.status.name }}
summary: {{ .fields.summary }}
project: {{ .fields.project.key }}
components: {{ range .fields.components }}{{ .name }} {{end}}
issuetype: {{ .fields.issuetype.name }}
assignee: {{ if .fields.assignee }}{{ .fields.assignee }}{{end}}
reporter: {{ .fields.reporter.name }}
watchers: {{ range .fields.customfield_10110 }}{{ .name }} {{end}}
blockers: {{ range .fields.issuelinks }}{{if .outwardIssue}}{{ .outwardIssue.key }}[{{.outwardIssue.fields.status.name}}]{{end}}{{end}}
depends: {{ range .fields.issuelinks }}{{if .inwardIssue}}{{ .inwardIssue.key }}[{{.inwardIssue.fields.status.name}}]{{end}}{{end}}
priority: {{ .fields.priority.name }}
description: |
  {{ .fields.description | indent 2 }}

comments:
{{ range .fields.comment.comments }}  - | # {{.author.name}} at {{.created}}
    {{ .body | indent 4}}
{{end}}
`
const default_edit_template = `update:
  comment:
    - add:
        body: |
          {{ or .overrides.comment ""}}
fields:
  summary: {{ or .overrides.summary .fields.summary }}
  components: # {{ range .meta.fields.components.allowedValues }}{{.name}}, {{end}}{{if .overrides.components }}{{ range (split "," .overrides.components)}}
    - name: {{.}}{{end}}{{else}}{{ range .fields.components }}
    - name: {{ .name }}{{end}}{{end}}
  assignee:
    name: {{ if .overrides.assignee }}{{.overrides.assignee}}{{else}}{{if .fields.assignee }}{{ .fields.assignee.name }}{{end}}{{end}}
  reporter:
    name: {{ or .overrides.reporter .fields.reporter.name }}
  # watchers
  customfield_10110: {{ range .fields.customfield_10110 }}
    - name: {{ .name }}{{end}}{{if .overrides.watcher}}
    - name: {{ .overrides.watcher}}{{end}}
  priority: # {{ range .meta.fields.priority.allowedValues }}{{.name}}, {{end}}
    name: {{ or .overrides.priority .fields.priority.name }}
  description: |
    {{ or .overrides.description (or .fields.description "") | indent 4 }}
`
const default_transitions_template = `{{ range .transitions }}{{.id }}: {{.name}}
{{end}}`

const default_issuetypes_template = `{{ range .projects }}{{ range .issuetypes }}{{color "+bh"}}{{.name | append ":" | printf "%-13s" }}{{color "reset"}} {{.description}}
{{end}}{{end}}`

const default_create_template = `fields:
  project:
    key: {{ .overrides.project }}
  issuetype:
    name: {{ .overrides.issuetype }}
  summary: {{ or .overrides.summary "" }}
  priority: # {{ range .meta.fields.priority.allowedValues }}{{.name}}, {{end}}
    name: {{ or .overrides.priority "" }}
  components: # {{ range .meta.fields.components.allowedValues }}{{.name}}, {{end}}{{ range split "," (or .overrides.components "")}}
    - name: {{ . }}{{end}}
  description: |
    {{ or .overrides.description "" | indent 4 }}
  assignee:
    name: {{ or .overrides.assignee .overrides.user}}
  reporter:
    name: {{ or .overrides.reporter .overrides.user }}
  # watchers
  customfield_10110:
    - name:
`

const default_comment_template = `body: |

`
<|endoftext|>


<commit_before>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	client_model "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/tomasen/fcgi_client"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{"socket_path"}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{"socket_path"}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{"socket_path"}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{"socket_path"}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{"socket_path"}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{"socket_path"}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{"socket_path"}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{"socket_path"}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{"socket_path"}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{"socket_path"}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{"socket_path"}, nil),
	}
)

func CollectStatusFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectStatusFromSocket(path string, statusPath string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = statusPath
	env["SCRIPT_NAME"] = statusPath
	env["REQUEST_METHOD"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}
	defer fcgi.Close()

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectStatusFromReader(resp.Body, path, ch)
}

func CollectMetricsFromScript(socketPaths []string, scriptPaths []string) ([]*client_model.MetricFamily, error) {
	var result []*client_model.MetricFamily

	for _, socketPath := range socketPaths {

		for _, scriptPath := range scriptPaths {
			fcgi, err := fcgiclient.Dial("unix", socketPath)
			if err != nil {
				return result, err
			}
			defer fcgi.Close()

			env := make(map[string]string)
			env["DOCUMENT_ROOT"] = path.Dir(scriptPath)
			env["SCRIPT_FILENAME"] = scriptPath
			env["SCRIPT_NAME"] = path.Base(scriptPath)
			env["REQUEST_METHOD"] = "GET"

			resp, err := fcgi.Get(env)
			if err != nil {
				return result, err
			}

			var parser expfmt.TextParser
			metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
			if err != nil {
				return result, err
			}

			for _, metric := range metricFamilies {
				result = append(result, metric)
			}
		}
	}
	return result, nil
}

type PhpfpmExporter struct {
	socketPaths []string
	statusPath  string
}

func NewPhpfpmExporter(socketPaths []string, statusPath string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		statusPath:  statusPath,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {

	for _, socketPath := range e.socketPaths {
		err := CollectStatusFromSocket(socketPath, e.statusPath, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress        = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9253").String()
		metricsPath          = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		socketPaths          = kingpin.Flag("phpfpm.socket-paths", "Paths of the PHP-FPM sockets.").Strings()
		statusPath           = kingpin.Flag("phpfpm.status-path", "Path which has been configured in PHP-FPM to show status page.").Default("/status").String()
		scriptCollectorPaths = kingpin.Flag("phpfpm.script-collector-paths", "Paths of the PHP file whose output needs to be collected.").Strings()
	)

	kingpin.Parse()

	exporter, err := NewPhpfpmExporter(*socketPaths, *statusPath)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	if len(*scriptCollectorPaths) != 0 {
		prometheus.DefaultGatherer = prometheus.Gatherers{
			prometheus.DefaultGatherer,
			prometheus.GathererFunc(func() ([]*client_model.MetricFamily, error) {
				return CollectMetricsFromScript(*socketPaths, *scriptCollectorPaths)
			}),
		}
	}

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Add socket_path and script_path labels to the script collector.<commit_after>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	client_model "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/tomasen/fcgi_client"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	phpfpmSocketPathLabel = "socket_path"
	phpfpmScriptPathLabel = "script_path"

	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{phpfpmSocketPathLabel}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{phpfpmSocketPathLabel}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{phpfpmSocketPathLabel}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{phpfpmSocketPathLabel}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{phpfpmSocketPathLabel}, nil),
	}
)

func CollectStatusFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectStatusFromSocket(path string, statusPath string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = statusPath
	env["SCRIPT_NAME"] = statusPath
	env["REQUEST_METHOD"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}
	defer fcgi.Close()

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectStatusFromReader(resp.Body, path, ch)
}

func CollectMetricsFromScript(socketPaths []string, scriptPaths []string) ([]*client_model.MetricFamily, error) {
	var result []*client_model.MetricFamily

	for _, socketPath := range socketPaths {

		for _, scriptPath := range scriptPaths {
			fcgi, err := fcgiclient.Dial("unix", socketPath)
			if err != nil {
				return result, err
			}
			defer fcgi.Close()

			env := make(map[string]string)
			env["DOCUMENT_ROOT"] = path.Dir(scriptPath)
			env["SCRIPT_FILENAME"] = scriptPath
			env["SCRIPT_NAME"] = path.Base(scriptPath)
			env["REQUEST_METHOD"] = "GET"

			resp, err := fcgi.Get(env)
			if err != nil {
				return result, err
			}

			var parser expfmt.TextParser
			metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
			if err != nil {
				return result, err
			}

			for _, metricFamily := range metricFamilies {
				for _, metric := range metricFamily.Metric {
					metric.Label = append(
						metric.Label,
						&client_model.LabelPair{
							Name:  &phpfpmSocketPathLabel,
							Value: &socketPath,
						},
						&client_model.LabelPair{
							Name:  &phpfpmScriptPathLabel,
							Value: &scriptPath,
						})
				}
				result = append(result, metricFamily)
			}
		}
	}
	return result, nil
}

type PhpfpmExporter struct {
	socketPaths []string
	statusPath  string
}

func NewPhpfpmExporter(socketPaths []string, statusPath string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		statusPath:  statusPath,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {

	for _, socketPath := range e.socketPaths {
		err := CollectStatusFromSocket(socketPath, e.statusPath, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress        = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9253").String()
		metricsPath          = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
		socketPaths          = kingpin.Flag("phpfpm.socket-paths", "Paths of the PHP-FPM sockets.").Strings()
		statusPath           = kingpin.Flag("phpfpm.status-path", "Path which has been configured in PHP-FPM to show status page.").Default("/status").String()
		scriptCollectorPaths = kingpin.Flag("phpfpm.script-collector-paths", "Paths of the PHP file whose output needs to be collected.").Strings()
	)

	kingpin.Parse()

	exporter, err := NewPhpfpmExporter(*socketPaths, *statusPath)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	if len(*scriptCollectorPaths) != 0 {
		prometheus.DefaultGatherer = prometheus.Gatherers{
			prometheus.DefaultGatherer,
			prometheus.GathererFunc(func() ([]*client_model.MetricFamily, error) {
				return CollectMetricsFromScript(*socketPaths, *scriptCollectorPaths)
			}),
		}
	}

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"math"
	"net/url"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	pkgTest "knative.dev/pkg/test"
	"knative.dev/pkg/test/spoof"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	"knative.dev/serving/pkg/pool"
	serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names"
	v1alpha1testing "knative.dev/serving/pkg/testing/v1alpha1"
	"knative.dev/serving/test"
	v1a1test "knative.dev/serving/test/v1alpha1"
)

// Latencies is an interface for providing mechanisms for recording timings
// for the parts of the scale test.
type Latencies interface {
	// Add takes the name of this measurement and the time at which it began.
	// This should be called at the moment of completion, so that duration may
	// be computed with `time.Since(start)`.  We use this signature to that this
	// function is suitable for use in a `defer`.
	Add(name string, start time.Time)
}

func abortOnTimeout(ctx context.Context) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		return true, ctx.Err()
	}
}

func ScaleToWithin(t *testing.T, scale int, duration time.Duration, latencies Latencies) {
	clients := Setup(t)

	cleanupCh := make(chan test.ResourceNames, scale)
	defer close(cleanupCh)

	// These are the local (per-probe) and global (all probes) targets for the scale test.
	// 90 = 18/20, so allow two failures with the minimum number of probes, but expect
	// us to have 2.5 9s overall.
	//
	// TODO(#3210): After moving to Istio 1.1 we need to revisit these SLOs.
	const (
		localSLO  = 0.90
		globalSLO = 0.995
		minProbes = 20
	)
	pm := test.NewProberManager(t.Logf, clients, minProbes)

	ctx, cancel := context.WithTimeout(context.Background(), duration)
	defer cancel()
	width := int(math.Ceil(math.Log10(float64(scale))))

	t.Log("Creating new Services")
	wg := pool.NewWithCapacity(50 /* maximum in-flight creates */, scale /* capacity */)
	for i := 0; i < scale; i++ {
		// https://golang.org/doc/faq#closures_and_goroutines
		i := i
		wg.Go(func() error {
			names := test.ResourceNames{
				Service: test.SubServiceNameForTest(t, fmt.Sprintf("%0[1]*[2]d", width, i)),
				Image:   "helloworld",
			}

			// Start the clock for various waypoints towards Service readiness.
			start := time.Now()
			// Record the overall completion time regardless of success/failure.
			defer latencies.Add("time-to-done", start)

			svc, err := v1a1test.CreateLatestService(t, clients, names,
				v1alpha1testing.WithResourceRequirements(corev1.ResourceRequirements{
					Limits: corev1.ResourceList{
						corev1.ResourceCPU:    resource.MustParse("10m"),
						corev1.ResourceMemory: resource.MustParse("50Mi"),
					},
					Requests: corev1.ResourceList{
						corev1.ResourceCPU:    resource.MustParse("10m"),
						corev1.ResourceMemory: resource.MustParse("20Mi"),
					},
				}),
				v1alpha1testing.WithConfigAnnotations(map[string]string{
					"autoscaling.knative.dev/maxScale": "1",
				}),
				v1alpha1testing.WithReadinessProbe(&corev1.Probe{
					Handler: corev1.Handler{
						HTTPGet: &corev1.HTTPGetAction{
							Path: "/",
						},
					},
				}),
				v1alpha1testing.WithRevisionTimeoutSeconds(10))

			if err != nil {
				t.Errorf("CreateLatestService() = %v", err)
				return fmt.Errorf("CreateLatestService() failed: %w", err)
			}
			// Record the time it took to create the service.
			latencies.Add("time-to-create", start)
			names.Route = serviceresourcenames.Route(svc)
			names.Config = serviceresourcenames.Configuration(svc)

			// Send it to our cleanup logic (below)
			cleanupCh <- names

			t.Logf("Wait for %s to become ready.", names.Service)
			var url *url.URL
			err = v1a1test.WaitForServiceState(clients.ServingAlphaClient, names.Service, func(s *v1alpha1.Service) (bool, error) {
				if ctx.Err() != nil {
					return false, ctx.Err()
				}
				if s.Status.URL == nil {
					return false, nil
				}
				url = s.Status.URL.URL()
				return v1a1test.IsServiceReady(s)
			}, "ServiceUpdatedWithURL")
			if err != nil {
				t.Errorf("WaitForServiceState(w/ Domain) = %v", err)
				return fmt.Errorf("WaitForServiceState(w/ Domain) failed: %w", err)
			}

			// Record the time it took to become ready.
			latencies.Add("time-to-ready", start)

			_, err = pkgTest.WaitForEndpointState(
				clients.KubeClient,
				t.Logf,
				url,
				v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesBody(test.HelloWorldText), abortOnTimeout(ctx))),
				"WaitForEndpointToServeText",
				test.ServingFlags.ResolvableDomain)
			if err != nil {
				t.Errorf("WaitForEndpointState(expected text) = %v", err)
				return fmt.Errorf("WaitForEndpointState(expected text) failed: %w", err)
			}

			// Record the time it took to get back a 200 with the expected text.
			latencies.Add("time-to-200", start)

			// Start probing the domain until the test is complete.
			pm.Spawn(url)

			t.Logf("%s is ready.", names.Service)
			return nil
		})
	}

	// Wait for all of the service creations to complete (possibly in failure),
	// and signal the done channel.
	doneCh := make(chan error)
	go func() {
		defer close(doneCh)
		if err := wg.Wait(); err != nil {
			doneCh <- err
		}
	}()

	for {
		// As services get created, add logic to clean them up.
		// When all of the creations have finished, then stop all of the active probers
		// and check our SLIs against our SLOs.
		// All of this has to finish within the configured timeout.
		select {
		case names := <-cleanupCh:
			t.Logf("Added %v to cleanup routine.", names)
			test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
			defer test.TearDown(clients, names)

		case err := <-doneCh:
			if err != nil {
				// If we don't do this first, then we'll see tons of 503s from the ongoing probes
				// as we tear down the things they are probing.
				defer pm.Stop()
				t.Fatalf("Unexpected error: %v", err)
			}

			// This ProberManager implementation waits for minProbes before actually stopping.
			if err := pm.Stop(); err != nil {
				t.Fatalf("Stop() = %v", err)
			}
			// Check each of the local SLOs
			pm.Foreach(func(u *url.URL, p test.Prober) {
				if err := test.CheckSLO(localSLO, u.String(), p); err != nil {
					t.Errorf("CheckSLO() = %v", err)
				}
			})
			// Check the global SLO
			if err := test.CheckSLO(globalSLO, "aggregate", pm); err != nil {
				t.Errorf("CheckSLO() = %v", err)
			}
			return
		}
	}
}
<commit_msg>Update the SLO for scale test and permit more resources (#6684)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"math"
	"net/url"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	pkgTest "knative.dev/pkg/test"
	"knative.dev/pkg/test/spoof"
	"knative.dev/serving/pkg/apis/serving/v1alpha1"
	"knative.dev/serving/pkg/pool"
	serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names"
	v1alpha1testing "knative.dev/serving/pkg/testing/v1alpha1"
	"knative.dev/serving/test"
	v1a1test "knative.dev/serving/test/v1alpha1"
)

// Latencies is an interface for providing mechanisms for recording timings
// for the parts of the scale test.
type Latencies interface {
	// Add takes the name of this measurement and the time at which it began.
	// This should be called at the moment of completion, so that duration may
	// be computed with `time.Since(start)`.  We use this signature to that this
	// function is suitable for use in a `defer`.
	Add(name string, start time.Time)
}

func abortOnTimeout(ctx context.Context) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		return true, ctx.Err()
	}
}

func ScaleToWithin(t *testing.T, scale int, duration time.Duration, latencies Latencies) {
	clients := Setup(t)

	cleanupCh := make(chan test.ResourceNames, scale)
	defer close(cleanupCh)

	// These are the local (per-probe) and global (all probes) targets for the scale test.
	// 95 = 19/20, so allow one failure within the minimum number of probes, but expect
	// us to have 3 9s overall.
	const (
		localSLO  = 0.95
		globalSLO = 0.999
		minProbes = 20
	)
	pm := test.NewProberManager(t.Logf, clients, minProbes)

	ctx, cancel := context.WithTimeout(context.Background(), duration)
	defer cancel()
	width := int(math.Ceil(math.Log10(float64(scale))))

	t.Log("Creating new Services")
	wg := pool.NewWithCapacity(50 /* maximum in-flight creates */, scale /* capacity */)
	for i := 0; i < scale; i++ {
		// https://golang.org/doc/faq#closures_and_goroutines
		i := i
		wg.Go(func() error {
			names := test.ResourceNames{
				Service: test.SubServiceNameForTest(t, fmt.Sprintf("%0[1]*[2]d", width, i)),
				Image:   "helloworld",
			}

			// Start the clock for various waypoints towards Service readiness.
			start := time.Now()
			// Record the overall completion time regardless of success/failure.
			defer latencies.Add("time-to-done", start)

			svc, err := v1a1test.CreateLatestService(t, clients, names,
				v1alpha1testing.WithResourceRequirements(corev1.ResourceRequirements{
					Limits: corev1.ResourceList{
						corev1.ResourceCPU:    resource.MustParse("50m"),
						corev1.ResourceMemory: resource.MustParse("50Mi"),
					},
					Requests: corev1.ResourceList{
						corev1.ResourceCPU:    resource.MustParse("30m"),
						corev1.ResourceMemory: resource.MustParse("20Mi"),
					},
				}),
				v1alpha1testing.WithConfigAnnotations(map[string]string{
					"autoscaling.knative.dev/maxScale": "1",
				}),
				v1alpha1testing.WithReadinessProbe(&corev1.Probe{
					Handler: corev1.Handler{
						HTTPGet: &corev1.HTTPGetAction{
							Path: "/",
						},
					},
				}),
				v1alpha1testing.WithRevisionTimeoutSeconds(10))

			if err != nil {
				t.Errorf("CreateLatestService() = %v", err)
				return fmt.Errorf("CreateLatestService() failed: %w", err)
			}
			// Record the time it took to create the service.
			latencies.Add("time-to-create", start)
			names.Route = serviceresourcenames.Route(svc)
			names.Config = serviceresourcenames.Configuration(svc)

			// Send it to our cleanup logic (below)
			cleanupCh <- names

			t.Logf("Wait for %s to become ready.", names.Service)
			var url *url.URL
			err = v1a1test.WaitForServiceState(clients.ServingAlphaClient, names.Service, func(s *v1alpha1.Service) (bool, error) {
				if ctx.Err() != nil {
					return false, ctx.Err()
				}
				if s.Status.URL == nil {
					return false, nil
				}
				url = s.Status.URL.URL()
				return v1a1test.IsServiceReady(s)
			}, "ServiceUpdatedWithURL")
			if err != nil {
				t.Errorf("WaitForServiceState(w/ Domain) = %v", err)
				return fmt.Errorf("WaitForServiceState(w/ Domain) failed: %w", err)
			}

			// Record the time it took to become ready.
			latencies.Add("time-to-ready", start)

			_, err = pkgTest.WaitForEndpointState(
				clients.KubeClient,
				t.Logf,
				url,
				v1a1test.RetryingRouteInconsistency(pkgTest.MatchesAllOf(pkgTest.IsStatusOK, pkgTest.MatchesBody(test.HelloWorldText), abortOnTimeout(ctx))),
				"WaitForEndpointToServeText",
				test.ServingFlags.ResolvableDomain)
			if err != nil {
				t.Errorf("WaitForEndpointState(expected text) = %v", err)
				return fmt.Errorf("WaitForEndpointState(expected text) failed: %w", err)
			}

			// Record the time it took to get back a 200 with the expected text.
			latencies.Add("time-to-200", start)

			// Start probing the domain until the test is complete.
			pm.Spawn(url)

			t.Logf("%s is ready.", names.Service)
			return nil
		})
	}

	// Wait for all of the service creations to complete (possibly in failure),
	// and signal the done channel.
	doneCh := make(chan error)
	go func() {
		defer close(doneCh)
		if err := wg.Wait(); err != nil {
			doneCh <- err
		}
	}()

	for {
		// As services get created, add logic to clean them up.
		// When all of the creations have finished, then stop all of the active probers
		// and check our SLIs against our SLOs.
		// All of this has to finish within the configured timeout.
		select {
		case names := <-cleanupCh:
			t.Logf("Added %v to cleanup routine.", names)
			test.CleanupOnInterrupt(func() { test.TearDown(clients, names) })
			defer test.TearDown(clients, names)

		case err := <-doneCh:
			if err != nil {
				// If we don't do this first, then we'll see tons of 503s from the ongoing probes
				// as we tear down the things they are probing.
				defer pm.Stop()
				t.Fatalf("Unexpected error: %v", err)
			}

			// This ProberManager implementation waits for minProbes before actually stopping.
			if err := pm.Stop(); err != nil {
				t.Fatalf("Stop() = %v", err)
			}
			// Check each of the local SLOs
			pm.Foreach(func(u *url.URL, p test.Prober) {
				if err := test.CheckSLO(localSLO, u.String(), p); err != nil {
					t.Errorf("CheckSLO() = %v", err)
				}
			})
			// Check the global SLO
			if err := test.CheckSLO(globalSLO, "aggregate", pm); err != nil {
				t.Errorf("CheckSLO() = %v", err)
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"

	"github.com/go-redis/redis"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	hooks      []Hook
	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	cd.beforeSet(item)
	_, err := cd.setItem(item)
	cd.afterSet(item)
	return err
}

func (cd *Codec) setItem(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(nil, key, object)
}

func (cd *Codec) GetContext(c context.Context, key string, object interface{}) error {
	return cd.get(c, key, object)
}

func (cd *Codec) get(c context.Context, key string, object interface{}) error {
	cd.beforeGet(c, key, object)
	err := cd._get(key, object, false)
	cd.afterGet(c, key, object)
	return err
}

func (cd *Codec) _get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	cd.beforeOnce(item)
	err := cd.once(item)
	cd.afterOnce(item)
	return err
}

func (cd *Codec) once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd.Delete(item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.setItem(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.delete(nil, key)
}

func (cd *Codec) DeleteContext(c context.Context, key string) error {
	return cd.delete(c, key)
}

func (cd *Codec) delete(c context.Context, key string) error {
	cd.beforeDelete(c, key)
	err := cd._delete(key)
	cd.afterDelete(c, key)
	return err
}

func (cd *Codec) _delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

type Hook interface {
	BeforeSet(item *Item)
	AfterSet(item *Item)

	BeforeGet(c context.Context, key string, object interface{})
	AfterGet(c context.Context, key string, object interface{})

	BeforeDelete(c context.Context, key string)
	AfterDelete(c context.Context, key string)

	BeforeOnce(item *Item)
	AfterOnce(item *Item)
}

func (cd *Codec) AddHook(h Hook) {
	cd.hooks = append(cd.hooks, h)
}

func (cd *Codec) beforeSet(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeSet(item)
	}
}

func (cd *Codec) afterSet(item *Item) {
	for _, h := range cd.hooks {
		h.AfterSet(item)
	}
}

func (cd *Codec) beforeGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.BeforeGet(c, key, object)
	}
}

func (cd *Codec) afterGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.AfterGet(c, key, object)
	}
}

func (cd *Codec) beforeDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.BeforeDelete(c, key)
	}
}

func (cd *Codec) afterDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.AfterDelete(c, key)
	}
}

func (cd *Codec) beforeOnce(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeOnce(item)
	}
}

func (cd *Codec) afterOnce(item *Item) {
	for _, h := range cd.hooks {
		h.AfterOnce(item)
	}
}

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<commit_msg>Fix Once hook<commit_after>package cache

import (
	"context"
	"errors"
	"log"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/internal/lrucache"
	"github.com/go-redis/cache/internal/singleflight"

	"github.com/go-redis/redis"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	hooks      []Hook
	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	cd.beforeSet(item)
	_, err := cd.setItem(item)
	cd.afterSet(item)
	return err
}

func (cd *Codec) setItem(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(nil, key, object)
}

func (cd *Codec) GetContext(c context.Context, key string, object interface{}) error {
	return cd.get(c, key, object)
}

func (cd *Codec) get(c context.Context, key string, object interface{}) error {
	cd.beforeGet(c, key, object)
	err := cd._get(key, object, false)
	cd.afterGet(c, key, object)
	return err
}

func (cd *Codec) _get(key string, object interface{}, onlyLocalCache bool) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	cd.beforeOnce(item)
	err := cd.once(item)
	cd.afterOnce(item)
	return err
}

func (cd *Codec) once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd._delete(item.Key)
			return cd.once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.setItem(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.delete(nil, key)
}

func (cd *Codec) DeleteContext(c context.Context, key string) error {
	return cd.delete(c, key)
}

func (cd *Codec) delete(c context.Context, key string) error {
	cd.beforeDelete(c, key)
	err := cd._delete(key)
	cd.afterDelete(c, key)
	return err
}

func (cd *Codec) _delete(key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

type Hook interface {
	BeforeSet(item *Item)
	AfterSet(item *Item)

	BeforeGet(c context.Context, key string, object interface{})
	AfterGet(c context.Context, key string, object interface{})

	BeforeDelete(c context.Context, key string)
	AfterDelete(c context.Context, key string)

	BeforeOnce(item *Item)
	AfterOnce(item *Item)
}

func (cd *Codec) AddHook(h Hook) {
	cd.hooks = append(cd.hooks, h)
}

func (cd *Codec) beforeSet(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeSet(item)
	}
}

func (cd *Codec) afterSet(item *Item) {
	for _, h := range cd.hooks {
		h.AfterSet(item)
	}
}

func (cd *Codec) beforeGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.BeforeGet(c, key, object)
	}
}

func (cd *Codec) afterGet(c context.Context, key string, object interface{}) {
	for _, h := range cd.hooks {
		h.AfterGet(c, key, object)
	}
}

func (cd *Codec) beforeDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.BeforeDelete(c, key)
	}
}

func (cd *Codec) afterDelete(c context.Context, key string) {
	for _, h := range cd.hooks {
		h.AfterDelete(c, key)
	}
}

func (cd *Codec) beforeOnce(item *Item) {
	for _, h := range cd.hooks {
		h.BeforeOnce(item)
	}
}

func (cd *Codec) afterOnce(item *Item) {
	for _, h := range cd.hooks {
		h.AfterOnce(item)
	}
}

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<|endoftext|>