<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	"net/http"
	"github.com/gorilla/mux"
	"strconv"
	"net/url"
	"errors"
	"encoding/json"
	"github.com/gorilla/handlers"
	"time"
)

type VariantResponse struct {
	Success bool `json:"success"`
	Variants []Variant `json:"variants"`
	Total []int `json:"total"`
	Error string `json:"error"`
}

type Variant struct {
	VariantId  string `json:"id"`
	Chromosome string `json:"chromosome"`
	Start int64 `json:"start"`
	Reference string `json:"reference"`
	Alternate string `json:"alternate"`
	DbSNP string `json:"dbSNP"`
	AC int64 `json:"AC"`
	AF JsonNullFloat64 `json:"AF"`
	NCalled int64 `json:"nCalled"`
	NNotCalled int64 `json:"nNotCalled"`
	NHomRef int64 `json:"nHomRef"`
	NHet int64 `json:"nHet"`
	NHomVar int64 `json:"nHomVar"`
	AltType string `json:"altType"`
	Consequence string `json:"consequence"`
	GeneMapping string `json:"geneMapping"`
	EXACMAF JsonNullString `json:"exacMAF"`
}

type VariantQuery struct {
	Chrom string
	Start int
	End int
	Lim int
	Skip int
}

func main() {
	var dsn string
	var connectionCheck bool
	flag.StringVar(&dsn, "d", "root:root@tcp(127.0.0.1:3306)/variants", "mysql dsn: [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]")
	flag.BoolVar(&connectionCheck, "u", false, "Connect to database and exit")

	flag.Usage = func() {
		fmt.Printf("Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	db, err := sql.Open("mysql", dsn)
	check(err)
	defer db.Close()
	err = retry(10, 3 * time.Second,
		func() (error) {
			log.Println("connecting to mysql server...")
			return db.Ping()
		})
	check(err)
	fmt.Println("Connected to mysql server")
	if connectionCheck {
		return
	}

	r := mux.NewRouter().StrictSlash(true)
	r.Methods("GET, OPTIONS")
	r.HandleFunc("/", Index(r))
	r.HandleFunc("/variants", VariantsIndex(db))

	headersOk := handlers.AllowedHeaders([]string{"Content-Type"})
	originsOk := handlers.AllowedOrigins([]string{"*"})
	methodsOk := handlers.AllowedMethods([]string{"GET", "OPTIONS"})
	log.Fatal(http.ListenAndServe(":8080", handlers.CORS(originsOk, headersOk, methodsOk)(r)))
}

func retry(attempts int, sleep time.Duration, action func() (error)) (error) {
	var err error
	for i := 0; i < attempts; i++ {
		err = action()
		if err == nil {
			return err
		}
		time.Sleep(sleep)
		log.Println("retrying after error: ", err)
	}
	return err
}

func Index(router *mux.Router) http.HandlerFunc {
	fn := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Simple variant store API")
		router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
			t, err := route.GetPathTemplate()
			if err != nil {
				return err
			}
			fmt.Fprintln(w, t)
			return nil
		})
	}
	return http.HandlerFunc(fn)
}

func VariantsIndex(db *sql.DB) http.HandlerFunc {
	fn := func(w http.ResponseWriter, r *http.Request) {
		response := VariantResponse{true, []Variant{}, []int{}, ""}
		vq, err := parse(r.URL.Query())
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}

		count, err := countVariants(db, vq)
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}
		vs, err := queryVariants(db, vq, count)
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}
		response.Total = []int{count}
		response.Variants = vs
		json.NewEncoder(w).Encode(response)
	}
	return http.HandlerFunc(fn)
}

func parse (params url.Values) (VariantQuery, error) {
	errs := make([]string, 0)
	trackErrors := func(err error) {
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	c, err := paramValue(params, "chromosome")
	trackErrors(err)
	start, err := parseNatural(params, "positionStart")
	trackErrors(err)
	end, err := parseNatural(params, "positionEnd")
	trackErrors(err)
	lim, err := parseNatural(params, "limit")
	if err != nil {
		lim = 500
	}
	skip, err := parseNatural(params, "skip")
	if err != nil {
		skip = 0
	}

	if len(errs) > 0 {
		return VariantQuery{}, errors.New(strings.Join(errs, ", "))
	}
	return VariantQuery{c, start, end, lim, skip}, nil
}

func countVariants(db *sql.DB, vq VariantQuery) (int, error) {
	count := "SELECT COUNT(*) AS size FROM vs where chromosome=? and start<=? and start>=?"
	rows, err := db.Query(count, vq.Chrom, vq.End, vq.Start)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	n := 0
	for rows.Next() {
		if err := rows.Scan(&n); err != nil {
			return -1, err
		}
	}
	if err := rows.Err(); err != nil || n <= 0 {
		return -1, err
	}
	return n, nil
}

func queryVariants(db *sql.DB, vq VariantQuery, count int) ([]Variant, error) {
	query := "SELECT * FROM vs where chromosome=? and start<=? and start>=? LIMIT ?, ?"
	variants, err := db.Query(query, vq.Chrom, vq.End, vq.Start, vq.Skip, vq.Lim)
	if err != nil {
		return nil, err
	}
	var size = count - vq.Skip
	if size < 0 {
		size = 0
	}
	if size > vq.Lim {
		size = vq.Lim
	}
	vs := make([]Variant, size)
	i := 0
	for variants.Next() {
		var v Variant
		//var ignored []byte
		if err := variants.Scan(&v.VariantId, &v.Chromosome, &v.Start, &v.Reference, &v.Alternate, &v.DbSNP, &v.AC, &v.AF, &v.NCalled, &v.NNotCalled, &v.NHomRef, &v.NHet, &v.NHomVar, &v.AltType); err != nil {
			return nil, err
		}
		vs[i] = v
		i++
	}
	if err := variants.Err(); err != nil {
		return nil, err
	}
	return vs, nil
}

func errorResponse(response *VariantResponse, err string) {
	response.Success = false
	response.Error = fmt.Sprintf("Errors: %s", err)
}

func paramValue(params url.Values, s string) (string, error) {
	if len(params) <= 0 || len(params[s]) <= 0 {
		return "", errors.New(fmt.Sprintf("Could not parse %s", s))
	}
	return params[s][0], nil
}

func parseNatural(params url.Values, s string) (int, error) {
	v, err := paramValue(params, s)
	if err != nil {
		return -1, err
	}
	vi, err := strconv.Atoi(v)
	if err != nil {
		return -1, err
	}

	if vi < 0 {
		return -1, errors.New(fmt.Sprintf("%s is less than 0", s))
	}
	return vi, nil
}

func check(err error) {
	if err == nil {
		return
	}
	panic(err)
}

type JsonNullFloat64 struct {
	sql.NullFloat64
}

type JsonNullString struct {
	sql.NullString
}

func (v JsonNullFloat64) MarshalJSON() ([]byte, error) {
	if v.Valid {
		return json.Marshal(v.Float64)
	} else {
		return json.Marshal(nil)
	}
}

func (v JsonNullString) MarshalJSON() ([]byte, error) {
	if v.Valid {
		return json.Marshal(v.String)
	} else {
		return json.Marshal(nil)
	}
}<commit_msg>update api with new fields<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	"net/http"
	"github.com/gorilla/mux"
	"strconv"
	"net/url"
	"errors"
	"encoding/json"
	"github.com/gorilla/handlers"
	"time"
)

type VariantResponse struct {
	Success bool `json:"success"`
	Variants []Variant `json:"variants"`
	Total []int `json:"total"`
	Error string `json:"error"`
}

type Variant struct {
	VariantId  string `json:"id"`
	Chromosome string `json:"chromosome"`
	Start int64 `json:"start"`
	Reference string `json:"reference"`
	Alternate string `json:"alternate"`
	DbSNP JsonNullString `json:"dbSNP"`
	AC int64 `json:"AC"`
	AF JsonNullFloat64 `json:"AF"`
	NHomRef int64 `json:"nHomRef"`
	NHet int64 `json:"nHet"`
	NHomVar int64 `json:"nHomVar"`
	AltType string `json:"altType"`
	CATO JsonNullFloat64 `json:"cato"`
	Eigen JsonNullFloat64 `json:"eigen"`
	Sift JsonNullString `json:"sift"`
	HGVSc JsonNullString `json:"hgvsc"`
	HGVSp JsonNullString `json:"hgvsp"`
	PolyPhen JsonNullString `json:"polyphen"`
	TgpAF JsonNullFloat64 `json:"tgpAF"`
	HrcAF JsonNullFloat64 `json:"hrcAF"`
	GnomadAF JsonNullFloat64 `json:"gnomadAF"`
	Feature JsonNullString `json:"feature"`
	Consequences JsonNullString `json:"consequences"`
	Gene JsonNullString `json:"gene"`
	Clinvar JsonNullString `json:"clinvar"`
}

type VariantQuery struct {
	Chrom string
	Start int
	End int
	Lim int
	Skip int
}

func main() {
	var dsn string
	var connectionCheck bool
	flag.StringVar(&dsn, "d", "root:root@tcp(127.0.0.1:3306)/variants", "mysql dsn: [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]")
	flag.BoolVar(&connectionCheck, "u", false, "Connect to database and exit")

	flag.Usage = func() {
		fmt.Printf("Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	db, err := sql.Open("mysql", dsn)
	check(err)
	defer db.Close()
	err = retry(10, 3 * time.Second,
		func() (error) {
			log.Println("connecting to mysql server...")
			return db.Ping()
		})
	check(err)
	fmt.Println("Connected to mysql server")
	if connectionCheck {
		return
	}

	r := mux.NewRouter().StrictSlash(true)
	r.Methods("GET, OPTIONS")
	r.HandleFunc("/", Index(r))
	r.HandleFunc("/variants", VariantsIndex(db))

	headersOk := handlers.AllowedHeaders([]string{"Content-Type"})
	originsOk := handlers.AllowedOrigins([]string{"*"})
	methodsOk := handlers.AllowedMethods([]string{"GET", "OPTIONS"})
	log.Fatal(http.ListenAndServe(":8080", handlers.CORS(originsOk, headersOk, methodsOk)(r)))
}

func retry(attempts int, sleep time.Duration, action func() (error)) (error) {
	var err error
	for i := 0; i < attempts; i++ {
		err = action()
		if err == nil {
			return err
		}
		time.Sleep(sleep)
		log.Println("retrying after error: ", err)
	}
	return err
}

func Index(router *mux.Router) http.HandlerFunc {
	fn := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Simple variant store API")
		router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
			t, err := route.GetPathTemplate()
			if err != nil {
				return err
			}
			fmt.Fprintln(w, t)
			return nil
		})
	}
	return http.HandlerFunc(fn)
}

func VariantsIndex(db *sql.DB) http.HandlerFunc {
	fn := func(w http.ResponseWriter, r *http.Request) {
		response := VariantResponse{true, []Variant{}, []int{}, ""}
		vq, err := parse(r.URL.Query())
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}

		count, err := countVariants(db, vq)
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}
		vs, err := queryVariants(db, vq, count)
		if err != nil {
			errorResponse(&response, err.Error())
			json.NewEncoder(w).Encode(response)
			return
		}
		response.Total = []int{count}
		response.Variants = vs
		json.NewEncoder(w).Encode(response)
	}
	return http.HandlerFunc(fn)
}

func parse (params url.Values) (VariantQuery, error) {
	errs := make([]string, 0)
	trackErrors := func(err error) {
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	c, err := paramValue(params, "chromosome")
	trackErrors(err)
	start, err := parseNatural(params, "positionStart")
	trackErrors(err)
	end, err := parseNatural(params, "positionEnd")
	trackErrors(err)
	lim, err := parseNatural(params, "limit")
	if err != nil {
		lim = 500
	}
	skip, err := parseNatural(params, "skip")
	if err != nil {
		skip = 0
	}

	if len(errs) > 0 {
		return VariantQuery{}, errors.New(strings.Join(errs, ", "))
	}
	return VariantQuery{c, start, end, lim, skip}, nil
}

func countVariants(db *sql.DB, vq VariantQuery) (int, error) {
	count := "SELECT COUNT(*) AS size FROM vs where chromosome=? and start<=? and start>=?"
	rows, err := db.Query(count, vq.Chrom, vq.End, vq.Start)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	n := 0
	for rows.Next() {
		if err := rows.Scan(&n); err != nil {
			return -1, err
		}
	}
	if err := rows.Err(); err != nil || n <= 0 {
		return -1, err
	}
	return n, nil
}

func queryVariants(db *sql.DB, vq VariantQuery, count int) ([]Variant, error) {
	query := "SELECT * FROM vs where chromosome=? and start<=? and start>=? LIMIT ?, ?"
	variants, err := db.Query(query, vq.Chrom, vq.End, vq.Start, vq.Skip, vq.Lim)
	if err != nil {
		return nil, err
	}
	var size = count - vq.Skip
	if size < 0 {
		size = 0
	}
	if size > vq.Lim {
		size = vq.Lim
	}
	vs := make([]Variant, size)
	i := 0
	for variants.Next() {
		var v Variant
		//var ignored []byte
		if err := variants.Scan(&v.VariantId,
								&v.Chromosome,
								&v.Start,
								&v.Reference,
								&v.Alternate,
								&v.DbSNP,
								&v.AC,
								&v.AF,
								&v.NHomRef,
								&v.NHet,
								&v.NHomVar,
								&v.AltType,
								&v.CATO,
								&v.Eigen,
								&v.Sift,
								&v.HGVSc,
								&v.HGVSp,
								&v.PolyPhen,
								&v.TgpAF,
								&v.HrcAF,
								&v.GnomadAF,
								&v.Feature,
								&v.Consequences,
								&v.Gene,
								&v.Clinvar); err != nil {
			return nil, err
		}
		vs[i] = v
		i++
	}
	if err := variants.Err(); err != nil {
		return nil, err
	}
	return vs, nil
}

func errorResponse(response *VariantResponse, err string) {
	response.Success = false
	response.Error = fmt.Sprintf("Errors: %s", err)
}

func paramValue(params url.Values, s string) (string, error) {
	if len(params) <= 0 || len(params[s]) <= 0 {
		return "", errors.New(fmt.Sprintf("Could not parse %s", s))
	}
	return params[s][0], nil
}

func parseNatural(params url.Values, s string) (int, error) {
	v, err := paramValue(params, s)
	if err != nil {
		return -1, err
	}
	vi, err := strconv.Atoi(v)
	if err != nil {
		return -1, err
	}

	if vi < 0 {
		return -1, errors.New(fmt.Sprintf("%s is less than 0", s))
	}
	return vi, nil
}

func check(err error) {
	if err == nil {
		return
	}
	panic(err)
}

type JsonNullFloat64 struct {
	sql.NullFloat64
}

type JsonNullString struct {
	sql.NullString
}

func (v JsonNullFloat64) MarshalJSON() ([]byte, error) {
	if v.Valid {
		return json.Marshal(v.Float64)
	} else {
		return json.Marshal(nil)
	}
}

func (v JsonNullString) MarshalJSON() ([]byte, error) {
	if v.Valid {
		return json.Marshal(v.String)
	} else {
		return json.Marshal(nil)
	}
}<|endoftext|>


<commit_before>package httpclient

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
)

// A transport that prints request and response

type LoggingTransport struct {
	t            *http.Transport
	requestBody  bool
	responseBody bool
}

func (lt *LoggingTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	dreq, _ := httputil.DumpRequest(req, lt.requestBody)
	fmt.Println("REQUEST:", strconv.Quote(string(dreq)))
	fmt.Println("")

	resp, err = lt.t.RoundTrip(req)
	if err != nil {
		if lt.requestBody {
			// don't print the body twice
			dreq, _ = httputil.DumpRequest(req, false)
		}
		fmt.Println("ERROR:", err, "REQUEST:", strconv.Quote(string(dreq)))
	} else {
		dresp, _ := httputil.DumpResponse(resp, lt.responseBody)
		fmt.Println("RESPONSE:", string(dresp))
	}

	fmt.Println("")
	return
}

func (lt *LoggingTransport) CancelRequest(req *http.Request) {
	dreq, _ := httputil.DumpRequest(req, false)
	fmt.Println("CANCEL REQUEST:", strconv.Quote(string(dreq)))
	lt.t.CancelRequest(req)
}

// Enable logging requests/response headers
//
// if requestBody == true, also log request body
// if responseBody == true, also log response body
func StartLogging(requestBody, responseBody bool) {
	http.DefaultTransport = &LoggingTransport{&http.Transport{}, requestBody, responseBody}
}

// Disable logging requests/responses
func StopLogging() {
	http.DefaultTransport = &http.Transport{}
}

// A Reader that "logs" progress

type ProgressReader struct {
	r         io.Reader
	c         [1]byte
	threshold int
	curr      int
}

func NewProgressReader(r io.Reader, c byte, threshold int) *ProgressReader {
	if c == 0 {
		c = '.'
	}
	if threshold <= 0 {
		threshold = 10240
	}
	p := &ProgressReader{r: r, c: [1]byte{c}, threshold: threshold, curr: 0}
	return p
}

func (p *ProgressReader) Read(b []byte) (int, error) {
	n, err := p.r.Read(b)

	p.curr += n

	if err == io.EOF {
		os.Stdout.Write([]byte{'\n'})
		os.Stdout.Sync()
	} else if p.curr >= p.threshold {
		p.curr -= p.threshold

		os.Stdout.Write(p.c[:])
		os.Stdout.Sync()
	}

	return n, err
}

func (p *ProgressReader) Close() error {
	if rc, ok := p.r.(io.ReadCloser); ok {
		return rc.Close()
	} else {
		return nil
	}
}

type DebugLog bool

func (d DebugLog) Println(args ...interface{}) {
	if d {
		log.Println(args...)
	}
}

func (d DebugLog) Printf(fmt string, args ...interface{}) {
	if d {
		log.Printf(fmt, args...)
	}
}
<commit_msg>Just in case we have a response with an error<commit_after>package httpclient

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
)

// A transport that prints request and response

type LoggingTransport struct {
	t            *http.Transport
	requestBody  bool
	responseBody bool
}

func (lt *LoggingTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	dreq, _ := httputil.DumpRequest(req, lt.requestBody)
	fmt.Println("REQUEST:", strconv.Quote(string(dreq)))
	fmt.Println("")

	resp, err = lt.t.RoundTrip(req)
	if err != nil {
		if lt.requestBody {
			// don't print the body twice
			dreq, _ = httputil.DumpRequest(req, false)
		}
		fmt.Println("ERROR:", err, "REQUEST:", strconv.Quote(string(dreq)))
	}
	if resp != nil {
		dresp, _ := httputil.DumpResponse(resp, lt.responseBody)
		fmt.Println("RESPONSE:", string(dresp))
	}

	fmt.Println("")
	return
}

func (lt *LoggingTransport) CancelRequest(req *http.Request) {
	dreq, _ := httputil.DumpRequest(req, false)
	fmt.Println("CANCEL REQUEST:", strconv.Quote(string(dreq)))
	lt.t.CancelRequest(req)
}

// Enable logging requests/response headers
//
// if requestBody == true, also log request body
// if responseBody == true, also log response body
func StartLogging(requestBody, responseBody bool) {
	http.DefaultTransport = &LoggingTransport{&http.Transport{}, requestBody, responseBody}
}

// Disable logging requests/responses
func StopLogging() {
	http.DefaultTransport = &http.Transport{}
}

// A Reader that "logs" progress

type ProgressReader struct {
	r         io.Reader
	c         [1]byte
	threshold int
	curr      int
}

func NewProgressReader(r io.Reader, c byte, threshold int) *ProgressReader {
	if c == 0 {
		c = '.'
	}
	if threshold <= 0 {
		threshold = 10240
	}
	p := &ProgressReader{r: r, c: [1]byte{c}, threshold: threshold, curr: 0}
	return p
}

func (p *ProgressReader) Read(b []byte) (int, error) {
	n, err := p.r.Read(b)

	p.curr += n

	if err == io.EOF {
		os.Stdout.Write([]byte{'\n'})
		os.Stdout.Sync()
	} else if p.curr >= p.threshold {
		p.curr -= p.threshold

		os.Stdout.Write(p.c[:])
		os.Stdout.Sync()
	}

	return n, err
}

func (p *ProgressReader) Close() error {
	if rc, ok := p.r.(io.ReadCloser); ok {
		return rc.Close()
	} else {
		return nil
	}
}

type DebugLog bool

func (d DebugLog) Println(args ...interface{}) {
	if d {
		log.Println(args...)
	}
}

func (d DebugLog) Printf(fmt string, args ...interface{}) {
	if d {
		log.Printf(fmt, args...)
	}
}
<|endoftext|>


<commit_before>// Discordgo - Discord bindings for Go
// Available at https://github.com/bwmarrin/discordgo

// Copyright 2015-2016 Bruce Marriner <bruce@sqls.net>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains code related to discordgo package logging

package discordgo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"runtime"
	"strings"
)

const (

	// Critical Errors that could lead to data loss or panic
	// Only errors that would not be returned to a calling function
	LogError int = iota

	// Very abnormal events.
	// Errors that are also returend to a calling function.
	LogWarning

	// Normal non-error activity
	// Generally, not overly spammy events
	LogInformational

	// Very detailed non-error activity
	// All HTTP/Websocket packets.
	// Very spammy and will impact performance
	LogDebug
)

// msglog provides package wide logging consistancy for discordgo
// the format, a...  portion this command follows that of fmt.Printf
//   msgL   : LogLevel of the message
//   caller : 1 + the number of callers away from the message source
//   format : Printf style message format
//   a ...  : comma seperated list of values to pass
func msglog(msgL, caller int, format string, a ...interface{}) {

	pc, file, line, _ := runtime.Caller(caller)

	files := strings.Split(file, "/")
	file = files[len(files)-1]

	name := runtime.FuncForPC(pc).Name()
	fns := strings.Split(name, ".")
	name = fns[len(fns)-1]

	msg := fmt.Sprintf(format, a...)

	log.Printf("[DG%d] %s:%d:%s() %s\n", msgL, file, line, name, msg)
}

// helper function that wraps msglog for the Session struct
// This adds a check to insure the message is only logged
// if the session log level is equal or higher than the
// message log level
func (s *Session) log(msgL int, format string, a ...interface{}) {

	if s.Debug { // Deprecated
		s.LogLevel = LogDebug
	}

	if msgL > s.LogLevel {
		return
	}

	msglog(msgL, 2, format, a...)
}

// helper function that wraps msglog for the VoiceConnection struct
// This adds a check to insure the message is only logged
// if the voice connection log level is equal or higher than the
// message log level
func (v *VoiceConnection) log(msgL int, format string, a ...interface{}) {

	if v.Debug { // Deprecated
		v.LogLevel = LogDebug
	}

	if msgL > v.LogLevel {
		return
	}

	msglog(msgL, 2, format, a...)
}

// printJSON is a helper function to display JSON data in a easy to read format.
func printJSON(body []byte) {
	var prettyJSON bytes.Buffer
	error := json.Indent(&prettyJSON, body, "", "\t")
	if error != nil {
		log.Print("JSON parse error: ", error)
	}
	log.Println(string(prettyJSON.Bytes()))
}
<commit_msg>Linting comments, remove deadcode.<commit_after>// Discordgo - Discord bindings for Go
// Available at https://github.com/bwmarrin/discordgo

// Copyright 2015-2016 Bruce Marriner <bruce@sqls.net>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains code related to discordgo package logging

package discordgo

import (
	"fmt"
	"log"
	"runtime"
	"strings"
)

const (

	// LogError level is used for critical errors that could lead to data loss
	// or panic that would not be returned to a calling function.
	LogError int = iota

	// LogWarning level is used for very abnormal events and errors that are
	// also returend to a calling function.
	LogWarning

	// LogInformational level is used for normal non-error activity
	LogInformational

	// LogDebug level is for very detailed non-error activity.  This is
	// very spammy and will impact performance.
	LogDebug
)

// msglog provides package wide logging consistancy for discordgo
// the format, a...  portion this command follows that of fmt.Printf
//   msgL   : LogLevel of the message
//   caller : 1 + the number of callers away from the message source
//   format : Printf style message format
//   a ...  : comma seperated list of values to pass
func msglog(msgL, caller int, format string, a ...interface{}) {

	pc, file, line, _ := runtime.Caller(caller)

	files := strings.Split(file, "/")
	file = files[len(files)-1]

	name := runtime.FuncForPC(pc).Name()
	fns := strings.Split(name, ".")
	name = fns[len(fns)-1]

	msg := fmt.Sprintf(format, a...)

	log.Printf("[DG%d] %s:%d:%s() %s\n", msgL, file, line, name, msg)
}

// helper function that wraps msglog for the Session struct
// This adds a check to insure the message is only logged
// if the session log level is equal or higher than the
// message log level
func (s *Session) log(msgL int, format string, a ...interface{}) {

	if s.Debug { // Deprecated
		s.LogLevel = LogDebug
	}

	if msgL > s.LogLevel {
		return
	}

	msglog(msgL, 2, format, a...)
}

// helper function that wraps msglog for the VoiceConnection struct
// This adds a check to insure the message is only logged
// if the voice connection log level is equal or higher than the
// message log level
func (v *VoiceConnection) log(msgL int, format string, a ...interface{}) {

	if v.Debug { // Deprecated
		v.LogLevel = LogDebug
	}

	if msgL > v.LogLevel {
		return
	}

	msglog(msgL, 2, format, a...)
}

// printJSON is a helper function to display JSON data in a easy to read format.
/* NOT USED ATM
func printJSON(body []byte) {
	var prettyJSON bytes.Buffer
	error := json.Indent(&prettyJSON, body, "", "\t")
	if error != nil {
		log.Print("JSON parse error: ", error)
	}
	log.Println(string(prettyJSON.Bytes()))
}
*/
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"net/url"
	"strings"

	log "github.com/lxc/lxd/shared/log15"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/logger"
)

// RestServer creates an http.Server capable of handling requests against the LXD REST
// API endpoint.
func RestServer(d *Daemon) *http.Server {
	/* Setup the web server */
	mux := mux.NewRouter()
	mux.StrictSlash(false)

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		SyncResponse(true, []string{"/1.0"}).Render(w)
	})

	for endpoint, f := range d.gateway.HandlerFuncs() {
		mux.HandleFunc(endpoint, f)
	}

	for _, c := range api10 {
		d.createCmd(mux, "1.0", c)
	}

	for _, c := range apiInternal {
		d.createCmd(mux, "internal", c)
	}

	mux.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger.Info("Sending top level 404", log.Ctx{"url": r.URL})
		w.Header().Set("Content-Type", "application/json")
		NotFound(nil).Render(w)
	})

	return &http.Server{Handler: &lxdHttpServer{r: mux, d: d}}
}

type lxdHttpServer struct {
	r *mux.Router
	d *Daemon
}

func (s *lxdHttpServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// Set CORS headers, unless this is an internal or gRPC request.
	if !strings.HasPrefix(req.URL.Path, "/internal") && !strings.HasPrefix(req.URL.Path, "/protocol.SQL") {
		<-s.d.setupChan
		err := s.d.cluster.Transaction(func(tx *db.ClusterTx) error {
			config, err := cluster.ConfigLoad(tx)
			if err != nil {
				return err
			}
			setCORSHeaders(rw, req, config)
			return nil
		})
		if err != nil {
			response := SmartError(err)
			response.Render(rw)
			return
		}
	}

	// OPTIONS request don't need any further processing
	if req.Method == "OPTIONS" {
		return
	}

	// Call the original server
	s.r.ServeHTTP(rw, req)
}

func setCORSHeaders(rw http.ResponseWriter, req *http.Request, config *cluster.Config) {
	allowedOrigin := config.HTTPSAllowedOrigin()
	origin := req.Header.Get("Origin")
	if allowedOrigin != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Origin", allowedOrigin)
	}

	allowedMethods := config.HTTPSAllowedMethods()
	if allowedMethods != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Methods", allowedMethods)
	}

	allowedHeaders := config.HTTPSAllowedHeaders()
	if allowedHeaders != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Headers", allowedHeaders)
	}

	allowedCredentials := config.HTTPSAllowedCredentials()
	if allowedCredentials {
		rw.Header().Set("Access-Control-Allow-Credentials", "true")
	}
}

// Return true if this an API request coming from a cluster node that is
// notifying us of some user-initiated API request that needs some action to be
// taken on this node as well.
func isClusterNotification(r *http.Request) bool {
	return r.Header.Get("User-Agent") == "lxd-cluster-notifier"
}

// Extract the project query parameter from the given request.
func projectParam(request *http.Request) string {
	project := queryParam(request, "project")
	if project == "" {
		project = "default"
	}
	return project
}

// Extract the given query parameter directly from the URL, never from an
// encoded body.
func queryParam(request *http.Request, key string) string {
	var values url.Values
	var err error

	if request.URL != nil {
		values, err = url.ParseQuery(request.URL.RawQuery)
		if err != nil {
			logger.Warnf("Failed to parse query string %q: %v", request.URL.RawQuery, err)
			return ""
		}
	}

	if values == nil {
		values = make(url.Values)
	}

	return values.Get(key)
}
<commit_msg>api: Updates handler funcs to take nodeRefreshFunc<commit_after>package main

import (
	"net/http"
	"net/url"
	"strings"

	log "github.com/lxc/lxd/shared/log15"

	"github.com/gorilla/mux"
	"github.com/lxc/lxd/lxd/cluster"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/shared/logger"
)

// RestServer creates an http.Server capable of handling requests against the LXD REST
// API endpoint.
func RestServer(d *Daemon) *http.Server {
	/* Setup the web server */
	mux := mux.NewRouter()
	mux.StrictSlash(false)

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		SyncResponse(true, []string{"/1.0"}).Render(w)
	})

	for endpoint, f := range d.gateway.HandlerFuncs(d.NodeRefreshTask) {
		mux.HandleFunc(endpoint, f)
	}

	for _, c := range api10 {
		d.createCmd(mux, "1.0", c)
	}

	for _, c := range apiInternal {
		d.createCmd(mux, "internal", c)
	}

	mux.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger.Info("Sending top level 404", log.Ctx{"url": r.URL})
		w.Header().Set("Content-Type", "application/json")
		NotFound(nil).Render(w)
	})

	return &http.Server{Handler: &lxdHttpServer{r: mux, d: d}}
}

type lxdHttpServer struct {
	r *mux.Router
	d *Daemon
}

func (s *lxdHttpServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// Set CORS headers, unless this is an internal or gRPC request.
	if !strings.HasPrefix(req.URL.Path, "/internal") && !strings.HasPrefix(req.URL.Path, "/protocol.SQL") {
		<-s.d.setupChan
		err := s.d.cluster.Transaction(func(tx *db.ClusterTx) error {
			config, err := cluster.ConfigLoad(tx)
			if err != nil {
				return err
			}
			setCORSHeaders(rw, req, config)
			return nil
		})
		if err != nil {
			response := SmartError(err)
			response.Render(rw)
			return
		}
	}

	// OPTIONS request don't need any further processing
	if req.Method == "OPTIONS" {
		return
	}

	// Call the original server
	s.r.ServeHTTP(rw, req)
}

func setCORSHeaders(rw http.ResponseWriter, req *http.Request, config *cluster.Config) {
	allowedOrigin := config.HTTPSAllowedOrigin()
	origin := req.Header.Get("Origin")
	if allowedOrigin != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Origin", allowedOrigin)
	}

	allowedMethods := config.HTTPSAllowedMethods()
	if allowedMethods != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Methods", allowedMethods)
	}

	allowedHeaders := config.HTTPSAllowedHeaders()
	if allowedHeaders != "" && origin != "" {
		rw.Header().Set("Access-Control-Allow-Headers", allowedHeaders)
	}

	allowedCredentials := config.HTTPSAllowedCredentials()
	if allowedCredentials {
		rw.Header().Set("Access-Control-Allow-Credentials", "true")
	}
}

// Return true if this an API request coming from a cluster node that is
// notifying us of some user-initiated API request that needs some action to be
// taken on this node as well.
func isClusterNotification(r *http.Request) bool {
	return r.Header.Get("User-Agent") == "lxd-cluster-notifier"
}

// Extract the project query parameter from the given request.
func projectParam(request *http.Request) string {
	project := queryParam(request, "project")
	if project == "" {
		project = "default"
	}
	return project
}

// Extract the given query parameter directly from the URL, never from an
// encoded body.
func queryParam(request *http.Request, key string) string {
	var values url.Values
	var err error

	if request.URL != nil {
		values, err = url.ParseQuery(request.URL.RawQuery)
		if err != nil {
			logger.Warnf("Failed to parse query string %q: %v", request.URL.RawQuery, err)
			return ""
		}
	}

	if values == nil {
		values = make(url.Values)
	}

	return values.Get(key)
}
<|endoftext|>


<commit_before>package incident

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"go.skia.org/infra/am/go/note"
	"go.skia.org/infra/am/go/silence"
	"go.skia.org/infra/go/alerts"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/ds/testutil"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/testutils/unittest"
)

func TestAreIncidentsFlaky(t *testing.T) {
	unittest.SmallTest(t)

	now := time.Now().Unix()

	// Add 10 incidents with duration = 10 mins and 10 with duration < 10 mins.
	incidents := []Incident{}
	for i := 0; i < 10; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 600})
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because 50% are flaky not matching the specified 60%.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 0.60))
	// Should be flaky because 50% are flaky matching the specifeid 50%.
	assert.True(t, AreIncidentsFlaky(incidents, 10, 600, 0.50))

	// Add 4 incidents with duration = 10 mins and 6 with duration < 10 mins.
	incidents = []Incident{}
	for i := 0; i < 4; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 600})
	}
	for i := 0; i < 6; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because 60% are flaky not matching the specified 61%.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 0.61))
	// Should be flaky because 60% are flaky matching the specified 60%.
	assert.True(t, AreIncidentsFlaky(incidents, 10, 600, 0.60))

	// Add 9 incidents with duration < 10 mins.
	incidents = []Incident{}
	for i := 0; i < 9; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because num incidents did not meet the specified threshold of 10.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 1.00))
	// Should be flaky because num incidents do meet the specified threshold of 9 and 100%.
	assert.True(t, AreIncidentsFlaky(incidents, 9, 600, 1.00))
}

func TestIsSilenced(t *testing.T) {
	unittest.SmallTest(t)

	i := Incident{
		Params: map[string]string{
			"foo": "2123",
			"bar": "aa",
		},
	}

	// Test with simple silences

	silences := []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"2123"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"abc", "xyz", "2123"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"blah": []string{"abc", "xyz", "2123"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"abc", "xyz", "32"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"blah": []string{"abc", "xyz", "2123"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))

	// Test with ignore.
	silences = []silence.Silence{
		{
			Active:   false,
			ParamSet: paramtools.ParamSet{"foo": []string{"2123"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))
	assert.True(t, i.IsSilenced(silences, false))

	// Tests with regexes.

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"2.*"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"aa"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))

	// Test with paramset with both regex and non-regex by adding another value to existing key.
	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb", "aa", "cc"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	// Test with silence that does not apply.
	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb", "aa", "cc"}, "blah": []string{"abc"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))
}

func TestAlertArrival(t *testing.T) {
	unittest.LargeTest(t)

	cleanup := testutil.InitDatastore(t, ds.INCIDENT_AM, ds.INCIDENT_ACTIVE_PARENT_AM)
	defer cleanup()

	st := NewStore(ds.DS, []string{"ignore"})

	m := map[string]string{
		"ignore":     "pod_123",
		alerts.TYPE:  alerts.TYPE_ALERTS,
		alerts.STATE: alerts.STATE_ACTIVE,
		ALERT_NAME:   "BotUnemployed",
		"bot":        "skia-rpi-064",
		CATEGORY:     "infra",
	}
	a, err := st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, a.Active)
	assert.NotEqual(t, "", a.Key)

	// A second alert arrival with the same config doesn't create
	// a new Incident.
	a, err = st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, a.Active)
	assert.NotEqual(t, "", a.Key)

	all, err := st.GetAll()
	assert.NoError(t, err)
	assert.Equal(t, 1, len(all))

	// But an alert arrival with a different config does create
	// a new incident.
	m[ALERT_NAME] = "BotMissing"
	delete(m, ID)
	b, err := st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, b.Active)
	assert.NotEqual(t, "", b.Key)
	assert.NotEqual(t, a.ID, b.ID)

	// Add a note.
	bb, err := st.AddNote(b.Key, note.Note{
		Text:   "Stuff happened.",
		Author: "fred@example.com",
		TS:     time.Now().Unix(),
	})
	assert.NoError(t, err)
	assert.Equal(t, "Stuff happened.", bb.Notes[0].Text)
	assert.Equal(t, b.Key, bb.Key)

	// Fail to add note, bad key.
	_, err = st.AddNote("badkey", note.Note{})
	assert.Error(t, err)

	// Delete note, bad index.
	_, err = st.DeleteNote(b.Key, 1)
	assert.Error(t, err)

	// Delete note.
	b, err = st.DeleteNote(b.Key, 0)
	assert.NoError(t, err)
	assert.Equal(t, 0, len(b.Notes))

	// Assign
	b, err = st.Assign(b.Key, "barney@example.org")
	assert.NoError(t, err)
	assert.Equal(t, "barney@example.org", b.Params[ASSIGNED_TO])

	m[alerts.STATE] = alerts.STATE_RESOLVED
	b, err = st.AlertArrival(m)
	assert.NoError(t, err)
	assert.False(t, b.Active)

	// Archive
	b, err = st.Archive(b.Key)
	assert.NoError(t, err)

	recent, err := st.GetRecentlyResolvedForID(b.ID, "")
	assert.NoError(t, err)
	assert.Len(t, recent, 1)

	recent, err = st.GetRecentlyResolvedForID(b.ID, b.Key)
	assert.NoError(t, err)
	assert.Len(t, recent, 0)
}

func TestIdForAlert(t *testing.T) {
	unittest.LargeTest(t)
	m := map[string]string{
		"__name__":   "ALERTS",
		"alertname":  "BotMissing",
		"alertstate": "firing",
		"bot":        "skia-rpi-064",
		"category":   "infra",
		"instance":   "skia-datahopper2:20000",
		"job":        "datahopper",
		"pool":       "Skia",
		"severity":   "critical",
		"swarming":   "chromium-swarm.appspot.com",
	}
	st := NewStore(nil, []string{})

	id1, err := st.idForAlert(m)
	assert.NoError(t, err)
	id2, err := st.idForAlert(m)
	assert.NoError(t, err)
	assert.Equal(t, id1, id2)

	m[alerts.STATE] = alerts.STATE_ACTIVE
	id2, err = st.idForAlert(m)
	assert.NoError(t, err)
	assert.Equal(t, id1, id2)
}

func TestGetRegexesToOwners(t *testing.T) {
	unittest.SmallTest(t)

	ownersRegexesStr := "owner1:abbr_regex1,abbr_regex2;owner2:abbr_regex3"
	m1, err := getRegexesToOwners(ownersRegexesStr)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(m1))
	assert.Equal(t, "owner1", m1["abbr_regex1"])
	assert.Equal(t, "owner1", m1["abbr_regex2"])
	assert.Equal(t, "owner2", m1["abbr_regex3"])

	// Test badly formed regex with missing owner1.
	m2, err := getRegexesToOwners("abbr_regex1,abbr2_regex;owner2:abbr_regex3")
	assert.Error(t, err)
	assert.Nil(t, m2)
}

func TestGetOwnerIfMatch(t *testing.T) {
	unittest.SmallTest(t)

	// Test matches.
	ownersRegexesStr := "superman@krypton.com:Bizarro.*,^Kryptonite.*Asteroid.*$;batman@gotham.com:Joker.*"

	ownerTest1, err := getOwnerIfMatch(ownersRegexesStr, "something Bizarro something")
	assert.NoError(t, err)
	assert.Equal(t, "superman@krypton.com", ownerTest1)

	ownerTest2, err := getOwnerIfMatch(ownersRegexesStr, "Kryptonite really big Asteroid thing")
	assert.NoError(t, err)
	assert.Equal(t, "superman@krypton.com", ownerTest2)

	ownerTest3, err := getOwnerIfMatch(ownersRegexesStr, "Joker is here!!!")
	assert.NoError(t, err)
	assert.Equal(t, "batman@gotham.com", ownerTest3)

	ownerTest4, err := getOwnerIfMatch(ownersRegexesStr, "Joker")
	assert.NoError(t, err)
	assert.Equal(t, "batman@gotham.com", ownerTest4)

	// Test misses.
	ownerMiss1, err := getOwnerIfMatch(ownersRegexesStr, "bizarro")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss1)

	ownerMiss2, err := getOwnerIfMatch(ownersRegexesStr, "wrong start Kryptonite really big Asteroid thing")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss2)

	ownerMiss3, err := getOwnerIfMatch(ownersRegexesStr, "joker")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss3)

	// Test badly formed regex.
	badRegex := "superman@krypton:.***"
	ownerBadTest, err := getOwnerIfMatch(badRegex, "Anything")
	assert.Error(t, err)
	assert.Equal(t, "", ownerBadTest)
}
<commit_msg>[am] Disable test that flakes due to cloud emulator<commit_after>package incident

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"go.skia.org/infra/am/go/note"
	"go.skia.org/infra/am/go/silence"
	"go.skia.org/infra/go/alerts"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/ds/testutil"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/testutils/unittest"
)

func TestAreIncidentsFlaky(t *testing.T) {
	unittest.SmallTest(t)

	now := time.Now().Unix()

	// Add 10 incidents with duration = 10 mins and 10 with duration < 10 mins.
	incidents := []Incident{}
	for i := 0; i < 10; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 600})
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because 50% are flaky not matching the specified 60%.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 0.60))
	// Should be flaky because 50% are flaky matching the specifeid 50%.
	assert.True(t, AreIncidentsFlaky(incidents, 10, 600, 0.50))

	// Add 4 incidents with duration = 10 mins and 6 with duration < 10 mins.
	incidents = []Incident{}
	for i := 0; i < 4; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 600})
	}
	for i := 0; i < 6; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because 60% are flaky not matching the specified 61%.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 0.61))
	// Should be flaky because 60% are flaky matching the specified 60%.
	assert.True(t, AreIncidentsFlaky(incidents, 10, 600, 0.60))

	// Add 9 incidents with duration < 10 mins.
	incidents = []Incident{}
	for i := 0; i < 9; i++ {
		incidents = append(incidents, Incident{LastSeen: now, Start: now - 10})
	}
	// Should not be flaky because num incidents did not meet the specified threshold of 10.
	assert.False(t, AreIncidentsFlaky(incidents, 10, 600, 1.00))
	// Should be flaky because num incidents do meet the specified threshold of 9 and 100%.
	assert.True(t, AreIncidentsFlaky(incidents, 9, 600, 1.00))
}

func TestIsSilenced(t *testing.T) {
	unittest.SmallTest(t)

	i := Incident{
		Params: map[string]string{
			"foo": "2123",
			"bar": "aa",
		},
	}

	// Test with simple silences

	silences := []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"2123"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"abc", "xyz", "2123"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"blah": []string{"abc", "xyz", "2123"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"abc", "xyz", "32"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"blah": []string{"abc", "xyz", "2123"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))

	// Test with ignore.
	silences = []silence.Silence{
		{
			Active:   false,
			ParamSet: paramtools.ParamSet{"foo": []string{"2123"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))
	assert.True(t, i.IsSilenced(silences, false))

	// Tests with regexes.

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"2.*"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"aa"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))

	// Test with paramset with both regex and non-regex by adding another value to existing key.
	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb", "aa", "cc"}},
		},
	}
	assert.True(t, i.IsSilenced(silences, true))

	// Test with silence that does not apply.
	silences = []silence.Silence{
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"foo": []string{"3.*"}},
		},
		{
			Active:   true,
			ParamSet: paramtools.ParamSet{"bar": []string{"bb", "aa", "cc"}, "blah": []string{"abc"}},
		},
	}
	assert.False(t, i.IsSilenced(silences, true))
}

func TestAlertArrival(t *testing.T) {
	unittest.ManualTest(t) // Disabled due to flaky cloud emulator on the CI

	cleanup := testutil.InitDatastore(t, ds.INCIDENT_AM, ds.INCIDENT_ACTIVE_PARENT_AM)
	defer cleanup()

	st := NewStore(ds.DS, []string{"ignore"})

	m := map[string]string{
		"ignore":     "pod_123",
		alerts.TYPE:  alerts.TYPE_ALERTS,
		alerts.STATE: alerts.STATE_ACTIVE,
		ALERT_NAME:   "BotUnemployed",
		"bot":        "skia-rpi-064",
		CATEGORY:     "infra",
	}
	a, err := st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, a.Active)
	assert.NotEqual(t, "", a.Key)

	// A second alert arrival with the same config doesn't create
	// a new Incident.
	a, err = st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, a.Active)
	assert.NotEqual(t, "", a.Key)

	all, err := st.GetAll()
	assert.NoError(t, err)
	assert.Equal(t, 1, len(all))

	// But an alert arrival with a different config does create
	// a new incident.
	m[ALERT_NAME] = "BotMissing"
	delete(m, ID)
	b, err := st.AlertArrival(m)
	assert.NoError(t, err)
	assert.Equal(t, true, b.Active)
	assert.NotEqual(t, "", b.Key)
	assert.NotEqual(t, a.ID, b.ID)

	// Add a note.
	bb, err := st.AddNote(b.Key, note.Note{
		Text:   "Stuff happened.",
		Author: "fred@example.com",
		TS:     time.Now().Unix(),
	})
	assert.NoError(t, err)
	assert.Equal(t, "Stuff happened.", bb.Notes[0].Text)
	assert.Equal(t, b.Key, bb.Key)

	// Fail to add note, bad key.
	_, err = st.AddNote("badkey", note.Note{})
	assert.Error(t, err)

	// Delete note, bad index.
	_, err = st.DeleteNote(b.Key, 1)
	assert.Error(t, err)

	// Delete note.
	b, err = st.DeleteNote(b.Key, 0)
	assert.NoError(t, err)
	assert.Equal(t, 0, len(b.Notes))

	// Assign
	b, err = st.Assign(b.Key, "barney@example.org")
	assert.NoError(t, err)
	assert.Equal(t, "barney@example.org", b.Params[ASSIGNED_TO])

	m[alerts.STATE] = alerts.STATE_RESOLVED
	b, err = st.AlertArrival(m)
	assert.NoError(t, err)
	assert.False(t, b.Active)

	// Archive
	b, err = st.Archive(b.Key)
	assert.NoError(t, err)

	recent, err := st.GetRecentlyResolvedForID(b.ID, "")
	assert.NoError(t, err)
	assert.Len(t, recent, 1)

	recent, err = st.GetRecentlyResolvedForID(b.ID, b.Key)
	assert.NoError(t, err)
	assert.Len(t, recent, 0)
}

func TestIdForAlert(t *testing.T) {
	unittest.LargeTest(t)
	m := map[string]string{
		"__name__":   "ALERTS",
		"alertname":  "BotMissing",
		"alertstate": "firing",
		"bot":        "skia-rpi-064",
		"category":   "infra",
		"instance":   "skia-datahopper2:20000",
		"job":        "datahopper",
		"pool":       "Skia",
		"severity":   "critical",
		"swarming":   "chromium-swarm.appspot.com",
	}
	st := NewStore(nil, []string{})

	id1, err := st.idForAlert(m)
	assert.NoError(t, err)
	id2, err := st.idForAlert(m)
	assert.NoError(t, err)
	assert.Equal(t, id1, id2)

	m[alerts.STATE] = alerts.STATE_ACTIVE
	id2, err = st.idForAlert(m)
	assert.NoError(t, err)
	assert.Equal(t, id1, id2)
}

func TestGetRegexesToOwners(t *testing.T) {
	unittest.SmallTest(t)

	ownersRegexesStr := "owner1:abbr_regex1,abbr_regex2;owner2:abbr_regex3"
	m1, err := getRegexesToOwners(ownersRegexesStr)
	assert.NoError(t, err)
	assert.Equal(t, 3, len(m1))
	assert.Equal(t, "owner1", m1["abbr_regex1"])
	assert.Equal(t, "owner1", m1["abbr_regex2"])
	assert.Equal(t, "owner2", m1["abbr_regex3"])

	// Test badly formed regex with missing owner1.
	m2, err := getRegexesToOwners("abbr_regex1,abbr2_regex;owner2:abbr_regex3")
	assert.Error(t, err)
	assert.Nil(t, m2)
}

func TestGetOwnerIfMatch(t *testing.T) {
	unittest.SmallTest(t)

	// Test matches.
	ownersRegexesStr := "superman@krypton.com:Bizarro.*,^Kryptonite.*Asteroid.*$;batman@gotham.com:Joker.*"

	ownerTest1, err := getOwnerIfMatch(ownersRegexesStr, "something Bizarro something")
	assert.NoError(t, err)
	assert.Equal(t, "superman@krypton.com", ownerTest1)

	ownerTest2, err := getOwnerIfMatch(ownersRegexesStr, "Kryptonite really big Asteroid thing")
	assert.NoError(t, err)
	assert.Equal(t, "superman@krypton.com", ownerTest2)

	ownerTest3, err := getOwnerIfMatch(ownersRegexesStr, "Joker is here!!!")
	assert.NoError(t, err)
	assert.Equal(t, "batman@gotham.com", ownerTest3)

	ownerTest4, err := getOwnerIfMatch(ownersRegexesStr, "Joker")
	assert.NoError(t, err)
	assert.Equal(t, "batman@gotham.com", ownerTest4)

	// Test misses.
	ownerMiss1, err := getOwnerIfMatch(ownersRegexesStr, "bizarro")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss1)

	ownerMiss2, err := getOwnerIfMatch(ownersRegexesStr, "wrong start Kryptonite really big Asteroid thing")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss2)

	ownerMiss3, err := getOwnerIfMatch(ownersRegexesStr, "joker")
	assert.NoError(t, err)
	assert.Equal(t, "", ownerMiss3)

	// Test badly formed regex.
	badRegex := "superman@krypton:.***"
	ownerBadTest, err := getOwnerIfMatch(badRegex, "Anything")
	assert.Error(t, err)
	assert.Equal(t, "", ownerBadTest)
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package plans_test

import (
	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/plan/plans"
	"github.com/pingcap/tidb/rset/rsets"
	"github.com/pingcap/tidb/util/mock"
)

type testOuterQuerySuite struct{}

var _ = Suite(&testOuterQuerySuite{})

func (s *testOuterQuerySuite) TestOuterQueryPlan(c *C) {
	var data = []*testRowData{
		{1, []interface{}{10, "hello"}},
	}

	pln := &testTablePlan{data, []string{"id", "name"}, 0}

	outerQuery := &plans.OuterQuery{}
	p := &plans.OuterQueryPlan{
		Src:        pln,
		OuterQuery: outerQuery,
	}

	fields := p.GetFields()
	c.Assert(fields, HasLen, 2)

	rset := rsets.Recordset{
		Plan: p,
		Ctx:  mock.NewContext()}
	_, err := rset.Rows(-1, 0)
	c.Assert(err, IsNil)

	c.Assert(outerQuery.FromData, DeepEquals, []interface{}{10, "hello"})
}

func (s *testOuterQuerySuite) TestouterQuery(c *C) {
	ctx := mock.NewContext()

	q := &plans.OuterQuery{}
}
<commit_msg>plans: fix test compile error<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package plans_test

import (
	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/plan/plans"
	"github.com/pingcap/tidb/rset/rsets"
	"github.com/pingcap/tidb/util/mock"
)

type testOuterQuerySuite struct{}

var _ = Suite(&testOuterQuerySuite{})

func (s *testOuterQuerySuite) TestOuterQueryPlan(c *C) {
	var data = []*testRowData{
		{1, []interface{}{10, "hello"}},
	}

	pln := &testTablePlan{data, []string{"id", "name"}, 0}

	outerQuery := &plans.OuterQuery{}
	p := &plans.OuterQueryPlan{
		Src:        pln,
		OuterQuery: outerQuery,
	}

	fields := p.GetFields()
	c.Assert(fields, HasLen, 2)

	rset := rsets.Recordset{
		Plan: p,
		Ctx:  mock.NewContext()}
	_, err := rset.Rows(-1, 0)
	c.Assert(err, IsNil)

	c.Assert(outerQuery.FromData, DeepEquals, []interface{}{10, "hello"})
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 The libusb developers. All rights reserved.
// Project site: https://github.com/gotmc/libusb
// Use of this source code is governed by a MIT-style license that
// can be found in the LICENSE.txt file for the project.

package libusb

import "testing"

func TestGetVersion(t *testing.T) {
	const major = 1
	const minor = 0
	const micro = 19
	const nano = 10903
	releaseCandidate := ""
	describe := "http://libusb.info"
	version := GetVersion()
	if version.Major != major {
		t.Errorf(
			"Major version == %d, want %d",
			version.Major, major)
	}
	if version.Minor != minor {
		t.Errorf(
			"Minor version == %d, want %d",
			version.Minor, minor)
	}
	if version.Micro != micro {
		t.Errorf(
			"Micro version == %d, want %d",
			version.Micro, micro)
	}
	if version.Nano != nano {
		t.Errorf(
			"Nano version == %d, want %d",
			version.Nano, nano)
	}
	if version.ReleaseCandidate != releaseCandidate {
		t.Errorf(
			"ReleaseCandidate == %v, want %v",
			version.ReleaseCandidate, releaseCandidate)
	}
	if version.Describe != describe {
		t.Errorf(
			"Describe == %v, want %v",
			version.Describe, describe)
	}
}
<commit_msg>Update version_test given updated libusb version.<commit_after>// Copyright (c) 2015 The libusb developers. All rights reserved.
// Project site: https://github.com/gotmc/libusb
// Use of this source code is governed by a MIT-style license that
// can be found in the LICENSE.txt file for the project.

package libusb

import "testing"

func TestGetVersion(t *testing.T) {
	const major = 1
	const minor = 0
	const micro = 20
	const nano = 11004
	releaseCandidate := ""
	describe := "http://libusb.info"
	version := GetVersion()
	if version.Major != major {
		t.Errorf(
			"Major version == %d, want %d",
			version.Major, major)
	}
	if version.Minor != minor {
		t.Errorf(
			"Minor version == %d, want %d",
			version.Minor, minor)
	}
	if version.Micro != micro {
		t.Errorf(
			"Micro version == %d, want %d",
			version.Micro, micro)
	}
	if version.Nano != nano {
		t.Errorf(
			"Nano version == %d, want %d",
			version.Nano, nano)
	}
	if version.ReleaseCandidate != releaseCandidate {
		t.Errorf(
			"ReleaseCandidate == %v, want %v",
			version.ReleaseCandidate, releaseCandidate)
	}
	if version.Describe != describe {
		t.Errorf(
			"Describe == %v, want %v",
			version.Describe, describe)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "testing"

func TestBuildInfo(t *testing.T) {
	cases := []struct {
		name     string
		in       BuildInfo
		want     string
		longWant string
	}{
		{"all specified", BuildInfo{
			Version:       "VER",
			GitRevision:   "GITREV",
			Host:          "HOST",
			GolangVersion: "GOLANGVER",
			DockerHub:     "DH",
			User:          "USER",
			BuildStatus:   "STATUS"},
			"USER@HOST-DH-VER-GITREV-STATUS", `Version: VER
GitRevision: GITREV
User: USER@HOST
Hub: DH
GolangVersion: GOLANGVER
BuildStatus: STATUS
`},

		{"init", Info, "unknown@unknown-unknown-unknown-unknown-unknown", `Version: unknown
GitRevision: unknown
User: unknown@unknown
Hub: unknown
GolangVersion: go1.10
BuildStatus: unknown
`},
	}

	for _, v := range cases {
		t.Run(v.name, func(t *testing.T) {
			if v.in.String() != v.want {
				t.Errorf("got %s; want %s", v.in.String(), v.want)
			}

			if v.in.LongForm() != v.longWant {
				t.Errorf("got\n%s\nwant\n%s", v.in.LongForm(), v.longWant)
			}
		})
	}
}
<commit_msg>Mac-motivated improvements to build (#4338)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

import "fmt"
import "runtime"
import "testing"

func TestBuildInfo(t *testing.T) {
	versionedString := fmt.Sprintf(`Version: unknown
GitRevision: unknown
User: unknown@unknown
Hub: unknown
GolangVersion: %v
BuildStatus: unknown
`,
		runtime.Version())

	cases := []struct {
		name     string
		in       BuildInfo
		want     string
		longWant string
	}{
		{"all specified", BuildInfo{
			Version:       "VER",
			GitRevision:   "GITREV",
			Host:          "HOST",
			GolangVersion: "GOLANGVER",
			DockerHub:     "DH",
			User:          "USER",
			BuildStatus:   "STATUS"},
			"USER@HOST-DH-VER-GITREV-STATUS", `Version: VER
GitRevision: GITREV
User: USER@HOST
Hub: DH
GolangVersion: GOLANGVER
BuildStatus: STATUS
`},

		{"init", Info, "unknown@unknown-unknown-unknown-unknown-unknown", versionedString}}

	for _, v := range cases {
		t.Run(v.name, func(t *testing.T) {
			if v.in.String() != v.want {
				t.Errorf("got %s; want %s", v.in.String(), v.want)
			}

			if v.in.LongForm() != v.longWant {
				t.Errorf("got\n%s\nwant\n%s", v.in.LongForm(), v.longWant)
			}
		})
	}
}
<|endoftext|>


<commit_before>package tcp

import (
	"context"
	"fmt"
	"net"
	"sync"

	logging "github.com/ipfs/go-log"
	reuseport "github.com/jbenet/go-reuseport"
	tpt "github.com/libp2p/go-libp2p-transport"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
	mafmt "github.com/whyrusleeping/mafmt"
)

var log = logging.Logger("tcp-tpt")

type TcpTransport struct {
	dlock   sync.Mutex
	dialers map[string]tpt.Dialer

	llock     sync.Mutex
	listeners map[string]tpt.Listener
}

var _ tpt.Transport = &TcpTransport{}

// NewTCPTransport creates a tcp transport object that tracks dialers and listeners
// created. It represents an entire tcp stack (though it might not necessarily be)
func NewTCPTransport() *TcpTransport {
	return &TcpTransport{
		dialers:   make(map[string]tpt.Dialer),
		listeners: make(map[string]tpt.Listener),
	}
}

func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dialer, error) {
	if laddr == nil {
		zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0")
		if err != nil {
			return nil, err
		}
		laddr = zaddr
	}
	t.dlock.Lock()
	defer t.dlock.Unlock()
	s := laddr.String()
	d, found := t.dialers[s]
	if found {
		return d, nil
	}
	var base manet.Dialer

	var doReuse bool
	for _, o := range opts {
		switch o := o.(type) {
		case tpt.ReuseportOpt:
			doReuse = bool(o)
		default:
			return nil, fmt.Errorf("unrecognized option: %#v", o)
		}
	}

	tcpd, err := t.newTcpDialer(base, laddr, doReuse)
	if err != nil {
		return nil, err
	}

	t.dialers[s] = tcpd
	return tcpd, nil
}

func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) {
	if !t.Matches(laddr) {
		return nil, fmt.Errorf("tcp transport cannot listen on %q", laddr)
	}

	t.llock.Lock()
	defer t.llock.Unlock()
	s := laddr.String()
	l, found := t.listeners[s]
	if found {
		return l, nil
	}

	list, err := manetListen(laddr)
	if err != nil {
		return nil, err
	}

	tlist := &tcpListener{
		list:      list,
		transport: t,
	}

	t.listeners[s] = tlist
	return tlist, nil
}

func manetListen(addr ma.Multiaddr) (manet.Listener, error) {
	network, naddr, err := manet.DialArgs(addr)
	if err != nil {
		return nil, err
	}

	if ReuseportIsAvailable() {
		nl, err := reuseport.Listen(network, naddr)
		if err == nil {
			// hey, it worked!
			return manet.WrapNetListener(nl)
		}
		// reuseport is available, but we failed to listen. log debug, and retry normally.
		log.Debugf("reuseport available, but failed to listen: %s %s, %s", network, naddr, err)
	}

	// either reuseport not available, or it failed. try normally.
	return manet.Listen(addr)
}

func (t *TcpTransport) Matches(a ma.Multiaddr) bool {
	return mafmt.TCP.Matches(a)
}

type tcpDialer struct {
	laddr ma.Multiaddr

	doReuse bool

	rd       reuseport.Dialer
	madialer manet.Dialer
	pattern  mafmt.Pattern

	transport tpt.Transport
}

var _ tpt.Dialer = &tcpDialer{}

func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) {
	// get the local net.Addr manually
	la, err := manet.ToNetAddr(laddr)
	if err != nil {
		return nil, err // something wrong with laddr.
	}

	var pattern mafmt.Pattern
	if TCP4.Matches(laddr) {
		pattern = TCP4
	} else if TCP6.Matches(laddr) {
		pattern = TCP6
	} else {
		return nil, fmt.Errorf("local addr did not match TCP4 or TCP6: %s", laddr)
	}

	if doReuse && ReuseportIsAvailable() {
		rd := reuseport.Dialer{
			D: net.Dialer{
				LocalAddr: la,
				Timeout:   base.Timeout,
			},
		}

		return &tcpDialer{
			doReuse:   true,
			laddr:     laddr,
			rd:        rd,
			madialer:  base,
			transport: t,
			pattern:   pattern,
		}, nil
	}

	return &tcpDialer{
		doReuse:   false,
		laddr:     laddr,
		pattern:   pattern,
		madialer:  base,
		transport: t,
	}, nil
}

func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) {
	return d.DialContext(context.Background(), raddr)
}

func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) {
	var c manet.Conn
	var err error
	if d.doReuse {
		c, err = d.reuseDial(ctx, raddr)
	} else {
		c, err = d.madialer.DialContext(ctx, raddr)
	}

	if err != nil {
		return nil, err
	}

	return &tcpConn{
		Conn: c,
		t:    d.transport,
	}, nil
}

func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	network, netraddr, err := manet.DialArgs(raddr)
	if err != nil {
		return nil, err
	}

	rpev := log.EventBegin(ctx, "tptDialReusePort", logging.LoggableMap{
		"raddr": raddr,
	})

	con, err := d.rd.DialContext(ctx, network, netraddr)
	if err == nil {
		rpev.Done()
		return manet.WrapNetConn(con)
	}
	rpev.SetError(err)
	rpev.Done()

	if !ReuseErrShouldRetry(err) {
		return nil, err
	}

	return d.madialer.DialContext(ctx, raddr)
}

var TCP4 = mafmt.And(mafmt.Base(ma.P_IP4), mafmt.Base(ma.P_TCP))
var TCP6 = mafmt.And(mafmt.Base(ma.P_IP6), mafmt.Base(ma.P_TCP))

func (d *tcpDialer) Matches(a ma.Multiaddr) bool {
	return d.pattern.Matches(a)
}

type tcpListener struct {
	list      manet.Listener
	transport tpt.Transport
}

var _ tpt.Listener = &tcpListener{}

func (d *tcpListener) Accept() (tpt.Conn, error) {
	c, err := d.list.Accept()
	if err != nil {
		return nil, err
	}

	return &tcpConn{
		Conn: c,
		t:    d.transport,
	}, nil
}

func (d *tcpListener) Addr() net.Addr {
	return d.list.Addr()
}

func (t *tcpListener) Multiaddr() ma.Multiaddr {
	return t.list.Multiaddr()
}

func (t *tcpListener) NetListener() net.Listener {
	return t.list.NetListener()
}

func (d *tcpListener) Close() error {
	return d.list.Close()
}

type tcpConn struct {
	manet.Conn
	t tpt.Transport
}

var _ tpt.Conn = &tcpConn{}
var _ tpt.SingleStreamConn = &tcpConn{}

func (c *tcpConn) Transport() tpt.Transport {
	return c.t
}
<commit_msg>remove reference to SingleStreamConn<commit_after>package tcp

import (
	"context"
	"fmt"
	"net"
	"sync"

	logging "github.com/ipfs/go-log"
	reuseport "github.com/jbenet/go-reuseport"
	tpt "github.com/libp2p/go-libp2p-transport"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
	mafmt "github.com/whyrusleeping/mafmt"
)

var log = logging.Logger("tcp-tpt")

type TcpTransport struct {
	dlock   sync.Mutex
	dialers map[string]tpt.Dialer

	llock     sync.Mutex
	listeners map[string]tpt.Listener
}

var _ tpt.Transport = &TcpTransport{}

// NewTCPTransport creates a tcp transport object that tracks dialers and listeners
// created. It represents an entire tcp stack (though it might not necessarily be)
func NewTCPTransport() *TcpTransport {
	return &TcpTransport{
		dialers:   make(map[string]tpt.Dialer),
		listeners: make(map[string]tpt.Listener),
	}
}

func (t *TcpTransport) Dialer(laddr ma.Multiaddr, opts ...tpt.DialOpt) (tpt.Dialer, error) {
	if laddr == nil {
		zaddr, err := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/0")
		if err != nil {
			return nil, err
		}
		laddr = zaddr
	}
	t.dlock.Lock()
	defer t.dlock.Unlock()
	s := laddr.String()
	d, found := t.dialers[s]
	if found {
		return d, nil
	}
	var base manet.Dialer

	var doReuse bool
	for _, o := range opts {
		switch o := o.(type) {
		case tpt.ReuseportOpt:
			doReuse = bool(o)
		default:
			return nil, fmt.Errorf("unrecognized option: %#v", o)
		}
	}

	tcpd, err := t.newTcpDialer(base, laddr, doReuse)
	if err != nil {
		return nil, err
	}

	t.dialers[s] = tcpd
	return tcpd, nil
}

func (t *TcpTransport) Listen(laddr ma.Multiaddr) (tpt.Listener, error) {
	if !t.Matches(laddr) {
		return nil, fmt.Errorf("tcp transport cannot listen on %q", laddr)
	}

	t.llock.Lock()
	defer t.llock.Unlock()
	s := laddr.String()
	l, found := t.listeners[s]
	if found {
		return l, nil
	}

	list, err := manetListen(laddr)
	if err != nil {
		return nil, err
	}

	tlist := &tcpListener{
		list:      list,
		transport: t,
	}

	t.listeners[s] = tlist
	return tlist, nil
}

func manetListen(addr ma.Multiaddr) (manet.Listener, error) {
	network, naddr, err := manet.DialArgs(addr)
	if err != nil {
		return nil, err
	}

	if ReuseportIsAvailable() {
		nl, err := reuseport.Listen(network, naddr)
		if err == nil {
			// hey, it worked!
			return manet.WrapNetListener(nl)
		}
		// reuseport is available, but we failed to listen. log debug, and retry normally.
		log.Debugf("reuseport available, but failed to listen: %s %s, %s", network, naddr, err)
	}

	// either reuseport not available, or it failed. try normally.
	return manet.Listen(addr)
}

func (t *TcpTransport) Matches(a ma.Multiaddr) bool {
	return mafmt.TCP.Matches(a)
}

type tcpDialer struct {
	laddr ma.Multiaddr

	doReuse bool

	rd       reuseport.Dialer
	madialer manet.Dialer
	pattern  mafmt.Pattern

	transport tpt.Transport
}

var _ tpt.Dialer = &tcpDialer{}

func (t *TcpTransport) newTcpDialer(base manet.Dialer, laddr ma.Multiaddr, doReuse bool) (*tcpDialer, error) {
	// get the local net.Addr manually
	la, err := manet.ToNetAddr(laddr)
	if err != nil {
		return nil, err // something wrong with laddr.
	}

	var pattern mafmt.Pattern
	if TCP4.Matches(laddr) {
		pattern = TCP4
	} else if TCP6.Matches(laddr) {
		pattern = TCP6
	} else {
		return nil, fmt.Errorf("local addr did not match TCP4 or TCP6: %s", laddr)
	}

	if doReuse && ReuseportIsAvailable() {
		rd := reuseport.Dialer{
			D: net.Dialer{
				LocalAddr: la,
				Timeout:   base.Timeout,
			},
		}

		return &tcpDialer{
			doReuse:   true,
			laddr:     laddr,
			rd:        rd,
			madialer:  base,
			transport: t,
			pattern:   pattern,
		}, nil
	}

	return &tcpDialer{
		doReuse:   false,
		laddr:     laddr,
		pattern:   pattern,
		madialer:  base,
		transport: t,
	}, nil
}

func (d *tcpDialer) Dial(raddr ma.Multiaddr) (tpt.Conn, error) {
	return d.DialContext(context.Background(), raddr)
}

func (d *tcpDialer) DialContext(ctx context.Context, raddr ma.Multiaddr) (tpt.Conn, error) {
	var c manet.Conn
	var err error
	if d.doReuse {
		c, err = d.reuseDial(ctx, raddr)
	} else {
		c, err = d.madialer.DialContext(ctx, raddr)
	}

	if err != nil {
		return nil, err
	}

	return &tcpConn{
		Conn: c,
		t:    d.transport,
	}, nil
}

func (d *tcpDialer) reuseDial(ctx context.Context, raddr ma.Multiaddr) (manet.Conn, error) {
	network, netraddr, err := manet.DialArgs(raddr)
	if err != nil {
		return nil, err
	}

	rpev := log.EventBegin(ctx, "tptDialReusePort", logging.LoggableMap{
		"raddr": raddr,
	})

	con, err := d.rd.DialContext(ctx, network, netraddr)
	if err == nil {
		rpev.Done()
		return manet.WrapNetConn(con)
	}
	rpev.SetError(err)
	rpev.Done()

	if !ReuseErrShouldRetry(err) {
		return nil, err
	}

	return d.madialer.DialContext(ctx, raddr)
}

var TCP4 = mafmt.And(mafmt.Base(ma.P_IP4), mafmt.Base(ma.P_TCP))
var TCP6 = mafmt.And(mafmt.Base(ma.P_IP6), mafmt.Base(ma.P_TCP))

func (d *tcpDialer) Matches(a ma.Multiaddr) bool {
	return d.pattern.Matches(a)
}

type tcpListener struct {
	list      manet.Listener
	transport tpt.Transport
}

var _ tpt.Listener = &tcpListener{}

func (d *tcpListener) Accept() (tpt.Conn, error) {
	c, err := d.list.Accept()
	if err != nil {
		return nil, err
	}

	return &tcpConn{
		Conn: c,
		t:    d.transport,
	}, nil
}

func (d *tcpListener) Addr() net.Addr {
	return d.list.Addr()
}

func (t *tcpListener) Multiaddr() ma.Multiaddr {
	return t.list.Multiaddr()
}

func (t *tcpListener) NetListener() net.Listener {
	return t.list.NetListener()
}

func (d *tcpListener) Close() error {
	return d.list.Close()
}

type tcpConn struct {
	manet.Conn
	t tpt.Transport
}

var _ tpt.Conn = &tcpConn{}

func (c *tcpConn) Transport() tpt.Transport {
	return c.t
}
<|endoftext|>


<commit_before>package tarfile

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Writer allows creating a (docker save)-formatted tar archive containing one or more images.
type Writer struct {
	writer io.Writer
	tar    *tar.Writer
	// Other state.
	blobs map[digest.Digest]types.BlobInfo // list of already-sent blobs
}

// NewWriter returns a Writer for the specified io.Writer.
// The caller must eventually call .Close() on the returned object to create a valid archive.
func NewWriter(dest io.Writer) *Writer {
	return &Writer{
		writer: dest,
		tar:    tar.NewWriter(dest),
		blobs:  make(map[digest.Digest]types.BlobInfo),
	}
}

// tryReusingBlob checks whether the transport already contains, a blob, and if so, returns its metadata.
// info.Digest must not be empty.
// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
// If the transport can not reuse the requested blob, tryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (w *Writer) tryReusingBlob(info types.BlobInfo) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
	}
	if blob, ok := w.blobs[info.Digest]; ok {
		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
	}
	return false, types.BlobInfo{}, nil
}

// recordBlob records metadata of a recorded blob, which must contain at least a digest and size.
func (w *Writer) recordBlob(info types.BlobInfo) {
	w.blobs[info.Digest] = info
}

// writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
func (w *Writer) writeLegacyLayerMetadata(layerDescriptors []manifest.Schema2Descriptor, configBytes []byte) (layerPaths []string, lastLayerID string, err error) {
	var chainID digest.Digest
	lastLayerID = ""
	for i, l := range layerDescriptors {
		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
		if chainID == "" {
			chainID = l.Digest
		} else {
			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
		}
		// … but note that this image ID does not match docker/docker/image/v1.CreateID. At least recent
		// versions allocate new IDs on load, as long as the IDs we use are unique / cannot loop.
		//
		// Overall, the goal of computing a digest dependent on the full history is to avoid reusing an image ID
		// (and possibly creating a loop in the "parent" links) if a layer with the same DiffID appears two or more
		// times in layersDescriptors.  The ChainID values are sufficient for this, the v1.CreateID computation
		// which also mixes in the full image configuration seems unnecessary, at least as long as we are storing
		// only a single image per tarball, i.e. all DiffID prefixes are unique (can’t differ only with
		// configuration).
		layerID := chainID.Hex()

		physicalLayerPath := w.physicalLayerPath(l.Digest)
		// The layer itself has been stored into physicalLayerPath in PutManifest.
		// So, use that path for layerPaths used in the non-legacy manifest
		layerPaths = append(layerPaths, physicalLayerPath)
		// ... and create a symlink for the legacy format;
		if err := w.sendSymlink(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
			return nil, "", errors.Wrap(err, "Error creating layer symbolic link")
		}

		b := []byte("1.0")
		if err := w.sendBytes(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
			return nil, "", errors.Wrap(err, "Error writing VERSION file")
		}

		// The legacy format requires a config file per layer
		layerConfig := make(map[string]interface{})
		layerConfig["id"] = layerID

		// The root layer doesn't have any parent
		if lastLayerID != "" {
			layerConfig["parent"] = lastLayerID
		}
		// The top layer configuration file is generated by using subpart of the image configuration
		if i == len(layerDescriptors)-1 {
			var config map[string]*json.RawMessage
			err := json.Unmarshal(configBytes, &config)
			if err != nil {
				return nil, "", errors.Wrap(err, "Error unmarshaling config")
			}
			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
				layerConfig[attr] = config[attr]
			}
		}
		b, err := json.Marshal(layerConfig)
		if err != nil {
			return nil, "", errors.Wrap(err, "Error marshaling layer config")
		}
		if err := w.sendBytes(filepath.Join(layerID, legacyConfigFileName), b); err != nil {
			return nil, "", errors.Wrap(err, "Error writing config json file")
		}

		lastLayerID = layerID
	}
	return layerPaths, lastLayerID, nil
}

func (w *Writer) createRepositoriesFile(rootLayerID string, repoTags []reference.NamedTagged) error {
	repositories := map[string]map[string]string{}
	for _, repoTag := range repoTags {
		if val, ok := repositories[repoTag.Name()]; ok {
			val[repoTag.Tag()] = rootLayerID
		} else {
			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
		}
	}

	b, err := json.Marshal(repositories)
	if err != nil {
		return errors.Wrap(err, "Error marshaling repositories")
	}
	if err := w.sendBytes(legacyRepositoriesFileName, b); err != nil {
		return errors.Wrap(err, "Error writing config json file")
	}
	return nil
}

func (w *Writer) createManifest(configDigest digest.Digest, layerPaths []string, repoTags []reference.NamedTagged) error {
	refStrings := []string{}
	for _, tag := range repoTags {
		// For github.com/docker/docker consumers, this works just as well as
		//   refString := ref.String()
		// because when reading the RepoTags strings, github.com/docker/docker/reference
		// normalizes both of them to the same value.
		//
		// Doing it this way to include the normalized-out `docker.io[/library]` does make
		// a difference for github.com/projectatomic/docker consumers, with the
		// “Add --add-registry and --block-registry options to docker daemon” patch.
		// These consumers treat reference strings which include a hostname and reference
		// strings without a hostname differently.
		//
		// Using the host name here is more explicit about the intent, and it has the same
		// effect as (docker pull) in projectatomic/docker, which tags the result using
		// a hostname-qualified reference.
		// See https://github.com/containers/image/issues/72 for a more detailed
		// analysis and explanation.
		refString := fmt.Sprintf("%s:%s", tag.Name(), tag.Tag())
		refStrings = append(refStrings, refString)
	}

	items := []ManifestItem{{
		Config:       w.configPath(configDigest),
		RepoTags:     refStrings,
		Layers:       layerPaths,
		Parent:       "",
		LayerSources: nil,
	}}
	itemsBytes, err := json.Marshal(&items)
	if err != nil {
		return err
	}

	return w.sendBytes(manifestFileName, itemsBytes)
}

// Close writes all outstanding data about images to the archive, and finishes writing data
// to the underlying io.Writer.
// No more images can be added after this is called.
func (w *Writer) Close() error {
	return w.tar.Close()
}

// configPath returns a path we choose for storing a config with the specified digest.
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) configPath(configDigest digest.Digest) string {
	return configDigest.Hex() + ".json"
}

// physicalLayerPath returns a path we choose for storing a layer with the specified digest
// (the actual path, i.e. a regular file, not a symlink that may be used in the legacy format).
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) physicalLayerPath(layerDigest digest.Digest) string {
	// Note that this can't be e.g. filepath.Join(l.Digest.Hex(), legacyLayerFileName); due to the way
	// writeLegacyLayerMetadata constructs layer IDs differently from inputinfo.Digest values (as described
	// inside it), most of the layers would end up in subdirectories alone without any metadata; (docker load)
	// tries to load every subdirectory as an image and fails if the config is missing.  So, keep the layers
	// in the root of the tarball.
	return layerDigest.Hex() + ".tar"
}

type tarFI struct {
	path      string
	size      int64
	isSymlink bool
}

func (t *tarFI) Name() string {
	return t.path
}
func (t *tarFI) Size() int64 {
	return t.size
}
func (t *tarFI) Mode() os.FileMode {
	if t.isSymlink {
		return os.ModeSymlink
	}
	return 0444
}
func (t *tarFI) ModTime() time.Time {
	return time.Unix(0, 0)
}
func (t *tarFI) IsDir() bool {
	return false
}
func (t *tarFI) Sys() interface{} {
	return nil
}

// sendSymlink sends a symlink into the tar stream.
func (w *Writer) sendSymlink(path string, target string) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: 0, isSymlink: true}, target)
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar link %s -> %s", path, target)
	return w.tar.WriteHeader(hdr)
}

// sendBytes sends a path into the tar stream.
func (w *Writer) sendBytes(path string, b []byte) error {
	return w.sendFile(path, int64(len(b)), bytes.NewReader(b))
}

// sendFile sends a file into the tar stream.
func (w *Writer) sendFile(path string, expectedSize int64, stream io.Reader) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar file %s", path)
	if err := w.tar.WriteHeader(hdr); err != nil {
		return err
	}
	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
	size, err := io.Copy(w.tar, stream)
	if err != nil {
		return err
	}
	if size != expectedSize {
		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
	}
	return nil
}
<commit_msg>Reorganize docker/internal/tarfile.Writer.createManifest a bit<commit_after>package tarfile

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/image/v5/docker/reference"
	"github.com/containers/image/v5/manifest"
	"github.com/containers/image/v5/types"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Writer allows creating a (docker save)-formatted tar archive containing one or more images.
type Writer struct {
	writer io.Writer
	tar    *tar.Writer
	// Other state.
	blobs map[digest.Digest]types.BlobInfo // list of already-sent blobs
}

// NewWriter returns a Writer for the specified io.Writer.
// The caller must eventually call .Close() on the returned object to create a valid archive.
func NewWriter(dest io.Writer) *Writer {
	return &Writer{
		writer: dest,
		tar:    tar.NewWriter(dest),
		blobs:  make(map[digest.Digest]types.BlobInfo),
	}
}

// tryReusingBlob checks whether the transport already contains, a blob, and if so, returns its metadata.
// info.Digest must not be empty.
// If the blob has been succesfully reused, returns (true, info, nil); info must contain at least a digest and size.
// If the transport can not reuse the requested blob, tryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
func (w *Writer) tryReusingBlob(info types.BlobInfo) (bool, types.BlobInfo, error) {
	if info.Digest == "" {
		return false, types.BlobInfo{}, errors.Errorf("Can not check for a blob with unknown digest")
	}
	if blob, ok := w.blobs[info.Digest]; ok {
		return true, types.BlobInfo{Digest: info.Digest, Size: blob.Size}, nil
	}
	return false, types.BlobInfo{}, nil
}

// recordBlob records metadata of a recorded blob, which must contain at least a digest and size.
func (w *Writer) recordBlob(info types.BlobInfo) {
	w.blobs[info.Digest] = info
}

// writeLegacyLayerMetadata writes legacy VERSION and configuration files for all layers
func (w *Writer) writeLegacyLayerMetadata(layerDescriptors []manifest.Schema2Descriptor, configBytes []byte) (layerPaths []string, lastLayerID string, err error) {
	var chainID digest.Digest
	lastLayerID = ""
	for i, l := range layerDescriptors {
		// This chainID value matches the computation in docker/docker/layer.CreateChainID …
		if chainID == "" {
			chainID = l.Digest
		} else {
			chainID = digest.Canonical.FromString(chainID.String() + " " + l.Digest.String())
		}
		// … but note that this image ID does not match docker/docker/image/v1.CreateID. At least recent
		// versions allocate new IDs on load, as long as the IDs we use are unique / cannot loop.
		//
		// Overall, the goal of computing a digest dependent on the full history is to avoid reusing an image ID
		// (and possibly creating a loop in the "parent" links) if a layer with the same DiffID appears two or more
		// times in layersDescriptors.  The ChainID values are sufficient for this, the v1.CreateID computation
		// which also mixes in the full image configuration seems unnecessary, at least as long as we are storing
		// only a single image per tarball, i.e. all DiffID prefixes are unique (can’t differ only with
		// configuration).
		layerID := chainID.Hex()

		physicalLayerPath := w.physicalLayerPath(l.Digest)
		// The layer itself has been stored into physicalLayerPath in PutManifest.
		// So, use that path for layerPaths used in the non-legacy manifest
		layerPaths = append(layerPaths, physicalLayerPath)
		// ... and create a symlink for the legacy format;
		if err := w.sendSymlink(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
			return nil, "", errors.Wrap(err, "Error creating layer symbolic link")
		}

		b := []byte("1.0")
		if err := w.sendBytes(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
			return nil, "", errors.Wrap(err, "Error writing VERSION file")
		}

		// The legacy format requires a config file per layer
		layerConfig := make(map[string]interface{})
		layerConfig["id"] = layerID

		// The root layer doesn't have any parent
		if lastLayerID != "" {
			layerConfig["parent"] = lastLayerID
		}
		// The top layer configuration file is generated by using subpart of the image configuration
		if i == len(layerDescriptors)-1 {
			var config map[string]*json.RawMessage
			err := json.Unmarshal(configBytes, &config)
			if err != nil {
				return nil, "", errors.Wrap(err, "Error unmarshaling config")
			}
			for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
				layerConfig[attr] = config[attr]
			}
		}
		b, err := json.Marshal(layerConfig)
		if err != nil {
			return nil, "", errors.Wrap(err, "Error marshaling layer config")
		}
		if err := w.sendBytes(filepath.Join(layerID, legacyConfigFileName), b); err != nil {
			return nil, "", errors.Wrap(err, "Error writing config json file")
		}

		lastLayerID = layerID
	}
	return layerPaths, lastLayerID, nil
}

func (w *Writer) createRepositoriesFile(rootLayerID string, repoTags []reference.NamedTagged) error {
	repositories := map[string]map[string]string{}
	for _, repoTag := range repoTags {
		if val, ok := repositories[repoTag.Name()]; ok {
			val[repoTag.Tag()] = rootLayerID
		} else {
			repositories[repoTag.Name()] = map[string]string{repoTag.Tag(): rootLayerID}
		}
	}

	b, err := json.Marshal(repositories)
	if err != nil {
		return errors.Wrap(err, "Error marshaling repositories")
	}
	if err := w.sendBytes(legacyRepositoriesFileName, b); err != nil {
		return errors.Wrap(err, "Error writing config json file")
	}
	return nil
}

func (w *Writer) createManifest(configDigest digest.Digest, layerPaths []string, repoTags []reference.NamedTagged) error {
	item := ManifestItem{
		Config:       w.configPath(configDigest),
		RepoTags:     []string{},
		Layers:       layerPaths,
		Parent:       "",
		LayerSources: nil,
	}

	for _, tag := range repoTags {
		// For github.com/docker/docker consumers, this works just as well as
		//   refString := ref.String()
		// because when reading the RepoTags strings, github.com/docker/docker/reference
		// normalizes both of them to the same value.
		//
		// Doing it this way to include the normalized-out `docker.io[/library]` does make
		// a difference for github.com/projectatomic/docker consumers, with the
		// “Add --add-registry and --block-registry options to docker daemon” patch.
		// These consumers treat reference strings which include a hostname and reference
		// strings without a hostname differently.
		//
		// Using the host name here is more explicit about the intent, and it has the same
		// effect as (docker pull) in projectatomic/docker, which tags the result using
		// a hostname-qualified reference.
		// See https://github.com/containers/image/issues/72 for a more detailed
		// analysis and explanation.
		refString := fmt.Sprintf("%s:%s", tag.Name(), tag.Tag())
		item.RepoTags = append(item.RepoTags, refString)
	}

	items := []ManifestItem{item}
	itemsBytes, err := json.Marshal(&items)
	if err != nil {
		return err
	}

	return w.sendBytes(manifestFileName, itemsBytes)
}

// Close writes all outstanding data about images to the archive, and finishes writing data
// to the underlying io.Writer.
// No more images can be added after this is called.
func (w *Writer) Close() error {
	return w.tar.Close()
}

// configPath returns a path we choose for storing a config with the specified digest.
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) configPath(configDigest digest.Digest) string {
	return configDigest.Hex() + ".json"
}

// physicalLayerPath returns a path we choose for storing a layer with the specified digest
// (the actual path, i.e. a regular file, not a symlink that may be used in the legacy format).
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) physicalLayerPath(layerDigest digest.Digest) string {
	// Note that this can't be e.g. filepath.Join(l.Digest.Hex(), legacyLayerFileName); due to the way
	// writeLegacyLayerMetadata constructs layer IDs differently from inputinfo.Digest values (as described
	// inside it), most of the layers would end up in subdirectories alone without any metadata; (docker load)
	// tries to load every subdirectory as an image and fails if the config is missing.  So, keep the layers
	// in the root of the tarball.
	return layerDigest.Hex() + ".tar"
}

type tarFI struct {
	path      string
	size      int64
	isSymlink bool
}

func (t *tarFI) Name() string {
	return t.path
}
func (t *tarFI) Size() int64 {
	return t.size
}
func (t *tarFI) Mode() os.FileMode {
	if t.isSymlink {
		return os.ModeSymlink
	}
	return 0444
}
func (t *tarFI) ModTime() time.Time {
	return time.Unix(0, 0)
}
func (t *tarFI) IsDir() bool {
	return false
}
func (t *tarFI) Sys() interface{} {
	return nil
}

// sendSymlink sends a symlink into the tar stream.
func (w *Writer) sendSymlink(path string, target string) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: 0, isSymlink: true}, target)
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar link %s -> %s", path, target)
	return w.tar.WriteHeader(hdr)
}

// sendBytes sends a path into the tar stream.
func (w *Writer) sendBytes(path string, b []byte) error {
	return w.sendFile(path, int64(len(b)), bytes.NewReader(b))
}

// sendFile sends a file into the tar stream.
func (w *Writer) sendFile(path string, expectedSize int64, stream io.Reader) error {
	hdr, err := tar.FileInfoHeader(&tarFI{path: path, size: expectedSize}, "")
	if err != nil {
		return nil
	}
	logrus.Debugf("Sending as tar file %s", path)
	if err := w.tar.WriteHeader(hdr); err != nil {
		return err
	}
	// TODO: This can take quite some time, and should ideally be cancellable using a context.Context.
	size, err := io.Copy(w.tar, stream)
	if err != nil {
		return err
	}
	if size != expectedSize {
		return errors.Errorf("Size mismatch when copying %s, expected %d, got %d", path, expectedSize, size)
	}
	return nil
}
<|endoftext|>