<commit_before>package ydls

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"sync"

	"github.com/wader/ydls/ffmpeg"
	"github.com/wader/ydls/id3v2"
	"github.com/wader/ydls/rereader"
	"github.com/wader/ydls/writelogger"
	"github.com/wader/ydls/youtubedl"
)

func firstNonEmpty(sl ...string) string {
	for _, s := range sl {
		if s != "" {
			return s
		}
	}
	return ""
}

func logOrDiscard(l *log.Logger) *log.Logger {
	if l != nil {
		return l
	}

	return log.New(ioutil.Discard, "", 0)
}

func id3v2FramesFromYoutueDLInfo(i *youtubedl.Info) []id3v2.Frame {
	frames := []id3v2.Frame{
		&id3v2.TextFrame{ID: "TPE1", Text: firstNonEmpty(i.Artist, i.Creator, i.Uploader)},
		&id3v2.TextFrame{ID: "TIT2", Text: i.Title},
		&id3v2.COMMFrame{Language: "XXX", Description: "", Text: i.Description},
	}
	if i.Duration > 0 {
		frames = append(frames, &id3v2.TextFrame{
			ID:   "TLEN",
			Text: fmt.Sprintf("%d", uint32(i.Duration*1000)),
		})
	}
	if len(i.ThumbnailBytes) > 0 {
		frames = append(frames, &id3v2.APICFrame{
			MIMEType:    http.DetectContentType(i.ThumbnailBytes),
			PictureType: id3v2.PictureTypeOther,
			Description: "",
			Data:        i.ThumbnailBytes,
		})
	}

	return frames
}

func findFormat(formats []*youtubedl.Format, protocol string, aCodecs *prioStringSet, vCodecs *prioStringSet) *youtubedl.Format {
	var matched []*youtubedl.Format

	for _, f := range formats {
		if protocol != "*" && f.Protocol != protocol {
			continue
		}
		if !(aCodecs == nil || (f.NormACodec == "" && aCodecs.empty()) || aCodecs.member(f.NormACodec)) {
			continue
		}
		if !(vCodecs == nil || (f.NormVCodec == "" && vCodecs.empty()) || vCodecs.member(f.NormVCodec)) {
			continue
		}

		matched = append(matched, f)
	}

	sort.Sort(youtubedl.FormatByNormBR(matched))

	if len(matched) > 0 {
		return matched[0]
	}

	return nil
}

func findBestFormats(ydlFormats []*youtubedl.Format, format *Format) (aFormat *youtubedl.Format, vFormat *youtubedl.Format) {
	type neededFormat struct {
		aCodecs    *prioStringSet
		vCodecs    *prioStringSet
		aYDLFormat **youtubedl.Format
		vYDLFormat **youtubedl.Format
	}

	// TODO: messy, needs rewrite

	var neededFormats []*neededFormat

	// match exactly, both audio/video codecs found or not found
	neededFormats = append(neededFormats, &neededFormat{
		format.ACodecs.PrioStringSet(),
		format.VCodecs.PrioStringSet(),
		&aFormat, &vFormat,
	})

	if !format.ACodecs.empty() {
		// matching audio codec with any video codec
		neededFormats = append(neededFormats, &neededFormat{format.ACodecs.PrioStringSet(), nil, &aFormat, nil})
		// match any audio codec and no video
		neededFormats = append(neededFormats, &neededFormat{nil, &prioStringSet{}, &aFormat, nil})
		// match any audio and video codec
		neededFormats = append(neededFormats, &neededFormat{nil, nil, &aFormat, nil})
	}
	if !format.VCodecs.empty() {
		// same logic as above
		neededFormats = append(neededFormats, &neededFormat{nil, format.VCodecs.PrioStringSet(), nil, &vFormat})
		neededFormats = append(neededFormats, &neededFormat{&prioStringSet{}, nil, nil, &vFormat})
		neededFormats = append(neededFormats, &neededFormat{nil, nil, nil, &vFormat})
	}

	// TODO: if only audio => stream with lowest video br?

	for _, proto := range []string{"https", "http", "*"} {
		for _, f := range neededFormats {
			m := findFormat(ydlFormats, proto, f.aCodecs, f.vCodecs)

			if m == nil {
				continue
			}

			if f.aYDLFormat != nil && *f.aYDLFormat == nil && m.NormACodec != "" {
				*f.aYDLFormat = m
			}
			if f.vYDLFormat != nil && *f.vYDLFormat == nil && m.NormVCodec != "" {
				*f.vYDLFormat = m
			}

			if (format.ACodecs.empty() || aFormat != nil) &&
				(format.VCodecs.empty() || vFormat != nil) {
				break
			}
		}
	}

	return aFormat, vFormat
}

func downloadAndProbeFormat(ctx context.Context, ydl *youtubedl.Info, filter string, debugLog *log.Logger) (r io.ReadCloser, pi *ffmpeg.ProbeInfo, err error) {
	log := logOrDiscard(debugLog)

	ydlStderr := writelogger.New(log, fmt.Sprintf("ydl-dl %s stderr> ", filter))
	r, err = ydl.Download(ctx, filter, ydlStderr)
	if err != nil {
		return nil, nil, err
	}

	rr := rereader.NewReReadCloser(r)
	ffprobeStderr := writelogger.New(log, fmt.Sprintf("ffprobe %s stderr> ", filter))
	const maxProbeByteSize = 10 * 1024 * 1024
	pi, err = ffmpeg.Probe(ctx, io.LimitReader(rr, maxProbeByteSize), log, ffprobeStderr)
	if err != nil {
		return nil, nil, err
	}
	// restart and replay buffer data used when probing
	rr.Restarted = true

	return rr, pi, nil
}

// YDLS youtubedl downloader with some extras
type YDLS struct {
	Formats *Formats
}

// NewFromFile new YDLs using formats file
func NewFromFile(formatsPath string) (*YDLS, error) {
	formatsFile, err := os.Open(formatsPath)
	if err != nil {
		return nil, err
	}
	defer formatsFile.Close()
	formats, err := parseFormats(formatsFile)
	if err != nil {
		return nil, err
	}

	return &YDLS{Formats: formats}, nil
}

// DownloadResult download result
type DownloadResult struct {
	Media    io.ReadCloser
	Filename string
	MIMEType string
}

func chooseFormatCodec(formats prioFormatCodecSet, probedCodec string) *FormatCodec {
	codecFormat := formats.findByCodecName(probedCodec)
	if codecFormat != nil {
		copyCodecFormat := *codecFormat
		copyCodecFormat.Codec = "copy"
		return &copyCodecFormat
	}

	return formats.first()
}

func fancyYDLFormatName(ydlFormat *youtubedl.Format) string {
	if ydlFormat == nil {
		return "n/a"
	}
	return ydlFormat.String()
}

// Download downloads media from URL using context and makes sure output is in specified format
func (ydls *YDLS) Download(ctx context.Context, url string, formatName string, debugLog *log.Logger) (*DownloadResult, error) {
	var closeOnDone []io.Closer
	closeOnDoneFn := func() {
		for _, c := range closeOnDone {
			c.Close()
		}
	}
	deferCloseFn := closeOnDoneFn
	defer func() {
		// will be nil if cmd starts and goroutine takes care of closing instead
		if deferCloseFn != nil {
			deferCloseFn()
		}
	}()

	log := logOrDiscard(debugLog)

	log.Printf("URL: %s", url)
	log.Printf("Output format: %s", formatName)

	var ydlStdout io.Writer
	ydlStdout = writelogger.New(log, "ydl-new stdout> ")
	ydl, err := youtubedl.NewFromURL(ctx, url, ydlStdout)
	if err != nil {
		log.Printf("Failed to download: %s", err)
		return nil, err
	}

	log.Printf("Title: %s", ydl.Title)
	log.Printf("Available youtubedl formats:")
	for _, f := range ydl.Formats {
		log.Printf("  %s", f)
	}

	dr := &DownloadResult{}

	if formatName == "" {
		var probeInfo *ffmpeg.ProbeInfo
		dr.Media, probeInfo, err = downloadAndProbeFormat(ctx, ydl, "best[protocol=https]/best[protocol=http]/best", log)
		if err != nil {
			return nil, err
		}

		log.Printf("Probed format %s", probeInfo)

		// see if we know about the probed format, otherwise fallback to "raw"
		outFormat := ydls.Formats.Find(probeInfo.FormatName(), probeInfo.ACodec(), probeInfo.VCodec())
		if outFormat != nil {
			dr.MIMEType = outFormat.MIMEType
			dr.Filename = ydl.Title + "." + outFormat.Ext
		} else {
			dr.MIMEType = "application/octet-stream"
			dr.Filename = ydl.Title + ".raw"
		}

		return dr, nil
	}

	outFormat := ydls.Formats.FindByName(formatName)
	if outFormat == nil {
		return nil, fmt.Errorf("could not find format")
	}

	dr.MIMEType = outFormat.MIMEType
	dr.Filename = ydl.Title + "." + outFormat.Ext

	aYDLFormat, vYDLFormat := findBestFormats(ydl.Formats, outFormat)

	var aProbeInfo *ffmpeg.ProbeInfo
	var aReader io.ReadCloser
	var aErr error
	var vProbeInfo *ffmpeg.ProbeInfo
	var vReader io.ReadCloser
	var vErr error

	if aYDLFormat != nil && vYDLFormat != nil {
		if aYDLFormat != vYDLFormat {
			// audio and video in different formats, download simultaneously
			var probeWG sync.WaitGroup
			probeWG.Add(2)
			go func() {
				defer probeWG.Done()
				aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
			}()
			go func() {
				defer probeWG.Done()
				vReader, vProbeInfo, vErr = downloadAndProbeFormat(ctx, ydl, vYDLFormat.FormatID, log)
			}()
			probeWG.Wait()
			if aReader != nil {
				closeOnDone = append(closeOnDone, aReader)
			}
			if vReader != nil {
				closeOnDone = append(closeOnDone, vReader)
			}
		} else {
			// audio and video in same format
			aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
			vReader, vProbeInfo, vErr = aReader, aProbeInfo, aErr
			if aReader != nil {
				closeOnDone = append(closeOnDone, aReader)
			}
		}
	} else if aYDLFormat != nil && vYDLFormat == nil {
		// only audio format
		aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
		if aReader != nil {
			closeOnDone = append(closeOnDone, aReader)
		}
	} else {
		// don't know, download and probe
		aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, "best", log)
		vReader, vProbeInfo, vErr = aReader, aProbeInfo, aErr
		if aReader != nil {
			closeOnDone = append(closeOnDone, aReader)
		}
	}
	if aErr != nil || vErr != nil {
		return nil, fmt.Errorf("failed to probe")
	}

	log.Printf("Stream mapping:")

	var streamMaps []ffmpeg.StreamMap
	ffmpegFormatFlags := outFormat.FormatFlags

	if len(outFormat.ACodecs) > 0 && aProbeInfo != nil && aProbeInfo.ACodec() != "" {
		codecFormat := chooseFormatCodec(outFormat.ACodecs, aProbeInfo.ACodec())
		streamMaps = append(streamMaps, ffmpeg.StreamMap{
			Reader:     aReader,
			Specifier:  "a:0",
			Codec:      "acodec:" + codecFormat.Codec,
			CodecFlags: codecFormat.CodecFlags,
		})
		ffmpegFormatFlags = append(ffmpegFormatFlags, codecFormat.FormatFlags...)

		log.Printf("  audio %s probed:%s -> %s",
			fancyYDLFormatName(aYDLFormat),
			aProbeInfo,
			codecFormat.Codec,
		)
	}
	if len(outFormat.VCodecs) > 0 && vProbeInfo != nil && vProbeInfo.VCodec() != "" {
		codecFormat := chooseFormatCodec(outFormat.VCodecs, vProbeInfo.VCodec())
		streamMaps = append(streamMaps, ffmpeg.StreamMap{
			Reader:     vReader,
			Specifier:  "v:0",
			Codec:      "vcodec:" + codecFormat.Codec,
			CodecFlags: codecFormat.CodecFlags,
		})
		ffmpegFormatFlags = append(ffmpegFormatFlags, codecFormat.FormatFlags...)

		log.Printf("  video %s probed:%s -> %s",
			fancyYDLFormatName(vYDLFormat),
			vProbeInfo,
			codecFormat.Codec,
		)
	}

	var ffmpegStderr io.Writer
	ffmpegStderr = writelogger.New(log, "ffmpeg stderr> ")
	ffmpegR, ffmpegW := io.Pipe()
	closeOnDone = append(closeOnDone, ffmpegR)

	ffmpegP := &ffmpeg.FFmpeg{
		StreamMaps: streamMaps,
		Format:     ffmpeg.Format{Name: outFormat.Formats.first(), Flags: ffmpegFormatFlags},
		DebugLog:   log,
		Stdout:     ffmpegW,
		Stderr:     ffmpegStderr,
	}

	if err := ffmpegP.Start(ctx); err != nil {
		return nil, err
	}

	// goroutine will take care of closing
	deferCloseFn = nil

	var w io.WriteCloser
	dr.Media, w = io.Pipe()
	closeOnDone = append(closeOnDone, w)

	go func() {
		if outFormat.Prepend == "id3v2" {
			id3v2.Write(w, id3v2FramesFromYoutueDLInfo(ydl))
		}
		n, err := io.Copy(w, ffmpegR)

		log.Printf("Copy ffmpeg done (n=%v err=%v)", n, err)

		closeOnDoneFn()
		ffmpegP.Wait()

		log.Printf("Done")
	}()

	return dr, nil
}
<commit_msg>Duplicate ffmpegFormatFlags to make it concurrent safe<commit_after>package ydls

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sort"
	"sync"

	"github.com/wader/ydls/ffmpeg"
	"github.com/wader/ydls/id3v2"
	"github.com/wader/ydls/rereader"
	"github.com/wader/ydls/writelogger"
	"github.com/wader/ydls/youtubedl"
)

func firstNonEmpty(sl ...string) string {
	for _, s := range sl {
		if s != "" {
			return s
		}
	}
	return ""
}

func logOrDiscard(l *log.Logger) *log.Logger {
	if l != nil {
		return l
	}

	return log.New(ioutil.Discard, "", 0)
}

func id3v2FramesFromYoutueDLInfo(i *youtubedl.Info) []id3v2.Frame {
	frames := []id3v2.Frame{
		&id3v2.TextFrame{ID: "TPE1", Text: firstNonEmpty(i.Artist, i.Creator, i.Uploader)},
		&id3v2.TextFrame{ID: "TIT2", Text: i.Title},
		&id3v2.COMMFrame{Language: "XXX", Description: "", Text: i.Description},
	}
	if i.Duration > 0 {
		frames = append(frames, &id3v2.TextFrame{
			ID:   "TLEN",
			Text: fmt.Sprintf("%d", uint32(i.Duration*1000)),
		})
	}
	if len(i.ThumbnailBytes) > 0 {
		frames = append(frames, &id3v2.APICFrame{
			MIMEType:    http.DetectContentType(i.ThumbnailBytes),
			PictureType: id3v2.PictureTypeOther,
			Description: "",
			Data:        i.ThumbnailBytes,
		})
	}

	return frames
}

func findFormat(formats []*youtubedl.Format, protocol string, aCodecs *prioStringSet, vCodecs *prioStringSet) *youtubedl.Format {
	var matched []*youtubedl.Format

	for _, f := range formats {
		if protocol != "*" && f.Protocol != protocol {
			continue
		}
		if !(aCodecs == nil || (f.NormACodec == "" && aCodecs.empty()) || aCodecs.member(f.NormACodec)) {
			continue
		}
		if !(vCodecs == nil || (f.NormVCodec == "" && vCodecs.empty()) || vCodecs.member(f.NormVCodec)) {
			continue
		}

		matched = append(matched, f)
	}

	sort.Sort(youtubedl.FormatByNormBR(matched))

	if len(matched) > 0 {
		return matched[0]
	}

	return nil
}

func findBestFormats(ydlFormats []*youtubedl.Format, format *Format) (aFormat *youtubedl.Format, vFormat *youtubedl.Format) {
	type neededFormat struct {
		aCodecs    *prioStringSet
		vCodecs    *prioStringSet
		aYDLFormat **youtubedl.Format
		vYDLFormat **youtubedl.Format
	}

	// TODO: messy, needs rewrite

	var neededFormats []*neededFormat

	// match exactly, both audio/video codecs found or not found
	neededFormats = append(neededFormats, &neededFormat{
		format.ACodecs.PrioStringSet(),
		format.VCodecs.PrioStringSet(),
		&aFormat, &vFormat,
	})

	if !format.ACodecs.empty() {
		// matching audio codec with any video codec
		neededFormats = append(neededFormats, &neededFormat{format.ACodecs.PrioStringSet(), nil, &aFormat, nil})
		// match any audio codec and no video
		neededFormats = append(neededFormats, &neededFormat{nil, &prioStringSet{}, &aFormat, nil})
		// match any audio and video codec
		neededFormats = append(neededFormats, &neededFormat{nil, nil, &aFormat, nil})
	}
	if !format.VCodecs.empty() {
		// same logic as above
		neededFormats = append(neededFormats, &neededFormat{nil, format.VCodecs.PrioStringSet(), nil, &vFormat})
		neededFormats = append(neededFormats, &neededFormat{&prioStringSet{}, nil, nil, &vFormat})
		neededFormats = append(neededFormats, &neededFormat{nil, nil, nil, &vFormat})
	}

	// TODO: if only audio => stream with lowest video br?

	for _, proto := range []string{"https", "http", "*"} {
		for _, f := range neededFormats {
			m := findFormat(ydlFormats, proto, f.aCodecs, f.vCodecs)

			if m == nil {
				continue
			}

			if f.aYDLFormat != nil && *f.aYDLFormat == nil && m.NormACodec != "" {
				*f.aYDLFormat = m
			}
			if f.vYDLFormat != nil && *f.vYDLFormat == nil && m.NormVCodec != "" {
				*f.vYDLFormat = m
			}

			if (format.ACodecs.empty() || aFormat != nil) &&
				(format.VCodecs.empty() || vFormat != nil) {
				break
			}
		}
	}

	return aFormat, vFormat
}

func downloadAndProbeFormat(ctx context.Context, ydl *youtubedl.Info, filter string, debugLog *log.Logger) (r io.ReadCloser, pi *ffmpeg.ProbeInfo, err error) {
	log := logOrDiscard(debugLog)

	ydlStderr := writelogger.New(log, fmt.Sprintf("ydl-dl %s stderr> ", filter))
	r, err = ydl.Download(ctx, filter, ydlStderr)
	if err != nil {
		return nil, nil, err
	}

	rr := rereader.NewReReadCloser(r)
	ffprobeStderr := writelogger.New(log, fmt.Sprintf("ffprobe %s stderr> ", filter))
	const maxProbeByteSize = 10 * 1024 * 1024
	pi, err = ffmpeg.Probe(ctx, io.LimitReader(rr, maxProbeByteSize), log, ffprobeStderr)
	if err != nil {
		return nil, nil, err
	}
	// restart and replay buffer data used when probing
	rr.Restarted = true

	return rr, pi, nil
}

// YDLS youtubedl downloader with some extras
type YDLS struct {
	Formats *Formats
}

// NewFromFile new YDLs using formats file
func NewFromFile(formatsPath string) (*YDLS, error) {
	formatsFile, err := os.Open(formatsPath)
	if err != nil {
		return nil, err
	}
	defer formatsFile.Close()
	formats, err := parseFormats(formatsFile)
	if err != nil {
		return nil, err
	}

	return &YDLS{Formats: formats}, nil
}

// DownloadResult download result
type DownloadResult struct {
	Media    io.ReadCloser
	Filename string
	MIMEType string
}

func chooseFormatCodec(formats prioFormatCodecSet, probedCodec string) *FormatCodec {
	codecFormat := formats.findByCodecName(probedCodec)
	if codecFormat != nil {
		copyCodecFormat := *codecFormat
		copyCodecFormat.Codec = "copy"
		return &copyCodecFormat
	}

	return formats.first()
}

func fancyYDLFormatName(ydlFormat *youtubedl.Format) string {
	if ydlFormat == nil {
		return "n/a"
	}
	return ydlFormat.String()
}

// Download downloads media from URL using context and makes sure output is in specified format
func (ydls *YDLS) Download(ctx context.Context, url string, formatName string, debugLog *log.Logger) (*DownloadResult, error) {
	var closeOnDone []io.Closer
	closeOnDoneFn := func() {
		for _, c := range closeOnDone {
			c.Close()
		}
	}
	deferCloseFn := closeOnDoneFn
	defer func() {
		// will be nil if cmd starts and goroutine takes care of closing instead
		if deferCloseFn != nil {
			deferCloseFn()
		}
	}()

	log := logOrDiscard(debugLog)

	log.Printf("URL: %s", url)
	log.Printf("Output format: %s", formatName)

	var ydlStdout io.Writer
	ydlStdout = writelogger.New(log, "ydl-new stdout> ")
	ydl, err := youtubedl.NewFromURL(ctx, url, ydlStdout)
	if err != nil {
		log.Printf("Failed to download: %s", err)
		return nil, err
	}

	log.Printf("Title: %s", ydl.Title)
	log.Printf("Available youtubedl formats:")
	for _, f := range ydl.Formats {
		log.Printf("  %s", f)
	}

	dr := &DownloadResult{}

	if formatName == "" {
		var probeInfo *ffmpeg.ProbeInfo
		dr.Media, probeInfo, err = downloadAndProbeFormat(ctx, ydl, "best[protocol=https]/best[protocol=http]/best", log)
		if err != nil {
			return nil, err
		}

		log.Printf("Probed format %s", probeInfo)

		// see if we know about the probed format, otherwise fallback to "raw"
		outFormat := ydls.Formats.Find(probeInfo.FormatName(), probeInfo.ACodec(), probeInfo.VCodec())
		if outFormat != nil {
			dr.MIMEType = outFormat.MIMEType
			dr.Filename = ydl.Title + "." + outFormat.Ext
		} else {
			dr.MIMEType = "application/octet-stream"
			dr.Filename = ydl.Title + ".raw"
		}

		return dr, nil
	}

	outFormat := ydls.Formats.FindByName(formatName)
	if outFormat == nil {
		return nil, fmt.Errorf("could not find format")
	}

	dr.MIMEType = outFormat.MIMEType
	dr.Filename = ydl.Title + "." + outFormat.Ext

	aYDLFormat, vYDLFormat := findBestFormats(ydl.Formats, outFormat)

	var aProbeInfo *ffmpeg.ProbeInfo
	var aReader io.ReadCloser
	var aErr error
	var vProbeInfo *ffmpeg.ProbeInfo
	var vReader io.ReadCloser
	var vErr error

	if aYDLFormat != nil && vYDLFormat != nil {
		if aYDLFormat != vYDLFormat {
			// audio and video in different formats, download simultaneously
			var probeWG sync.WaitGroup
			probeWG.Add(2)
			go func() {
				defer probeWG.Done()
				aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
			}()
			go func() {
				defer probeWG.Done()
				vReader, vProbeInfo, vErr = downloadAndProbeFormat(ctx, ydl, vYDLFormat.FormatID, log)
			}()
			probeWG.Wait()
			if aReader != nil {
				closeOnDone = append(closeOnDone, aReader)
			}
			if vReader != nil {
				closeOnDone = append(closeOnDone, vReader)
			}
		} else {
			// audio and video in same format
			aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
			vReader, vProbeInfo, vErr = aReader, aProbeInfo, aErr
			if aReader != nil {
				closeOnDone = append(closeOnDone, aReader)
			}
		}
	} else if aYDLFormat != nil && vYDLFormat == nil {
		// only audio format
		aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, aYDLFormat.FormatID, log)
		if aReader != nil {
			closeOnDone = append(closeOnDone, aReader)
		}
	} else {
		// don't know, download and probe
		aReader, aProbeInfo, aErr = downloadAndProbeFormat(ctx, ydl, "best", log)
		vReader, vProbeInfo, vErr = aReader, aProbeInfo, aErr
		if aReader != nil {
			closeOnDone = append(closeOnDone, aReader)
		}
	}
	if aErr != nil || vErr != nil {
		return nil, fmt.Errorf("failed to probe")
	}

	log.Printf("Stream mapping:")

	var streamMaps []ffmpeg.StreamMap
	ffmpegFormatFlags := make([]string, len(outFormat.FormatFlags))
	copy(ffmpegFormatFlags, outFormat.FormatFlags)

	if len(outFormat.ACodecs) > 0 && aProbeInfo != nil && aProbeInfo.ACodec() != "" {
		codecFormat := chooseFormatCodec(outFormat.ACodecs, aProbeInfo.ACodec())
		streamMaps = append(streamMaps, ffmpeg.StreamMap{
			Reader:     aReader,
			Specifier:  "a:0",
			Codec:      "acodec:" + codecFormat.Codec,
			CodecFlags: codecFormat.CodecFlags,
		})
		ffmpegFormatFlags = append(ffmpegFormatFlags, codecFormat.FormatFlags...)

		log.Printf("  audio %s probed:%s -> %s",
			fancyYDLFormatName(aYDLFormat),
			aProbeInfo,
			codecFormat.Codec,
		)
	}
	if len(outFormat.VCodecs) > 0 && vProbeInfo != nil && vProbeInfo.VCodec() != "" {
		codecFormat := chooseFormatCodec(outFormat.VCodecs, vProbeInfo.VCodec())
		streamMaps = append(streamMaps, ffmpeg.StreamMap{
			Reader:     vReader,
			Specifier:  "v:0",
			Codec:      "vcodec:" + codecFormat.Codec,
			CodecFlags: codecFormat.CodecFlags,
		})
		ffmpegFormatFlags = append(ffmpegFormatFlags, codecFormat.FormatFlags...)

		log.Printf("  video %s probed:%s -> %s",
			fancyYDLFormatName(vYDLFormat),
			vProbeInfo,
			codecFormat.Codec,
		)
	}

	var ffmpegStderr io.Writer
	ffmpegStderr = writelogger.New(log, "ffmpeg stderr> ")
	ffmpegR, ffmpegW := io.Pipe()
	closeOnDone = append(closeOnDone, ffmpegR)

	ffmpegP := &ffmpeg.FFmpeg{
		StreamMaps: streamMaps,
		Format:     ffmpeg.Format{Name: outFormat.Formats.first(), Flags: ffmpegFormatFlags},
		DebugLog:   log,
		Stdout:     ffmpegW,
		Stderr:     ffmpegStderr,
	}

	if err := ffmpegP.Start(ctx); err != nil {
		return nil, err
	}

	// goroutine will take care of closing
	deferCloseFn = nil

	var w io.WriteCloser
	dr.Media, w = io.Pipe()
	closeOnDone = append(closeOnDone, w)

	go func() {
		if outFormat.Prepend == "id3v2" {
			id3v2.Write(w, id3v2FramesFromYoutueDLInfo(ydl))
		}
		n, err := io.Copy(w, ffmpegR)

		log.Printf("Copy ffmpeg done (n=%v err=%v)", n, err)

		closeOnDoneFn()
		ffmpegP.Wait()

		log.Printf("Done")
	}()

	return dr, nil
}
<|endoftext|>


<commit_before>package ts

import "github.com/32bitkid/mpeg/util"
import "io"
import "errors"

const (
	SyncByte       = 0x47
	MaxPayloadSize = 184
)

var (
	ErrNoSyncByte = errors.New("no sync byte")
)

type Packet struct {
	TransportErrorIndicator    bool
	PayloadUnitStartIndicator  bool
	TransportPriority          bool
	PID                        uint32
	TransportScramblingControl uint32
	AdaptationFieldControl     uint32
	ContinuityCounter          uint32
	AdaptationField            *AdaptationField
	Payload                    []byte

	payloadBuffer [MaxPayloadSize]byte
}

func (packet *Packet) ReadFrom(tsr util.BitReader32) (err error) {

	aligned, err := isAligned(tsr)
	if err != nil {
		return
	}

	if !aligned {
		if err = realign(tsr); err != nil {
			return
		}
	}

	if err = tsr.Trash(8); err != nil {
		return
	}

	packet.TransportErrorIndicator, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.PayloadUnitStartIndicator, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.TransportPriority, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.PID, err = tsr.Read32(13)
	if err != nil {
		return
	}

	packet.TransportScramblingControl, err = tsr.Read32(2)
	if err != nil {
		return
	}

	packet.AdaptationFieldControl, err = tsr.Read32(2)
	if err != nil {
		return
	}

	packet.ContinuityCounter, err = tsr.Read32(4)
	if err != nil {
		return
	}

	var payloadSize uint32 = MaxPayloadSize

	if packet.AdaptationFieldControl == FieldOnly || packet.AdaptationFieldControl == FieldThenPayload {
		packet.AdaptationField, err = ReadAdaptationField(tsr)

		if err != nil {
			return
		}
		payloadSize -= packet.AdaptationField.Length + 1
	}

	if packet.AdaptationFieldControl == PayloadOnly || packet.AdaptationFieldControl == FieldThenPayload {
		packet.Payload = packet.payloadBuffer[0:payloadSize]

		// TODO replace with reader
		_, err = io.ReadAtLeast(tsr, packet.Payload, int(payloadSize))
		if err != nil {
			return
		}
	}

	return nil
}

func isAligned(tsr util.BitReader32) (bool, error) {
	if !tsr.IsByteAligned() {
		return false, nil
	}
	val, err := tsr.Peek32(8)
	if err == io.ErrUnexpectedEOF {
		return false, io.EOF
	} else if err != err {
		return false, err
	}
	return val == SyncByte, nil
}

func realign(tsr util.BitReader32) error {
	if !tsr.IsByteAligned() {
		tsr.ByteAlign()
	}

	for i := 0; i < 188; i++ {
		isAligned, err := isAligned(tsr)
		if err != nil {
			return err
		}
		if isAligned {
			return nil
		}
		if err := tsr.Trash(8); err != nil {
			return err
		}
	}
	return ErrNoSyncByte
}
<commit_msg>pass actual error from BitReader back up.<commit_after>package ts

import "github.com/32bitkid/mpeg/util"
import "io"
import "errors"

const (
	SyncByte       = 0x47
	MaxPayloadSize = 184
)

var (
	ErrNoSyncByte = errors.New("no sync byte")
)

type Packet struct {
	TransportErrorIndicator    bool
	PayloadUnitStartIndicator  bool
	TransportPriority          bool
	PID                        uint32
	TransportScramblingControl uint32
	AdaptationFieldControl     uint32
	ContinuityCounter          uint32
	AdaptationField            *AdaptationField
	Payload                    []byte

	payloadBuffer [MaxPayloadSize]byte
}

func (packet *Packet) ReadFrom(tsr util.BitReader32) (err error) {

	aligned, err := isAligned(tsr)
	if err != nil {
		return
	}

	if !aligned {
		if err = realign(tsr); err != nil {
			return
		}
	}

	if err = tsr.Trash(8); err != nil {
		return
	}

	packet.TransportErrorIndicator, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.PayloadUnitStartIndicator, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.TransportPriority, err = tsr.ReadBit()
	if err != nil {
		return
	}

	packet.PID, err = tsr.Read32(13)
	if err != nil {
		return
	}

	packet.TransportScramblingControl, err = tsr.Read32(2)
	if err != nil {
		return
	}

	packet.AdaptationFieldControl, err = tsr.Read32(2)
	if err != nil {
		return
	}

	packet.ContinuityCounter, err = tsr.Read32(4)
	if err != nil {
		return
	}

	var payloadSize uint32 = MaxPayloadSize

	if packet.AdaptationFieldControl == FieldOnly || packet.AdaptationFieldControl == FieldThenPayload {
		packet.AdaptationField, err = ReadAdaptationField(tsr)

		if err != nil {
			return
		}
		payloadSize -= packet.AdaptationField.Length + 1
	}

	if packet.AdaptationFieldControl == PayloadOnly || packet.AdaptationFieldControl == FieldThenPayload {
		packet.Payload = packet.payloadBuffer[0:payloadSize]

		// TODO replace with reader
		_, err = io.ReadAtLeast(tsr, packet.Payload, int(payloadSize))
		if err != nil {
			return
		}
	}

	return nil
}

func isAligned(tsr util.BitReader32) (bool, error) {
	if !tsr.IsByteAligned() {
		return false, nil
	}
	val, err := tsr.Peek32(8)
	if err != nil {
		return false, err
	}
	return val == SyncByte, nil
}

func realign(tsr util.BitReader32) error {
	if !tsr.IsByteAligned() {
		tsr.ByteAlign()
	}

	for i := 0; i < 188; i++ {
		isAligned, err := isAligned(tsr)
		if err != nil {
			return err
		}
		if isAligned {
			return nil
		}
		if err := tsr.Trash(8); err != nil {
			return err
		}
	}
	return ErrNoSyncByte
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAwsServerlessRepositoryApplication_Basic(t *testing.T) {
	var stack cloudformation.Stack
	stackName := fmt.Sprintf("tf-acc-test-basic-%s", acctest.RandString(10))

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloudFormationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAwsServerlessRepositoryApplicationConfig(stackName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "application_id", "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"),
					resource.TestCheckResourceAttrSet("aws_serverlessrepository_application.postgres-rotator", "semantic_version"),
				),
			},
		},
	})
}

func TestAccAwsServerlessRepositoryApplication_Versioned(t *testing.T) {
	var stack cloudformation.Stack
	stackName := fmt.Sprintf("tf-acc-test-versioned-%s", acctest.RandString(10))
	const version = "1.0.15"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloudFormationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, version),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "semantic_version", version),
				),
			},
		},
	})
}

func testAccAwsServerlessRepositoryApplicationConfig(stackName string) string {
	return fmt.Sprintf(`
resource "aws_serverlessrepository_application" "postgres-rotator" {
  name           = "%[1]s"
  application_id = "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"
  parameters = {
    functionName = "func-%[1]s"
    endpoint     = "secretsmanager.us-east-2.amazonaws.com"
  }
}`, stackName)
}

func testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, version string) string {
	return fmt.Sprintf(`
resource "aws_serverlessrepository_application" "postgres-rotator" {
  name             = "%[1]s"
  application_id   = "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"
  semantic_version = "%[2]s"
  parameters = {
    functionName = "func-%[1]s"
    endpoint     = "secretsmanager.us-east-2.amazonaws.com"
  }
}`, stackName, version)
}

func testAccCheckerverlessRepositoryApplicationExists(n string, stack *cloudformation.Stack) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		conn := testAccProvider.Meta().(*AWSClient).cfconn
		params := &cloudformation.DescribeStacksInput{
			StackName: aws.String(rs.Primary.ID),
		}
		resp, err := conn.DescribeStacks(params)
		if err != nil {
			return err
		}
		if len(resp.Stacks) == 0 {
			return fmt.Errorf("CloudFormation stack not found")
		}

		return nil
	}
}
<commit_msg>Added test for changing application version<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAwsServerlessRepositoryApplication_Basic(t *testing.T) {
	var stack cloudformation.Stack
	stackName := fmt.Sprintf("tf-acc-test-basic-%s", acctest.RandString(10))

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloudFormationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAwsServerlessRepositoryApplicationConfig(stackName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "application_id", "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"),
					resource.TestCheckResourceAttrSet("aws_serverlessrepository_application.postgres-rotator", "semantic_version"),
				),
			},
		},
	})
}

func TestAccAwsServerlessRepositoryApplication_Versioned(t *testing.T) {
	var stack cloudformation.Stack
	stackName := fmt.Sprintf("tf-acc-test-versioned-%s", acctest.RandString(10))
	const version = "1.0.15"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloudFormationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, version),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "semantic_version", version),
				),
			},
		},
	})
}

func TestAccAwsServerlessRepositoryApplication_UpdateVersion(t *testing.T) {
	var stack cloudformation.Stack
	stackName := fmt.Sprintf("tf-acc-test-update-%s", acctest.RandString(10))
	const initialVersion = "1.0.15"
	const updateVersion = "1.0.36"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSCloudFormationDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, initialVersion),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "application_id", "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "semantic_version", initialVersion),
				),
			},
			{
				Config: testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, updateVersion),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckerverlessRepositoryApplicationExists("aws_serverlessrepository_application.postgres-rotator", &stack),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "application_id", "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"),
					resource.TestCheckResourceAttr("aws_serverlessrepository_application.postgres-rotator", "semantic_version", updateVersion),
				),
			},
		},
	})
}

func testAccAwsServerlessRepositoryApplicationConfig(stackName string) string {
	return fmt.Sprintf(`
resource "aws_serverlessrepository_application" "postgres-rotator" {
  name           = "%[1]s"
  application_id = "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"
  parameters = {
    functionName = "func-%[1]s"
    endpoint     = "secretsmanager.us-east-2.amazonaws.com"
  }
}`, stackName)
}

func testAccAWSServerlessRepositoryApplicationConfig_versioned(stackName, version string) string {
	return fmt.Sprintf(`
resource "aws_serverlessrepository_application" "postgres-rotator" {
  name             = "%[1]s"
  application_id   = "arn:aws:serverlessrepo:us-east-1:297356227824:applications/SecretsManagerRDSPostgreSQLRotationSingleUser"
  semantic_version = "%[2]s"
  parameters = {
    functionName = "func-%[1]s"
    endpoint     = "secretsmanager.us-east-2.amazonaws.com"
  }
}`, stackName, version)
}

func testAccCheckerverlessRepositoryApplicationExists(n string, stack *cloudformation.Stack) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		conn := testAccProvider.Meta().(*AWSClient).cfconn
		params := &cloudformation.DescribeStacksInput{
			StackName: aws.String(rs.Primary.ID),
		}
		resp, err := conn.DescribeStacks(params)
		if err != nil {
			return err
		}
		if len(resp.Stacks) == 0 {
			return fmt.Errorf("CloudFormation stack not found")
		}

		return nil
	}
}
<|endoftext|>


<commit_before>// Package fserrors provides errors and error handling
package fserrors

import (
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"

	"github.com/pkg/errors"
)

// Retrier is an optional interface for error as to whether the
// operation should be retried at a high level.
//
// This should be returned from Update or Put methods as required
type Retrier interface {
	error
	Retry() bool
}

// retryError is a type of error
type retryError string

// Error interface
func (r retryError) Error() string {
	return string(r)
}

// Retry interface
func (r retryError) Retry() bool {
	return true
}

// Check interface
var _ Retrier = retryError("")

// RetryErrorf makes an error which indicates it would like to be retried
func RetryErrorf(format string, a ...interface{}) error {
	return retryError(fmt.Sprintf(format, a...))
}

// wrappedRetryError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedRetryError struct {
	error
}

// Retry interface
func (err wrappedRetryError) Retry() bool {
	return true
}

// Check interface
var _ Retrier = wrappedRetryError{(error)(nil)}

// RetryError makes an error which indicates it would like to be retried
func RetryError(err error) error {
	if err == nil {
		err = errors.New("needs retry")
	}
	return wrappedRetryError{err}
}

// IsRetryError returns true if err conforms to the Retry interface
// and calling the Retry method returns true.
func IsRetryError(err error) bool {
	if err == nil {
		return false
	}
	err = errors.Cause(err)
	if r, ok := err.(Retrier); ok {
		return r.Retry()
	}
	return false
}

// Fataler is an optional interface for error as to whether the
// operation should cause the entire operation to finish immediately.
//
// This should be returned from Update or Put methods as required
type Fataler interface {
	error
	Fatal() bool
}

// wrappedFatalError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedFatalError struct {
	error
}

// Fatal interface
func (err wrappedFatalError) Fatal() bool {
	return true
}

// Check interface
var _ Fataler = wrappedFatalError{(error)(nil)}

// FatalError makes an error which indicates it is a fatal error and
// the sync should stop.
func FatalError(err error) error {
	if err == nil {
		err = errors.New("fatal error")
	}
	return wrappedFatalError{err}
}

// IsFatalError returns true if err conforms to the Fatal interface
// and calling the Fatal method returns true.
func IsFatalError(err error) bool {
	if err == nil {
		return false
	}
	err = errors.Cause(err)
	if r, ok := err.(Fataler); ok {
		return r.Fatal()
	}
	return false
}

// NoRetrier is an optional interface for error as to whether the
// operation should not be retried at a high level.
//
// If only NoRetry errors are returned in a sync then the sync won't
// be retried.
//
// This should be returned from Update or Put methods as required
type NoRetrier interface {
	error
	NoRetry() bool
}

// wrappedNoRetryError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedNoRetryError struct {
	error
}

// NoRetry interface
func (err wrappedNoRetryError) NoRetry() bool {
	return true
}

// Check interface
var _ NoRetrier = wrappedNoRetryError{(error)(nil)}

// NoRetryError makes an error which indicates the sync shouldn't be
// retried.
func NoRetryError(err error) error {
	return wrappedNoRetryError{err}
}

// IsNoRetryError returns true if err conforms to the NoRetry
// interface and calling the NoRetry method returns true.
func IsNoRetryError(err error) bool {
	if err == nil {
		return false
	}
	err = errors.Cause(err)
	if r, ok := err.(NoRetrier); ok {
		return r.NoRetry()
	}
	return false
}

// Cause is a souped up errors.Cause which can unwrap some standard
// library errors too.  It returns true if any of the intermediate
// errors had a Timeout() or Temporary() method which returned true.
func Cause(cause error) (retriable bool, err error) {
	err = cause
	for prev := err; err != nil; prev = err {
		// Check for net error Timeout()
		if x, ok := err.(interface {
			Timeout() bool
		}); ok && x.Timeout() {
			retriable = true
		}

		// Check for net error Temporary()
		if x, ok := err.(interface {
			Temporary() bool
		}); ok && x.Temporary() {
			retriable = true
		}

		// Unwrap 1 level if possible
		err = errors.Cause(err)
		if err == prev {
			// Unpack any struct or *struct with a field
			// of name Err which satisfies the error
			// interface.  This includes *url.Error,
			// *net.OpError, *os.SyscallError and many
			// others in the stdlib
			errType := reflect.TypeOf(err)
			errValue := reflect.ValueOf(err)
			if errType.Kind() == reflect.Ptr {
				errType = errType.Elem()
				errValue = errValue.Elem()
			}
			if errType.Kind() == reflect.Struct {
				if errField := errValue.FieldByName("Err"); errField.IsValid() {
					errFieldValue := errField.Interface()
					if newErr, ok := errFieldValue.(error); ok {
						err = newErr
					}
				}
			}
		}
		if err == prev {
			break
		}
	}
	return retriable, err
}

// retriableErrorStrings is a list of phrases which when we find it
// in an an error, we know it is a networking error which should be
// retried.
//
// This is incredibly ugly - if only errors.Cause worked for all
// errors and all errors were exported from the stdlib.
var retriableErrorStrings = []string{
	"use of closed network connection", // internal/poll/fd.go
	"unexpected EOF reading trailer",   // net/http/transfer.go
	"transport connection broken",      // net/http/transport.go
	"http: ContentLength=",             // net/http/transfer.go
}

// Errors which indicate networking errors which should be retried
//
// These are added to in retriable_errors*.go
var retriableErrors = []error{
	io.EOF,
	io.ErrUnexpectedEOF,
}

// ShouldRetry looks at an error and tries to work out if retrying the
// operation that caused it would be a good idea. It returns true if
// the error implements Timeout() or Temporary() or if the error
// indicates a premature closing of the connection.
func ShouldRetry(err error) bool {
	if err == nil {
		return false
	}

	// Find root cause if available
	retriable, err := Cause(err)
	if retriable {
		return true
	}

	// Check if it is a retriable error
	for _, retriableErr := range retriableErrors {
		if err == retriableErr {
			return true
		}
	}

	// Check error strings (yuch!) too
	errString := err.Error()
	for _, phrase := range retriableErrorStrings {
		if strings.Contains(errString, phrase) {
			return true
		}
	}

	return false
}

// ShouldRetryHTTP returns a boolean as to whether this resp deserves.
// It checks to see if the HTTP response code is in the slice
// retryErrorCodes.
func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
	if resp == nil {
		return false
	}
	for _, e := range retryErrorCodes {
		if resp.StatusCode == e {
			return true
		}
	}
	return false
}
<commit_msg>fserrors: Look deeper into errors for Fatal/Retry/NoRetry errors.<commit_after>// Package fserrors provides errors and error handling
package fserrors

import (
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"

	"github.com/pkg/errors"
)

// Retrier is an optional interface for error as to whether the
// operation should be retried at a high level.
//
// This should be returned from Update or Put methods as required
type Retrier interface {
	error
	Retry() bool
}

// retryError is a type of error
type retryError string

// Error interface
func (r retryError) Error() string {
	return string(r)
}

// Retry interface
func (r retryError) Retry() bool {
	return true
}

// Check interface
var _ Retrier = retryError("")

// RetryErrorf makes an error which indicates it would like to be retried
func RetryErrorf(format string, a ...interface{}) error {
	return retryError(fmt.Sprintf(format, a...))
}

// wrappedRetryError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedRetryError struct {
	error
}

// Retry interface
func (err wrappedRetryError) Retry() bool {
	return true
}

// Check interface
var _ Retrier = wrappedRetryError{(error)(nil)}

// RetryError makes an error which indicates it would like to be retried
func RetryError(err error) error {
	if err == nil {
		err = errors.New("needs retry")
	}
	return wrappedRetryError{err}
}

// IsRetryError returns true if err conforms to the Retry interface
// and calling the Retry method returns true.
func IsRetryError(err error) bool {
	if err == nil {
		return false
	}
	_, err = Cause(err)
	if r, ok := err.(Retrier); ok {
		return r.Retry()
	}
	return false
}

// Fataler is an optional interface for error as to whether the
// operation should cause the entire operation to finish immediately.
//
// This should be returned from Update or Put methods as required
type Fataler interface {
	error
	Fatal() bool
}

// wrappedFatalError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedFatalError struct {
	error
}

// Fatal interface
func (err wrappedFatalError) Fatal() bool {
	return true
}

// Check interface
var _ Fataler = wrappedFatalError{(error)(nil)}

// FatalError makes an error which indicates it is a fatal error and
// the sync should stop.
func FatalError(err error) error {
	if err == nil {
		err = errors.New("fatal error")
	}
	return wrappedFatalError{err}
}

// IsFatalError returns true if err conforms to the Fatal interface
// and calling the Fatal method returns true.
func IsFatalError(err error) bool {
	if err == nil {
		return false
	}
	_, err = Cause(err)
	if r, ok := err.(Fataler); ok {
		return r.Fatal()
	}
	return false
}

// NoRetrier is an optional interface for error as to whether the
// operation should not be retried at a high level.
//
// If only NoRetry errors are returned in a sync then the sync won't
// be retried.
//
// This should be returned from Update or Put methods as required
type NoRetrier interface {
	error
	NoRetry() bool
}

// wrappedNoRetryError is an error wrapped so it will satisfy the
// Retrier interface and return true
type wrappedNoRetryError struct {
	error
}

// NoRetry interface
func (err wrappedNoRetryError) NoRetry() bool {
	return true
}

// Check interface
var _ NoRetrier = wrappedNoRetryError{(error)(nil)}

// NoRetryError makes an error which indicates the sync shouldn't be
// retried.
func NoRetryError(err error) error {
	return wrappedNoRetryError{err}
}

// IsNoRetryError returns true if err conforms to the NoRetry
// interface and calling the NoRetry method returns true.
func IsNoRetryError(err error) bool {
	if err == nil {
		return false
	}
	_, err = Cause(err)
	if r, ok := err.(NoRetrier); ok {
		return r.NoRetry()
	}
	return false
}

// Cause is a souped up errors.Cause which can unwrap some standard
// library errors too.  It returns true if any of the intermediate
// errors had a Timeout() or Temporary() method which returned true.
func Cause(cause error) (retriable bool, err error) {
	err = cause
	for prev := err; err != nil; prev = err {
		// Check for net error Timeout()
		if x, ok := err.(interface {
			Timeout() bool
		}); ok && x.Timeout() {
			retriable = true
		}

		// Check for net error Temporary()
		if x, ok := err.(interface {
			Temporary() bool
		}); ok && x.Temporary() {
			retriable = true
		}

		// Unwrap 1 level if possible
		err = errors.Cause(err)
		if err == prev {
			// Unpack any struct or *struct with a field
			// of name Err which satisfies the error
			// interface.  This includes *url.Error,
			// *net.OpError, *os.SyscallError and many
			// others in the stdlib
			errType := reflect.TypeOf(err)
			errValue := reflect.ValueOf(err)
			if errType.Kind() == reflect.Ptr {
				errType = errType.Elem()
				errValue = errValue.Elem()
			}
			if errType.Kind() == reflect.Struct {
				if errField := errValue.FieldByName("Err"); errField.IsValid() {
					errFieldValue := errField.Interface()
					if newErr, ok := errFieldValue.(error); ok {
						err = newErr
					}
				}
			}
		}
		if err == prev {
			break
		}
	}
	return retriable, err
}

// retriableErrorStrings is a list of phrases which when we find it
// in an an error, we know it is a networking error which should be
// retried.
//
// This is incredibly ugly - if only errors.Cause worked for all
// errors and all errors were exported from the stdlib.
var retriableErrorStrings = []string{
	"use of closed network connection", // internal/poll/fd.go
	"unexpected EOF reading trailer",   // net/http/transfer.go
	"transport connection broken",      // net/http/transport.go
	"http: ContentLength=",             // net/http/transfer.go
}

// Errors which indicate networking errors which should be retried
//
// These are added to in retriable_errors*.go
var retriableErrors = []error{
	io.EOF,
	io.ErrUnexpectedEOF,
}

// ShouldRetry looks at an error and tries to work out if retrying the
// operation that caused it would be a good idea. It returns true if
// the error implements Timeout() or Temporary() or if the error
// indicates a premature closing of the connection.
func ShouldRetry(err error) bool {
	if err == nil {
		return false
	}

	// Find root cause if available
	retriable, err := Cause(err)
	if retriable {
		return true
	}

	// Check if it is a retriable error
	for _, retriableErr := range retriableErrors {
		if err == retriableErr {
			return true
		}
	}

	// Check error strings (yuch!) too
	errString := err.Error()
	for _, phrase := range retriableErrorStrings {
		if strings.Contains(errString, phrase) {
			return true
		}
	}

	return false
}

// ShouldRetryHTTP returns a boolean as to whether this resp deserves.
// It checks to see if the HTTP response code is in the slice
// retryErrorCodes.
func ShouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
	if resp == nil {
		return false
	}
	for _, e := range retryErrorCodes {
		if resp.StatusCode == e {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package events

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	eventsv1 "k8s.io/api/events/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/record/util"
	"k8s.io/client-go/tools/reference"
	"k8s.io/klog/v2"
)

type recorderImpl struct {
	scheme              *runtime.Scheme
	reportingController string
	reportingInstance   string
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{}) {
	timestamp := metav1.MicroTime{time.Now()}
	message := fmt.Sprintf(note, args...)
	refRegarding, err := reference.GetReference(recorder.scheme, regarding)
	if err != nil {
		klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", regarding, err, eventtype, reason, message)
		return
	}
	refRelated, err := reference.GetReference(recorder.scheme, related)
	if err != nil {
		klog.V(9).Infof("Could not construct reference to: '%#v' due to: '%v'.", related, err)
	}
	if !util.ValidateEventType(eventtype) {
		klog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}
	event := recorder.makeEvent(refRegarding, refRelated, timestamp, eventtype, reason, message, recorder.reportingController, recorder.reportingInstance, action)
	go func() {
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func (recorder *recorderImpl) makeEvent(refRegarding *v1.ObjectReference, refRelated *v1.ObjectReference, timestamp metav1.MicroTime, eventtype, reason, message string, reportingController string, reportingInstance string, action string) *eventsv1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := refRegarding.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &eventsv1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", refRegarding.Name, t.UnixNano()),
			Namespace: namespace,
		},
		EventTime:           timestamp,
		Series:              nil,
		ReportingController: reportingController,
		ReportingInstance:   reportingInstance,
		Action:              action,
		Reason:              reason,
		Regarding:           *refRegarding,
		Related:             refRelated,
		Note:                message,
		Type:                eventtype,
		// TODO: remove this when we change conversion to convert eventSource
		// to reportingController
		DeprecatedSource: v1.EventSource{Component: reportingController},
	}
}
<commit_msg>Remove DeprecatedSource assignment to avoid validation failure<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package events

import (
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	eventsv1 "k8s.io/api/events/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/clock"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/record/util"
	"k8s.io/client-go/tools/reference"
	"k8s.io/klog/v2"
)

type recorderImpl struct {
	scheme              *runtime.Scheme
	reportingController string
	reportingInstance   string
	*watch.Broadcaster
	clock clock.Clock
}

func (recorder *recorderImpl) Eventf(regarding runtime.Object, related runtime.Object, eventtype, reason, action, note string, args ...interface{}) {
	timestamp := metav1.MicroTime{time.Now()}
	message := fmt.Sprintf(note, args...)
	refRegarding, err := reference.GetReference(recorder.scheme, regarding)
	if err != nil {
		klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", regarding, err, eventtype, reason, message)
		return
	}
	refRelated, err := reference.GetReference(recorder.scheme, related)
	if err != nil {
		klog.V(9).Infof("Could not construct reference to: '%#v' due to: '%v'.", related, err)
	}
	if !util.ValidateEventType(eventtype) {
		klog.Errorf("Unsupported event type: '%v'", eventtype)
		return
	}
	event := recorder.makeEvent(refRegarding, refRelated, timestamp, eventtype, reason, message, recorder.reportingController, recorder.reportingInstance, action)
	go func() {
		defer utilruntime.HandleCrash()
		recorder.Action(watch.Added, event)
	}()
}

func (recorder *recorderImpl) makeEvent(refRegarding *v1.ObjectReference, refRelated *v1.ObjectReference, timestamp metav1.MicroTime, eventtype, reason, message string, reportingController string, reportingInstance string, action string) *eventsv1.Event {
	t := metav1.Time{Time: recorder.clock.Now()}
	namespace := refRegarding.Namespace
	if namespace == "" {
		namespace = metav1.NamespaceDefault
	}
	return &eventsv1.Event{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%v.%x", refRegarding.Name, t.UnixNano()),
			Namespace: namespace,
		},
		EventTime:           timestamp,
		Series:              nil,
		ReportingController: reportingController,
		ReportingInstance:   reportingInstance,
		Action:              action,
		Reason:              reason,
		Regarding:           *refRegarding,
		Related:             refRelated,
		Note:                message,
		Type:                eventtype,
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"fmt"
	"reflect"
	"strings"
)

import (
	"github.com/lxn/walk"
)

var conditionsByName = make(map[string]walk.Condition)

func MustRegisterCondition(name string, condition walk.Condition) {
	if name == "" {
		panic(`name == ""`)
	}
	if condition == nil {
		panic("condition == nil")
	}
	if _, ok := conditionsByName[name]; ok {
		panic("name already registered")
	}

	conditionsByName[name] = condition
}

type declWidget struct {
	d Widget
	w walk.Widget
}

type Builder struct {
	level                    int
	parent                   walk.Container
	declWidgets              []declWidget
	name2Widget              map[string]walk.Widget
	deferredFuncs            []func() error
	knownCompositeConditions map[string]walk.Condition
}

func NewBuilder(parent walk.Container) *Builder {
	return &Builder{
		parent:                   parent,
		name2Widget:              make(map[string]walk.Widget),
		knownCompositeConditions: make(map[string]walk.Condition),
	}
}

func (b *Builder) Parent() walk.Container {
	return b.parent
}

func (b *Builder) Defer(f func() error) {
	b.deferredFuncs = append(b.deferredFuncs, f)
}

func (b *Builder) deferBuildMenuActions(menu *walk.Menu, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				if _, err := item.createAction(b, menu); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) deferBuildActions(actionList *walk.ActionList, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				action, err := item.createAction(b, nil)
				if err != nil {
					return err
				}
				if err := actionList.Add(action); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) InitWidget(d Widget, w walk.Widget, customInit func() error) error {
	b.level++
	defer func() {
		b.level--
	}()

	var succeeded bool
	defer func() {
		if !succeeded {
			w.Dispose()
		}
	}()

	b.declWidgets = append(b.declWidgets, declWidget{d, w})

	// Widget
	name, _, _, font, toolTipText, minSize, maxSize, stretchFactor, row, rowSpan, column, columnSpan, contextMenuItems, onKeyDown, onMouseDown, onMouseMove, onMouseUp, onSizeChanged := d.WidgetInfo()

	w.SetName(name)

	if name != "" {
		b.name2Widget[name] = w
	}

	if toolTipText != "" {
		if err := w.SetToolTipText(toolTipText); err != nil {
			return err
		}
	}

	if err := w.SetMinMaxSize(minSize.toW(), maxSize.toW()); err != nil {
		return err
	}

	if len(contextMenuItems) > 0 {
		cm, err := walk.NewMenu()
		if err != nil {
			return err
		}

		b.deferBuildMenuActions(cm, contextMenuItems)

		w.SetContextMenu(cm)
	}

	if onKeyDown != nil {
		w.KeyDown().Attach(onKeyDown)
	}

	if onMouseDown != nil {
		w.MouseDown().Attach(onMouseDown)
	}

	if onMouseMove != nil {
		w.MouseMove().Attach(onMouseMove)
	}

	if onMouseUp != nil {
		w.MouseUp().Attach(onMouseUp)
	}

	if onSizeChanged != nil {
		w.SizeChanged().Attach(onSizeChanged)
	}

	if p := w.Parent(); p != nil {
		switch l := p.Layout().(type) {
		case *walk.BoxLayout:
			if stretchFactor < 1 {
				stretchFactor = 1
			}
			if err := l.SetStretchFactor(w, stretchFactor); err != nil {
				return err
			}

		case *walk.GridLayout:
			cs := columnSpan
			if cs < 1 {
				cs = 1
			}
			rs := rowSpan
			if rs < 1 {
				rs = 1
			}
			r := walk.Rectangle{column, row, cs, rs}

			if err := l.SetRange(w, r); err != nil {
				return err
			}
		}
	}

	oldParent := b.parent

	// Container
	var db *walk.DataBinder
	if dc, ok := d.(Container); ok {
		if wc, ok := w.(walk.Container); ok {
			dataBinder, layout, children := dc.ContainerInfo()

			if layout != nil {
				l, err := layout.Create()
				if err != nil {
					return err
				}

				if err := wc.SetLayout(l); err != nil {
					return err
				}
			}

			b.parent = wc
			defer func() {
				b.parent = oldParent
			}()

			for _, child := range children {
				if err := child.Create(b); err != nil {
					return err
				}
			}

			var err error
			if db, err = dataBinder.create(); err != nil {
				return err
			}
		}
	}

	// Custom
	if customInit != nil {
		if err := customInit(); err != nil {
			return err
		}
	}

	b.parent = oldParent

	// Widget continued
	if font != nil {
		if f, err := font.Create(); err != nil {
			return err
		} else if f != nil {
			w.SetFont(f)
		}
	}

	if b.level == 1 {
		if err := b.initProperties(); err != nil {
			return err
		}
	}

	// Call Reset on DataBinder after customInit, so a Dialog gets a chance to first
	// wire up its DefaultButton to the CanSubmitChanged event of a DataBinder.
	if db != nil {
		if _, ok := d.(Container); ok {
			if wc, ok := w.(walk.Container); ok {
				// FIXME: Currently SetDataBinder must be called after initProperties.
				wc.SetDataBinder(db)

				if err := db.Reset(); err != nil {
					return err
				}
			}
		}
	}

	if b.level == 1 {
		for _, f := range b.deferredFuncs {
			if err := f(); err != nil {
				return err
			}
		}
	}

	succeeded = true

	return nil
}

func (b *Builder) initProperties() error {
	for _, dw := range b.declWidgets {
		d, w := dw.d, dw.w

		sv := reflect.ValueOf(d)
		st := sv.Type()
		if st.Kind() != reflect.Struct {
			panic("d must be a struct value")
		}

		wb := w.BaseWidget()

		fieldCount := st.NumField()
		for i := 0; i < fieldCount; i++ {
			sf := st.Field(i)

			prop := wb.Property(sf.Name)

			switch val := sv.Field(i).Interface().(type) {
			case nil:
				// nop

			case bindData:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				src := b.conditionOrProperty(val)

				if src == nil {
					// No luck so far, so we assume the expression refers to
					// something in the data source.
					src = val.expression

					if val.validator != nil {
						validator, err := val.validator.Create()
						if err != nil {
							return err
						}
						if err := prop.SetValidator(validator); err != nil {
							return err
						}
					}
				}

				if err := prop.SetSource(src); err != nil {
					return err
				}

			case walk.Condition:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				if err := prop.SetSource(val); err != nil {
					return err
				}

			default:
				if prop == nil {
					continue
				}

				v := prop.Get()
				valt, vt := reflect.TypeOf(val), reflect.TypeOf(v)

				if valt != vt {
					panic(fmt.Sprintf("cannot assign value %v of type %T to property %s of type %T", val, val, sf.Name, v))
				}
				if err := prop.Set(val); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (b *Builder) conditionOrProperty(data Property) interface{} {
	switch val := data.(type) {
	case bindData:
		if c, ok := b.knownCompositeConditions[val.expression]; ok {
			return c
		} else if conds := strings.Split(val.expression, "&&"); len(conds) > 1 {
			// This looks like a composite condition.
			for i, s := range conds {
				conds[i] = strings.TrimSpace(s)
			}

			var conditions []walk.Condition

			for _, cond := range conds {
				if p := b.property(cond); p != nil {
					conditions = append(conditions, p.(walk.Condition))
				} else if c, ok := conditionsByName[cond]; ok {
					conditions = append(conditions, c)
				} else {
					panic("unknown condition or property name: " + cond)
				}
			}

			var condition walk.Condition
			if len(conditions) > 1 {
				condition = walk.NewAllCondition(conditions...)
				b.knownCompositeConditions[val.expression] = condition
			} else {
				condition = conditions[0]
			}

			return condition
		}

		if p := b.property(val.expression); p != nil {
			return p
		}

		return conditionsByName[val.expression]

	case walk.Condition:
		return val
	}

	return nil
}

func (b *Builder) property(expression string) walk.Property {
	if parts := strings.Split(expression, "."); len(parts) == 2 {
		if sw, ok := b.name2Widget[parts[0]]; ok {
			return sw.BaseWidget().Property(parts[1])
		}
	}

	return nil
}
<commit_msg>declarative/Builder: Defer Container.SetDataBinder call until after initProperties<commit_after>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package declarative

import (
	"fmt"
	"reflect"
	"strings"
)

import (
	"github.com/lxn/walk"
)

var conditionsByName = make(map[string]walk.Condition)

func MustRegisterCondition(name string, condition walk.Condition) {
	if name == "" {
		panic(`name == ""`)
	}
	if condition == nil {
		panic("condition == nil")
	}
	if _, ok := conditionsByName[name]; ok {
		panic("name already registered")
	}

	conditionsByName[name] = condition
}

type declWidget struct {
	d Widget
	w walk.Widget
}

type Builder struct {
	level                    int
	parent                   walk.Container
	declWidgets              []declWidget
	name2Widget              map[string]walk.Widget
	deferredFuncs            []func() error
	knownCompositeConditions map[string]walk.Condition
}

func NewBuilder(parent walk.Container) *Builder {
	return &Builder{
		parent:                   parent,
		name2Widget:              make(map[string]walk.Widget),
		knownCompositeConditions: make(map[string]walk.Condition),
	}
}

func (b *Builder) Parent() walk.Container {
	return b.parent
}

func (b *Builder) Defer(f func() error) {
	b.deferredFuncs = append(b.deferredFuncs, f)
}

func (b *Builder) deferBuildMenuActions(menu *walk.Menu, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				if _, err := item.createAction(b, menu); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) deferBuildActions(actionList *walk.ActionList, items []MenuItem) {
	if len(items) > 0 {
		b.Defer(func() error {
			for _, item := range items {
				action, err := item.createAction(b, nil)
				if err != nil {
					return err
				}
				if err := actionList.Add(action); err != nil {
					return err
				}
			}

			return nil
		})
	}
}

func (b *Builder) InitWidget(d Widget, w walk.Widget, customInit func() error) error {
	b.level++
	defer func() {
		b.level--
	}()

	var succeeded bool
	defer func() {
		if !succeeded {
			w.Dispose()
		}
	}()

	b.declWidgets = append(b.declWidgets, declWidget{d, w})

	// Widget
	name, _, _, font, toolTipText, minSize, maxSize, stretchFactor, row, rowSpan, column, columnSpan, contextMenuItems, onKeyDown, onMouseDown, onMouseMove, onMouseUp, onSizeChanged := d.WidgetInfo()

	w.SetName(name)

	if name != "" {
		b.name2Widget[name] = w
	}

	if toolTipText != "" {
		if err := w.SetToolTipText(toolTipText); err != nil {
			return err
		}
	}

	if err := w.SetMinMaxSize(minSize.toW(), maxSize.toW()); err != nil {
		return err
	}

	if len(contextMenuItems) > 0 {
		cm, err := walk.NewMenu()
		if err != nil {
			return err
		}

		b.deferBuildMenuActions(cm, contextMenuItems)

		w.SetContextMenu(cm)
	}

	if onKeyDown != nil {
		w.KeyDown().Attach(onKeyDown)
	}

	if onMouseDown != nil {
		w.MouseDown().Attach(onMouseDown)
	}

	if onMouseMove != nil {
		w.MouseMove().Attach(onMouseMove)
	}

	if onMouseUp != nil {
		w.MouseUp().Attach(onMouseUp)
	}

	if onSizeChanged != nil {
		w.SizeChanged().Attach(onSizeChanged)
	}

	if p := w.Parent(); p != nil {
		switch l := p.Layout().(type) {
		case *walk.BoxLayout:
			if stretchFactor < 1 {
				stretchFactor = 1
			}
			if err := l.SetStretchFactor(w, stretchFactor); err != nil {
				return err
			}

		case *walk.GridLayout:
			cs := columnSpan
			if cs < 1 {
				cs = 1
			}
			rs := rowSpan
			if rs < 1 {
				rs = 1
			}
			r := walk.Rectangle{column, row, cs, rs}

			if err := l.SetRange(w, r); err != nil {
				return err
			}
		}
	}

	oldParent := b.parent

	// Container
	var db *walk.DataBinder
	if dc, ok := d.(Container); ok {
		if wc, ok := w.(walk.Container); ok {
			dataBinder, layout, children := dc.ContainerInfo()

			if layout != nil {
				l, err := layout.Create()
				if err != nil {
					return err
				}

				if err := wc.SetLayout(l); err != nil {
					return err
				}
			}

			b.parent = wc
			defer func() {
				b.parent = oldParent
			}()

			for _, child := range children {
				if err := child.Create(b); err != nil {
					return err
				}
			}

			var err error
			if db, err = dataBinder.create(); err != nil {
				return err
			}
		}
	}

	// Custom
	if customInit != nil {
		if err := customInit(); err != nil {
			return err
		}
	}

	b.parent = oldParent

	// Widget continued
	if font != nil {
		if f, err := font.Create(); err != nil {
			return err
		} else if f != nil {
			w.SetFont(f)
		}
	}

	if b.level == 1 {
		if err := b.initProperties(); err != nil {
			return err
		}
	}

	// Call Reset on DataBinder after customInit, so a Dialog gets a chance to first
	// wire up its DefaultButton to the CanSubmitChanged event of a DataBinder.
	if db != nil {
		if _, ok := d.(Container); ok {
			if wc, ok := w.(walk.Container); ok {
				b.Defer(func() error {
					// FIXME: Currently SetDataBinder must be called after initProperties.
					wc.SetDataBinder(db)

					return db.Reset()
				})
			}
		}
	}

	if b.level == 1 {
		for _, f := range b.deferredFuncs {
			if err := f(); err != nil {
				return err
			}
		}
	}

	succeeded = true

	return nil
}

func (b *Builder) initProperties() error {
	for _, dw := range b.declWidgets {
		d, w := dw.d, dw.w

		sv := reflect.ValueOf(d)
		st := sv.Type()
		if st.Kind() != reflect.Struct {
			panic("d must be a struct value")
		}

		wb := w.BaseWidget()

		fieldCount := st.NumField()
		for i := 0; i < fieldCount; i++ {
			sf := st.Field(i)

			prop := wb.Property(sf.Name)

			switch val := sv.Field(i).Interface().(type) {
			case nil:
				// nop

			case bindData:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				src := b.conditionOrProperty(val)

				if src == nil {
					// No luck so far, so we assume the expression refers to
					// something in the data source.
					src = val.expression

					if val.validator != nil {
						validator, err := val.validator.Create()
						if err != nil {
							return err
						}
						if err := prop.SetValidator(validator); err != nil {
							return err
						}
					}
				}

				if err := prop.SetSource(src); err != nil {
					return err
				}

			case walk.Condition:
				if prop == nil {
					panic(sf.Name + " is not a property")
				}

				if err := prop.SetSource(val); err != nil {
					return err
				}

			default:
				if prop == nil {
					continue
				}

				v := prop.Get()
				valt, vt := reflect.TypeOf(val), reflect.TypeOf(v)

				if valt != vt {
					panic(fmt.Sprintf("cannot assign value %v of type %T to property %s of type %T", val, val, sf.Name, v))
				}
				if err := prop.Set(val); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func (b *Builder) conditionOrProperty(data Property) interface{} {
	switch val := data.(type) {
	case bindData:
		if c, ok := b.knownCompositeConditions[val.expression]; ok {
			return c
		} else if conds := strings.Split(val.expression, "&&"); len(conds) > 1 {
			// This looks like a composite condition.
			for i, s := range conds {
				conds[i] = strings.TrimSpace(s)
			}

			var conditions []walk.Condition

			for _, cond := range conds {
				if p := b.property(cond); p != nil {
					conditions = append(conditions, p.(walk.Condition))
				} else if c, ok := conditionsByName[cond]; ok {
					conditions = append(conditions, c)
				} else {
					panic("unknown condition or property name: " + cond)
				}
			}

			var condition walk.Condition
			if len(conditions) > 1 {
				condition = walk.NewAllCondition(conditions...)
				b.knownCompositeConditions[val.expression] = condition
			} else {
				condition = conditions[0]
			}

			return condition
		}

		if p := b.property(val.expression); p != nil {
			return p
		}

		return conditionsByName[val.expression]

	case walk.Condition:
		return val
	}

	return nil
}

func (b *Builder) property(expression string) walk.Property {
	if parts := strings.Split(expression, "."); len(parts) == 2 {
		if sw, ok := b.name2Widget[parts[0]]; ok {
			return sw.BaseWidget().Property(parts[1])
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/obscuren/mutan"
	"strings"
)

// General compile function
func Compile(script string) ([]byte, error) {
	asm, errors := mutan.Compile(strings.NewReader(script), false)
	if len(errors) > 0 {
		var errs string
		for _, er := range errors {
			if er != nil {
				errs += er.Error()
			}
		}
		return nil, fmt.Errorf("%v", errs)
	}

	return ethutil.Assemble(asm...), nil
}

func CompileScript(script string) ([]byte, []byte, error) {
	// Preprocess
	mainInput, initInput := ethutil.PreProcess(script)
	// Compile main script
	mainScript, err := Compile(mainInput)
	if err != nil {
		return nil, nil, err
	}

	// Compile init script
	initScript, err := Compile(initInput)
	if err != nil {
		return nil, nil, err
	}

	return mainScript, initScript, nil
}
<commit_msg>Using mutan assembler stage<commit_after>package utils

import (
	"fmt"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/obscuren/mutan"
	"strings"
)

// General compile function
func Compile(script string) ([]byte, error) {
	byteCode, errors := mutan.Compile(strings.NewReader(script), false)
	if len(errors) > 0 {
		var errs string
		for _, er := range errors {
			if er != nil {
				errs += er.Error()
			}
		}
		return nil, fmt.Errorf("%v", errs)
	}

	return byteCode, nil
}

func CompileScript(script string) ([]byte, []byte, error) {
	// Preprocess
	mainInput, initInput := ethutil.PreProcess(script)
	// Compile main script
	mainScript, err := Compile(mainInput)
	if err != nil {
		return nil, nil, err
	}

	// Compile init script
	initScript, err := Compile(initInput)
	if err != nil {
		return nil, nil, err
	}

	return mainScript, initScript, nil
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"io"
	"net"
	"os"

	"reflect"

	"github.com/containernetworking/cni/pkg/ip"
	"github.com/containernetworking/cni/pkg/ns"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types/current"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

// DoNetworking performs the networking for the given config and IPAM result
func DoNetworking(args *skel.CmdArgs, conf NetConf, result *current.Result, logger *log.Entry, desiredVethName string) (hostVethName, contVethMAC string, err error) {
	// Select the first 11 characters of the containerID for the host veth.
	hostVethName = "cali" + args.ContainerID[:Min(11, len(args.ContainerID))]
	contVethName := args.IfName
	var hasIPv4, hasIPv6 bool

	// If a desired veth name was passed in, use that instead.
	if desiredVethName != "" {
		hostVethName = desiredVethName
	}

	// Clean up if hostVeth exists.
	if oldHostVeth, err := netlink.LinkByName(hostVethName); err == nil {
		if err = netlink.LinkDel(oldHostVeth); err != nil {
			return "", "", fmt.Errorf("failed to delete old hostVeth %v: %v", hostVethName, err)
		}
		logger.Infof("clean old hostVeth: %v", hostVethName)
	}

	err = ns.WithNetNSPath(args.Netns, func(hostNS ns.NetNS) error {
		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:  contVethName,
				Flags: net.FlagUp,
				MTU:   conf.MTU,
			},
			PeerName: hostVethName,
		}

		if err := netlink.LinkAdd(veth); err != nil {
			logger.Errorf("Error adding veth %+v: %s", veth, err)
			return err
		}

		hostVeth, err := netlink.LinkByName(hostVethName)
		if err != nil {
			err = fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
			return err
		}

		// Explicitly set the veth to UP state, because netlink doesn't always do that on all the platforms with net.FlagUp.
		// veth won't get a link local address unless it's set to UP state.
		if err = netlink.LinkSetUp(hostVeth); err != nil {
			return fmt.Errorf("failed to set %q up: %v", hostVethName, err)
		}

		contVeth, err := netlink.LinkByName(contVethName)
		if err != nil {
			err = fmt.Errorf("failed to lookup %q: %v", contVethName, err)
			return err
		}

		// Fetch the MAC from the container Veth. This is needed by Calico.
		contVethMAC = contVeth.Attrs().HardwareAddr.String()
		logger.WithField("MAC", contVethMAC).Debug("Found MAC for container veth")

		// At this point, the virtual ethernet pair has been created, and both ends have the right names.
		// Both ends of the veth are still in the container's network namespace.

		for _, addr := range result.IPs {

			// Before returning, create the routes inside the namespace, first for IPv4 then IPv6.
			if addr.Version == "4" {
				// Add a connected route to a dummy next hop so that a default route can be set
				gw := net.IPv4(169, 254, 1, 1)
				gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)}
				err := netlink.RouteAdd(
					&netlink.Route{
						LinkIndex: contVeth.Attrs().Index,
						Scope:     netlink.SCOPE_LINK,
						Dst:       gwNet,
					},
				)

				if err != nil {
					return fmt.Errorf("failed to add route inside the container: %v", err)
				}

				if err = ip.AddDefaultRoute(gw, contVeth); err != nil {
					return fmt.Errorf("failed to add the default route inside the container: %v", err)
				}

				if err = netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address}); err != nil {
					return fmt.Errorf("failed to add IP addr to %q: %v", contVethName, err)
				}
				// Set hasIPv4 to true so sysctls for IPv4 can be programmed when the host side of
				// the veth finishes moving to the host namespace.
				hasIPv4 = true
			}

			// Handle IPv6 routes
			if addr.Version == "6" {
				// No need to add a dummy next hop route as the host veth device will already have an IPv6
				// link local address that can be used as a next hop.
				// Just fetch the address of the host end of the veth and use it as the next hop.
				addresses, err := netlink.AddrList(hostVeth, netlink.FAMILY_V6)
				if err != nil {
					logger.Errorf("Error listing IPv6 addresses: %s", err)
					return err
				}

				if len(addresses) < 1 {
					// If the hostVeth doesn't have an IPv6 address then this host probably doesn't
					// support IPv6. Since a IPv6 address has been allocated that can't be used,
					// return an error.
					return fmt.Errorf("failed to get IPv6 addresses for host side of the veth pair")
				}

				hostIPv6Addr := addresses[0].IP

				_, defNet, _ := net.ParseCIDR("::/0")
				if err = ip.AddRoute(defNet, hostIPv6Addr, contVeth); err != nil {
					return fmt.Errorf("failed to add default gateway to %v %v", hostIPv6Addr, err)
				}

				if err = netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address}); err != nil {
					return fmt.Errorf("failed to add IP addr to %q: %v", contVeth, err)
				}

				// Set hasIPv6 to true so sysctls for IPv6 can be programmed when the host side of
				// the veth finishes moving to the host namespace.
				hasIPv6 = true
			}
		}

		// Now that the everything has been successfully set up in the container, move the "host" end of the
		// veth into the host namespace.
		if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
			return fmt.Errorf("failed to move veth to host netns: %v", err)
		}

		return nil
	})

	if err != nil {
		logger.Errorf("Error creating veth: %s", err)
		return "", "", err
	}

	err = configureSysctls(hostVethName, hasIPv4, hasIPv6)
	if err != nil {
		return "", "", fmt.Errorf("error configuring sysctls for interface: %s, error: %s", hostVethName, err)
	}

	// Moving a veth between namespaces always leaves it in the "DOWN" state. Set it back to "UP" now that we're
	// back in the host namespace.
	hostVeth, err := netlink.LinkByName(hostVethName)
	if err != nil {
		return "", "", fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
	}

	if err = netlink.LinkSetUp(hostVeth); err != nil {
		return "", "", fmt.Errorf("failed to set %q up: %v", hostVethName, err)
	}

	// Now that the host side of the veth is moved, state set to UP, and configured with sysctls, we can add the routes to it in the host namespace.
	err = setupRoutes(hostVeth, result)
	if err != nil {
		return "", "", fmt.Errorf("error adding host side routes for interface: %s, error: %s", hostVeth.Attrs().Name, err)
	}

	return hostVethName, contVethMAC, err
}

var errFileExists = fmt.Errorf("file exists")

// setupRoutes sets up the routes for the host side of the veth pair.
func setupRoutes(hostVeth netlink.Link, result *current.Result) error {

	// Go through all the IPs and add routes for each IP in the result.
	for _, ipAddr := range result.IPs {
		route := netlink.Route{
			LinkIndex: hostVeth.Attrs().Index,
			Scope:     netlink.SCOPE_LINK,
			Dst:       &ipAddr.Address,
		}
		err := netlink.RouteAdd(&route)

		if err != nil {
			switch err {

			// Route already exists, but not necessarily pointing to the same interface.
			case errFileExists:
				// List all the routes for the interface.
				routes, err := netlink.RouteList(hostVeth, netlink.FAMILY_ALL)
				if err != nil {
					return fmt.Errorf("error listing routes")
				}

				// Go through all the routes pointing to the interface, and see if any of them is
				// exactly what we are intending to program.
				// If the route we want is already there then most likely it's programmed by Felix, so we ignore it,
				// and we return an error if none of the routes match the route we're trying to program.
				for _, r := range routes {
					if reflect.DeepEqual(r, route) {
						// Route was already present on the host.
						log.Infof("CNI skipping add route. Route already exists for %s\n", hostVeth.Attrs().Name)
						return nil
					}
				}
				return fmt.Errorf("route (Dst: %s, Scope: %s) already exists for an interface other than '%s'", route.Dst.String(), route.Scope, hostVeth.Attrs().Name)
			default:
				return fmt.Errorf("failed to add route (Dst: %s, Scope: %s, Iface: %s): %v", route.Dst.String(), route.Scope, hostVeth.Attrs().Name, err)
			}
		}

		log.Debugf("CNI adding route for interface: %v, IP: %s", hostVeth, ipAddr.Address)
	}
	return nil
}

// configureSysctls configures necessary sysctls required for the host side of the veth pair for IPv4 and/or IPv6.
func configureSysctls(hostVethName string, hasIPv4, hasIPv6 bool) error {
	var err error

	if hasIPv4 {
		// Enable proxy ARP, this makes the host respond to all ARP requests with its own
		// MAC. We install explicit routes into the containers network
		// namespace and we use a link-local address for the gateway.  Turing on proxy ARP
		// means that we don't need to assign the link local address explicitly to each
		// host side of the veth, which is one fewer thing to maintain and one fewer
		// thing we may clash over.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/proxy_arp", hostVethName), "1"); err != nil {
			return err
		}

		// Normally, the kernel has a delay before responding to proxy ARP but we know
		// that's not needed in a Calico network so we disable it.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/neigh/%s/proxy_delay", hostVethName), "0"); err != nil {
			return err
		}

		// Enable IP forwarding of packets coming _from_ this interface.  For packets to
		// be forwarded in both directions we need this flag to be set on the fabric-facing
		// interface too (or for the global default to be set).
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/forwarding", hostVethName), "1"); err != nil {
			return err
		}
	}

	if hasIPv6 {
		// Enable proxy NDP, similarly to proxy ARP, described above in IPv4 section.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/proxy_ndp", hostVethName), "1"); err != nil {
			return err
		}

		// Enable IP forwarding of packets coming _from_ this interface.  For packets to
		// be forwarded in both directions we need this flag to be set on the fabric-facing
		// interface too (or for the global default to be set).
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/forwarding", hostVethName), "1"); err != nil {
			return err
		}
	}

	return nil
}

// writeProcSys takes the sysctl path and a string value to set i.e. "0" or "1" and sets the sysctl.
func writeProcSys(path, value string) error {
	f, err := os.OpenFile(path, os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	n, err := f.Write([]byte(value))
	if err == nil && n < len(value) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}
<commit_msg>Disable IP forwarding inside the container netns.<commit_after>package utils

import (
	"fmt"
	"io"
	"net"
	"os"

	"reflect"

	"github.com/containernetworking/cni/pkg/ip"
	"github.com/containernetworking/cni/pkg/ns"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types/current"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
)

// DoNetworking performs the networking for the given config and IPAM result
func DoNetworking(args *skel.CmdArgs, conf NetConf, result *current.Result, logger *log.Entry, desiredVethName string) (hostVethName, contVethMAC string, err error) {
	// Select the first 11 characters of the containerID for the host veth.
	hostVethName = "cali" + args.ContainerID[:Min(11, len(args.ContainerID))]
	contVethName := args.IfName
	var hasIPv4, hasIPv6 bool

	// If a desired veth name was passed in, use that instead.
	if desiredVethName != "" {
		hostVethName = desiredVethName
	}

	// Clean up if hostVeth exists.
	if oldHostVeth, err := netlink.LinkByName(hostVethName); err == nil {
		if err = netlink.LinkDel(oldHostVeth); err != nil {
			return "", "", fmt.Errorf("failed to delete old hostVeth %v: %v", hostVethName, err)
		}
		logger.Infof("clean old hostVeth: %v", hostVethName)
	}

	err = ns.WithNetNSPath(args.Netns, func(hostNS ns.NetNS) error {
		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:  contVethName,
				Flags: net.FlagUp,
				MTU:   conf.MTU,
			},
			PeerName: hostVethName,
		}

		if err := netlink.LinkAdd(veth); err != nil {
			logger.Errorf("Error adding veth %+v: %s", veth, err)
			return err
		}

		hostVeth, err := netlink.LinkByName(hostVethName)
		if err != nil {
			err = fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
			return err
		}

		// Explicitly set the veth to UP state, because netlink doesn't always do that on all the platforms with net.FlagUp.
		// veth won't get a link local address unless it's set to UP state.
		if err = netlink.LinkSetUp(hostVeth); err != nil {
			return fmt.Errorf("failed to set %q up: %v", hostVethName, err)
		}

		contVeth, err := netlink.LinkByName(contVethName)
		if err != nil {
			err = fmt.Errorf("failed to lookup %q: %v", contVethName, err)
			return err
		}

		// Fetch the MAC from the container Veth. This is needed by Calico.
		contVethMAC = contVeth.Attrs().HardwareAddr.String()
		logger.WithField("MAC", contVethMAC).Debug("Found MAC for container veth")

		// At this point, the virtual ethernet pair has been created, and both ends have the right names.
		// Both ends of the veth are still in the container's network namespace.

		for _, addr := range result.IPs {

			// Before returning, create the routes inside the namespace, first for IPv4 then IPv6.
			if addr.Version == "4" {
				// Add a connected route to a dummy next hop so that a default route can be set
				gw := net.IPv4(169, 254, 1, 1)
				gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)}
				err := netlink.RouteAdd(
					&netlink.Route{
						LinkIndex: contVeth.Attrs().Index,
						Scope:     netlink.SCOPE_LINK,
						Dst:       gwNet,
					},
				)

				if err != nil {
					return fmt.Errorf("failed to add route inside the container: %v", err)
				}

				if err = ip.AddDefaultRoute(gw, contVeth); err != nil {
					return fmt.Errorf("failed to add the default route inside the container: %v", err)
				}

				if err = netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address}); err != nil {
					return fmt.Errorf("failed to add IP addr to %q: %v", contVethName, err)
				}
				// Set hasIPv4 to true so sysctls for IPv4 can be programmed when the host side of
				// the veth finishes moving to the host namespace.
				hasIPv4 = true
			}

			// Handle IPv6 routes
			if addr.Version == "6" {
				// No need to add a dummy next hop route as the host veth device will already have an IPv6
				// link local address that can be used as a next hop.
				// Just fetch the address of the host end of the veth and use it as the next hop.
				addresses, err := netlink.AddrList(hostVeth, netlink.FAMILY_V6)
				if err != nil {
					logger.Errorf("Error listing IPv6 addresses: %s", err)
					return err
				}

				if len(addresses) < 1 {
					// If the hostVeth doesn't have an IPv6 address then this host probably doesn't
					// support IPv6. Since a IPv6 address has been allocated that can't be used,
					// return an error.
					return fmt.Errorf("failed to get IPv6 addresses for host side of the veth pair")
				}

				hostIPv6Addr := addresses[0].IP

				_, defNet, _ := net.ParseCIDR("::/0")
				if err = ip.AddRoute(defNet, hostIPv6Addr, contVeth); err != nil {
					return fmt.Errorf("failed to add default gateway to %v %v", hostIPv6Addr, err)
				}

				if err = netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address}); err != nil {
					return fmt.Errorf("failed to add IP addr to %q: %v", contVeth, err)
				}

				// Set hasIPv6 to true so sysctls for IPv6 can be programmed when the host side of
				// the veth finishes moving to the host namespace.
				hasIPv6 = true
			}
		}

		// Now that the everything has been successfully set up in the container, move the "host" end of the
		// veth into the host namespace.
		if err = netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
			return fmt.Errorf("failed to move veth to host netns: %v", err)
		}

		err = configureContainerSysctls(hasIPv4, hasIPv6)
		if err != nil {
			return fmt.Errorf("error configuring sysctls for the container netns, error: %s", err)
		}

		return nil
	})

	if err != nil {
		logger.Errorf("Error creating veth: %s", err)
		return "", "", err
	}

	err = configureSysctls(hostVethName, hasIPv4, hasIPv6)
	if err != nil {
		return "", "", fmt.Errorf("error configuring sysctls for interface: %s, error: %s", hostVethName, err)
	}

	// Moving a veth between namespaces always leaves it in the "DOWN" state. Set it back to "UP" now that we're
	// back in the host namespace.
	hostVeth, err := netlink.LinkByName(hostVethName)
	if err != nil {
		return "", "", fmt.Errorf("failed to lookup %q: %v", hostVethName, err)
	}

	if err = netlink.LinkSetUp(hostVeth); err != nil {
		return "", "", fmt.Errorf("failed to set %q up: %v", hostVethName, err)
	}

	// Now that the host side of the veth is moved, state set to UP, and configured with sysctls, we can add the routes to it in the host namespace.
	err = setupRoutes(hostVeth, result)
	if err != nil {
		return "", "", fmt.Errorf("error adding host side routes for interface: %s, error: %s", hostVeth.Attrs().Name, err)
	}

	return hostVethName, contVethMAC, err
}

var errFileExists = fmt.Errorf("file exists")

// setupRoutes sets up the routes for the host side of the veth pair.
func setupRoutes(hostVeth netlink.Link, result *current.Result) error {

	// Go through all the IPs and add routes for each IP in the result.
	for _, ipAddr := range result.IPs {
		route := netlink.Route{
			LinkIndex: hostVeth.Attrs().Index,
			Scope:     netlink.SCOPE_LINK,
			Dst:       &ipAddr.Address,
		}
		err := netlink.RouteAdd(&route)

		if err != nil {
			switch err {

			// Route already exists, but not necessarily pointing to the same interface.
			case errFileExists:
				// List all the routes for the interface.
				routes, err := netlink.RouteList(hostVeth, netlink.FAMILY_ALL)
				if err != nil {
					return fmt.Errorf("error listing routes")
				}

				// Go through all the routes pointing to the interface, and see if any of them is
				// exactly what we are intending to program.
				// If the route we want is already there then most likely it's programmed by Felix, so we ignore it,
				// and we return an error if none of the routes match the route we're trying to program.
				for _, r := range routes {
					if reflect.DeepEqual(r, route) {
						// Route was already present on the host.
						log.Infof("CNI skipping add route. Route already exists for %s\n", hostVeth.Attrs().Name)
						return nil
					}
				}
				return fmt.Errorf("route (Dst: %s, Scope: %s) already exists for an interface other than '%s'", route.Dst.String(), route.Scope, hostVeth.Attrs().Name)
			default:
				return fmt.Errorf("failed to add route (Dst: %s, Scope: %s, Iface: %s): %v", route.Dst.String(), route.Scope, hostVeth.Attrs().Name, err)
			}
		}

		log.Debugf("CNI adding route for interface: %v, IP: %s", hostVeth, ipAddr.Address)
	}
	return nil
}

// configureSysctls configures necessary sysctls required for the host side of the veth pair for IPv4 and/or IPv6.
func configureSysctls(hostVethName string, hasIPv4, hasIPv6 bool) error {
	var err error

	if hasIPv4 {
		// Enable proxy ARP, this makes the host respond to all ARP requests with its own
		// MAC. We install explicit routes into the containers network
		// namespace and we use a link-local address for the gateway.  Turing on proxy ARP
		// means that we don't need to assign the link local address explicitly to each
		// host side of the veth, which is one fewer thing to maintain and one fewer
		// thing we may clash over.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/proxy_arp", hostVethName), "1"); err != nil {
			return err
		}

		// Normally, the kernel has a delay before responding to proxy ARP but we know
		// that's not needed in a Calico network so we disable it.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/neigh/%s/proxy_delay", hostVethName), "0"); err != nil {
			return err
		}

		// Enable IP forwarding of packets coming _from_ this interface.  For packets to
		// be forwarded in both directions we need this flag to be set on the fabric-facing
		// interface too (or for the global default to be set).
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/forwarding", hostVethName), "1"); err != nil {
			return err
		}
	}

	if hasIPv6 {
		// Enable proxy NDP, similarly to proxy ARP, described above in IPv4 section.
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/proxy_ndp", hostVethName), "1"); err != nil {
			return err
		}

		// Enable IP forwarding of packets coming _from_ this interface.  For packets to
		// be forwarded in both directions we need this flag to be set on the fabric-facing
		// interface too (or for the global default to be set).
		if err = writeProcSys(fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/forwarding", hostVethName), "1"); err != nil {
			return err
		}
	}

	return nil
}

// configureContainerSysctls configures necessary sysctls required inside the container netns.
func configureContainerSysctls(hasIPv4, hasIPv6 bool) error {
	var err error

	// Globally disable IP forwarding of packets inside the container netns.
	// Generally, we don't expect containers to be routing anything.

	if hasIPv4 {
		if err = writeProcSys("/proc/sys/net/ipv4/ip_forward", "0"); err != nil {
			return err
		}
	}

	if hasIPv6 {
		if err = writeProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "0"); err != nil {
			return err
		}
	}

	return nil
}

// writeProcSys takes the sysctl path and a string value to set i.e. "0" or "1" and sets the sysctl.
func writeProcSys(path, value string) error {
	f, err := os.OpenFile(path, os.O_WRONLY, 0)
	if err != nil {
		return err
	}
	n, err := f.Write([]byte(value))
	if err == nil && n < len(value) {
		err = io.ErrShortWrite
	}
	if err1 := f.Close(); err == nil {
		err = err1
	}
	return err
}
<|endoftext|>


<commit_before>package hfile

import (
	"bytes"
	"fmt"
)

type OrderedOps struct {
	LastKey *[]byte
}

func (s *OrderedOps) ResetState() {
	s.LastKey = nil
}

func (s *OrderedOps) CheckIfKeyOutOfOrder(key []byte) error {
	if s.LastKey != nil && bytes.Compare(*s.LastKey, key) > 0 {
		return fmt.Errorf("Keys out of order! %v > %v", *s.LastKey, key)
	}
	s.LastKey = &key
	return nil
}
<commit_msg>add Same method to ordered keys helper<commit_after>package hfile

import (
	"bytes"
	"fmt"
)

type OrderedOps struct {
	LastKey *[]byte
}

func (s *OrderedOps) ResetState() {
	s.LastKey = nil
}

func (s *OrderedOps) Same(key []byte) bool {
	return s.LastKey != nil && bytes.Equal(*s.LastKey, key)
}

func (s *OrderedOps) CheckIfKeyOutOfOrder(key []byte) error {
	if s.LastKey != nil && bytes.Compare(*s.LastKey, key) > 0 {
		return fmt.Errorf("Keys out of order! %v > %v", *s.LastKey, key)
	}
	s.LastKey = &key
	return nil
}
<|endoftext|>