<commit_before>package mp3

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"time"

	"github.com/mattetti/audio/mp3/id3v1"
	"github.com/mattetti/audio/mp3/id3v2"
)

// Decoder operates on a reader and extracts important information
// See http://www.mp3-converter.com/mp3codec/mp3_anatomy.htm
type Decoder struct {
	r         io.Reader
	NbrFrames int

	ID3v2tag *id3v2.Tag
}

// NewDecoder creates a new reader reading the given reader and parsing its data.
// It is the caller's responsibility to call Close on the reader when done.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// SeemsValid checks if the mp3 file looks like a valid mp3 file by looking at the first few bytes.
// The data can be corrupt but at least the header seems alright.
// It is the caller's responsibility to rewind/close the reader when done.
func SeemsValid(r io.Reader) bool {
	d := New(r)
	fr := &Frame{}
	var frameDuration time.Duration
	var duration time.Duration
	var err error
	var badFrames int
	for {
		err = d.Next(fr)
		if err != nil {
			badFrames++
			if err == ErrInvalidHeader {
				continue
			}
			break
		}
		// garbage needing to be skipped probably means bad frame
		if fr.SkippedBytes > 20 {
			badFrames++
		}
		frameDuration = fr.Duration()

		if frameDuration > 0 {
			duration += frameDuration
		}
		d.NbrFrames++
	}
	if err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrShortBuffer {
		err = nil
	}
	if d.NbrFrames <= 0 {
		return false
	}
	percentBadFrames := (float64(badFrames) * 100) / float64(d.NbrFrames)
	// more than 10% frames with issues or a zero/negative duration means bad file
	if percentBadFrames > 10 {
		return false
	}
	if duration <= 0 {
		return false
	}
	return true
}

// Duration returns the time duration for the current mp3 file
// The entire reader will be consumed, the consumer might want to rewind the reader
// if they want to read more from the feed.
func (d *Decoder) Duration() (time.Duration, error) {
	if d == nil {
		return 0, errors.New("can't calculate the duration of a nil pointer")
	}
	fr := &Frame{}
	var frameDuration time.Duration
	var duration time.Duration
	var err error
	for {
		err = d.Next(fr)
		if err != nil {
			// bad headers can be ignored and hopefully skipped
			if err == ErrInvalidHeader {
				continue
			}
			break
		}
		frameDuration = fr.Duration()
		if frameDuration > 0 {
			duration += frameDuration
		}
		d.NbrFrames++
	}
	if err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrShortBuffer {
		err = nil
	}

	return duration, err
}

// Next decodes the next frame into the provided frame structure.
func (d *Decoder) Next(f *Frame) error {
	if f == nil {
		return fmt.Errorf("can't decode to a nil Frame")
	}

	var n int
	f.SkippedBytes = 0
	f.Counter++

	hLen := 4
	if f.buf == nil {
		f.buf = make([]byte, hLen)
	} else {
		f.buf = f.buf[:hLen]
	}

	_, err := io.ReadAtLeast(d.r, f.buf, hLen)
	if err != nil {
		return err
	}

	// ID3v1 tag at the beggining
	if bytes.Compare(f.buf[:3], id3v1.HeaderTagID) == 0 {
		// the ID3v1 tag is always 128 bytes long, we already read 4 bytes
		// so we need to read the rest.
		buf := make([]byte, 124)
		// TODO: parse the actual header
		if _, err := io.ReadAtLeast(d.r, buf, 124); err != nil {
			return ErrInvalidHeader
		}
		buf = append(f.buf, buf...)
		// that wasn't a frame
		f = &Frame{}
		return nil
	}

	// ID3v2 tag
	if bytes.Compare(f.buf[:3], id3v2.HeaderTagID) == 0 {
		d.ID3v2tag = &id3v2.Tag{}
		// we already read 4 bytes, an id3v2 tag header is of size 10, read the rest
		// and append it to what we already have.
		buf := make([]byte, 6)
		n, err := d.r.Read(buf)
		if err != nil || n != 6 {
			return ErrInvalidHeader
		}
		buf = append(f.buf, buf...)

		th := id3v2.TagHeader{}
		copy(th[:], buf)
		if err = d.ID3v2tag.ReadHeader(th); err != nil {
			return err
		}
		// TODO: parse the actual tag
		// Skip the tag for now
		bytesToSkip := int64(d.ID3v2tag.Header.Size)
		var cn int64
		if cn, err = io.CopyN(ioutil.Discard, d.r, bytesToSkip); cn != bytesToSkip {
			return ErrInvalidHeader
		}
		f = &Frame{}
		return err
	}

	f.Header = FrameHeader(f.buf)
	if !f.Header.IsValid() {
		f.Header, n, err = d.skipToNextFrame()
		if err != nil {
			return err
		}
		f.SkippedBytes = n
	}

	dataSize := f.Header.Size()
	if dataSize > 4 {
		// substract the 4 bytes we already read
		dataSize -= 4
		f.buf = append(f.buf, make([]byte, dataSize)...)
		_, err = io.ReadAtLeast(d.r, f.buf[4:], int(dataSize))
	}
	return err
}

// skipToSyncWord reads until it finds a frame header
func (d *Decoder) skipToNextFrame() (fh FrameHeader, readN int, err error) {
	if d == nil {
		return nil, readN, errors.New("nil decoder")
	}
	buf := make([]byte, 1)
	lookAheadBuf := make([]byte, 1)
	var n int
	for {
		n, err = d.r.Read(buf)
		readN += n
		if err != nil {
			return nil, readN, err
		}
		readN++
		if buf[0] == 0xFF {
			if _, err := d.r.Read(lookAheadBuf); err != nil {
				return nil, readN, err
			}
			readN++
			if lookAheadBuf[0]&0xE0 == 0xE0 {
				buf = []byte{0xff, lookAheadBuf[0], 0, 0}
				n, err := d.r.Read(buf[2:])
				if err != nil {
					return nil, readN + n, err
				}
				if n != 2 {
					return nil, readN + n, io.ErrUnexpectedEOF
				}
				readN += 2
			}
			return buf, readN, err
		}
	}
}
<commit_msg>mp3: doc update<commit_after>package mp3

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"time"

	"github.com/mattetti/audio/mp3/id3v1"
	"github.com/mattetti/audio/mp3/id3v2"
)

// Decoder operates on a reader and extracts important information
// See http://www.mp3-converter.com/mp3codec/mp3_anatomy.htm
type Decoder struct {
	r         io.Reader
	NbrFrames int

	ID3v2tag *id3v2.Tag
}

// NewDecoder creates a new reader reading the given reader and parsing its data.
// It is the caller's responsibility to call Close on the reader when done.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// SeemsValid checks if the mp3 file looks like a valid mp3 file by looking at the first few bytes.
// The data can be corrupt but at least the header seems alright.
// It is the caller's responsibility to rewind/close the reader when done.
func SeemsValid(r io.Reader) bool {
	d := New(r)
	fr := &Frame{}
	var frameDuration time.Duration
	var duration time.Duration
	var err error
	var badFrames int
	for {
		err = d.Next(fr)
		if err != nil {
			badFrames++
			if err == ErrInvalidHeader {
				continue
			}
			break
		}
		// garbage needing to be skipped probably means bad frame
		if fr.SkippedBytes > 20 {
			badFrames++
		}
		frameDuration = fr.Duration()

		if frameDuration > 0 {
			duration += frameDuration
		}
		d.NbrFrames++
	}
	if err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrShortBuffer {
		err = nil
	}
	if d.NbrFrames <= 0 {
		return false
	}
	percentBadFrames := (float64(badFrames) * 100) / float64(d.NbrFrames)
	// more than 10% frames with issues or a zero/negative duration means bad file
	if percentBadFrames > 10 {
		return false
	}
	if duration <= 0 {
		return false
	}
	return true
}

// Duration returns the time duration for the current mp3 file
// The entire reader will be consumed, the consumer might want to rewind the reader
// if they want to read more from the feed.
// Note that this is an estimated duration based on how the frames look. An invalid file might have
// a duration.
func (d *Decoder) Duration() (time.Duration, error) {
	if d == nil {
		return 0, errors.New("can't calculate the duration of a nil pointer")
	}
	fr := &Frame{}
	var frameDuration time.Duration
	var duration time.Duration
	var err error
	for {
		err = d.Next(fr)
		if err != nil {
			// bad headers can be ignored and hopefully skipped
			if err == ErrInvalidHeader {
				continue
			}
			break
		}
		frameDuration = fr.Duration()
		if frameDuration > 0 {
			duration += frameDuration
		}
		d.NbrFrames++
	}
	if err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrShortBuffer {
		err = nil
	}

	return duration, err
}

// Next decodes the next frame into the provided frame structure.
func (d *Decoder) Next(f *Frame) error {
	if f == nil {
		return fmt.Errorf("can't decode to a nil Frame")
	}

	var n int
	f.SkippedBytes = 0
	f.Counter++

	hLen := 4
	if f.buf == nil {
		f.buf = make([]byte, hLen)
	} else {
		f.buf = f.buf[:hLen]
	}

	_, err := io.ReadAtLeast(d.r, f.buf, hLen)
	if err != nil {
		return err
	}

	// ID3v1 tag at the beggining
	if bytes.Compare(f.buf[:3], id3v1.HeaderTagID) == 0 {
		// the ID3v1 tag is always 128 bytes long, we already read 4 bytes
		// so we need to read the rest.
		buf := make([]byte, 124)
		// TODO: parse the actual header
		if _, err := io.ReadAtLeast(d.r, buf, 124); err != nil {
			return ErrInvalidHeader
		}
		buf = append(f.buf, buf...)
		// that wasn't a frame
		f = &Frame{}
		return nil
	}

	// ID3v2 tag
	if bytes.Compare(f.buf[:3], id3v2.HeaderTagID) == 0 {
		d.ID3v2tag = &id3v2.Tag{}
		// we already read 4 bytes, an id3v2 tag header is of size 10, read the rest
		// and append it to what we already have.
		buf := make([]byte, 6)
		n, err := d.r.Read(buf)
		if err != nil || n != 6 {
			return ErrInvalidHeader
		}
		buf = append(f.buf, buf...)

		th := id3v2.TagHeader{}
		copy(th[:], buf)
		if err = d.ID3v2tag.ReadHeader(th); err != nil {
			return err
		}
		// TODO: parse the actual tag
		// Skip the tag for now
		bytesToSkip := int64(d.ID3v2tag.Header.Size)
		var cn int64
		if cn, err = io.CopyN(ioutil.Discard, d.r, bytesToSkip); cn != bytesToSkip {
			return ErrInvalidHeader
		}
		f = &Frame{}
		return err
	}

	f.Header = FrameHeader(f.buf)
	if !f.Header.IsValid() {
		f.Header, n, err = d.skipToNextFrame()
		if err != nil {
			return err
		}
		f.SkippedBytes = n
	}

	dataSize := f.Header.Size()
	if dataSize > 4 {
		// substract the 4 bytes we already read
		dataSize -= 4
		f.buf = append(f.buf, make([]byte, dataSize)...)
		_, err = io.ReadAtLeast(d.r, f.buf[4:], int(dataSize))
	}
	return err
}

// skipToSyncWord reads until it finds a frame header
func (d *Decoder) skipToNextFrame() (fh FrameHeader, readN int, err error) {
	if d == nil {
		return nil, readN, errors.New("nil decoder")
	}
	buf := make([]byte, 1)
	lookAheadBuf := make([]byte, 1)
	var n int
	for {
		n, err = d.r.Read(buf)
		readN += n
		if err != nil {
			return nil, readN, err
		}
		readN++
		if buf[0] == 0xFF {
			if _, err := d.r.Read(lookAheadBuf); err != nil {
				return nil, readN, err
			}
			readN++
			if lookAheadBuf[0]&0xE0 == 0xE0 {
				buf = []byte{0xff, lookAheadBuf[0], 0, 0}
				n, err := d.r.Read(buf[2:])
				if err != nil {
					return nil, readN + n, err
				}
				if n != 2 {
					return nil, readN + n, io.ErrUnexpectedEOF
				}
				readN += 2
			}
			return buf, readN, err
		}
	}
}
<|endoftext|>


<commit_before>package anidb

import (
	"github.com/Kovensky/go-anidb/udp"
	"github.com/Kovensky/go-fscache"
	"strconv"
	"strings"
	"time"
)

func (e *MyListEntry) setCachedTS(ts time.Time) {
	e.Cached = ts
}

func (e *MyListEntry) IsStale() bool {
	if e == nil {
		return true
	}

	max := MyListCacheDuration
	if !e.DateWatched.IsZero() {
		max = MyListWatchedCacheDuration
	}
	return time.Now().Sub(e.Cached) > max
}

var _ cacheable = &MyListEntry{}

func (uid UID) MyList(fid FID) *MyListEntry {
	if f := fid.File(); f == nil {
		return nil
	} else if lid := f.LID[uid]; lid < 1 {
		return nil
	} else {
		return f.LID[uid].MyListEntry()
	}
}

func (lid LID) MyListEntry() *MyListEntry {
	var e MyListEntry
	if CacheGet(&e, "lid", lid) == nil {
		return &e
	}
	return nil
}

func (adb *AniDB) MyListByFile(f *File) <-chan *MyListEntry {
	ch := make(chan *MyListEntry, 1)

	if f == nil {
		ch <- nil
		close(ch)
		return ch
	}

	go func() {
		user := <-adb.GetCurrentUser()

		var entry *MyListEntry

		if lid := f.LID[user.UID]; lid != 0 {
			entry = <-adb.MyListByLID(lid)
		}
		if entry == nil {
			entry = <-adb.MyListByFID(f.FID)
		}
		ch <- entry
		close(ch)
	}()
	return ch
}

func (adb *AniDB) MyListByLID(lid LID) <-chan *MyListEntry {
	key := []fscache.CacheKey{"mylist", lid}
	ch := make(chan *MyListEntry, 1)

	if lid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	ic := make(chan notification, 1)
	go func() { ch <- (<-ic).(*MyListEntry); close(ch) }()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose((*MyListEntry)(nil), key...)
		return ch
	}

	entry := lid.MyListEntry()
	if !entry.IsStale() {
		intentMap.NotifyClose(entry, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("MYLIST", paramMap{"lid": lid})

		switch reply.Code() {
		case 221:
			entry = adb.parseMylistReply(reply) // caches
		case 312:
			panic("Multiple MYLIST entries when querying for single LID")
		case 321:
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(entry, key...)
	}()
	return ch
}

func (adb *AniDB) MyListByFID(fid FID) <-chan *MyListEntry {
	ch := make(chan *MyListEntry, 1)

	if fid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	// This is an odd one: we lack enough data at first to create the cache key
	go func() {
		user := <-adb.GetCurrentUser()
		if user == nil || user.UID < 1 {
			ch <- nil
			close(ch)
			return
		}

		key := []fscache.CacheKey{"mylist", "by-fid", fid, user.UID}

		ic := make(chan notification, 1)
		go func() { ch <- (<-ic).(*MyListEntry); close(ch) }()
		if intentMap.Intent(ic, key...) {
			return
		}

		if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
			intentMap.NotifyClose((*MyListEntry)(nil), key...)
			return
		}

		lid := LID(0)
		switch ts, err := Cache.Get(&lid, key...); {
		case err == nil && time.Now().Sub(ts) < LIDCacheDuration:
			intentMap.NotifyClose(<-adb.MyListByLID(lid), key...)
			return
		}

		reply := <-adb.udp.SendRecv("MYLIST", paramMap{"fid": fid})

		var entry *MyListEntry

		switch reply.Code() {
		case 221:
			entry = adb.parseMylistReply(reply) // caches
		case 312:
			panic("Multiple MYLIST entries when querying for single FID")
		case 321:
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(entry, key...)
	}()
	return ch
}

func (adb *AniDB) parseMylistReply(reply udpapi.APIReply) *MyListEntry {
	// 221: MYLIST ok, 310: MYLISTADD conflict (same return format as 221)
	if reply.Code() != 221 && reply.Code() != 310 {
		return nil
	}

	parts := strings.Split(reply.Lines()[1], "|")
	ints := make([]int64, len(parts))
	for i := range parts {
		ints[i], _ = strconv.ParseInt(parts[i], 10, 64)
	}

	da := time.Unix(ints[5], 0)
	if ints[5] == 0 {
		da = time.Time{}
	}
	dw := time.Unix(ints[7], 0)
	if ints[7] == 0 {
		dw = time.Time{}
	}

	e := &MyListEntry{
		LID: LID(ints[0]),

		FID: FID(ints[1]),
		EID: EID(ints[2]),
		AID: AID(ints[3]),
		GID: GID(ints[4]),

		DateAdded:   da,
		DateWatched: dw,

		State:       FileState(ints[11]),
		MyListState: MyListState(ints[6]),

		Storage: parts[8],
		Source:  parts[9],
		Other:   parts[10],
	}

	user := <-adb.GetCurrentUser()

	if user != nil {
		if f := e.FID.File(); f != nil {
			f.LID[user.UID] = e.LID
			Cache.Set(f, "fid", f.FID)
			Cache.Chtime(f.Cached, "fid", f.FID)

			now := time.Now()
			mla := <-adb.MyListAnime(f.AID)

			key := []fscache.CacheKey{"mylist-anime", user.UID, f.AID}

			intentMap.Intent(nil, key...)

			if mla == nil {
				mla = &MyListAnime{}
			}

			if mla.Cached.Before(now) {
				el := mla.EpisodesWithState[e.MyListState]
				el.Add(f.EpisodeNumber)
				mla.EpisodesWithState[e.MyListState] = el

				if e.DateWatched.IsZero() {
					mla.WatchedEpisodes.Sub(f.EpisodeNumber)
				} else {
					mla.WatchedEpisodes.Add(f.EpisodeNumber)
				}

				eg := mla.EpisodesPerGroup[f.GID]
				eg.Add(f.EpisodeNumber)
				mla.EpisodesPerGroup[f.GID] = eg

				if mla.Cached.IsZero() {
					// as attractive as such an ancient mtime would be,
					// few filesystems can represent it; just make it old enough
					mla.Cached = time.Unix(0, 0)
				}

				Cache.Set(mla, key...)
				Cache.Chtime(mla.Cached, key...)
			}

			// this unfortunately races if Intent returns true:
			// only the first NotifyClose call actually notifies
			go intentMap.NotifyClose(mla, key...)
		}

		CacheSet(e, "mylist", "by-fid", e.FID, user.UID)
	}

	CacheSet(e, "mylist", e.LID)

	return e
}
<commit_msg>anidb: Correct cache key for LID.MyListEntry<commit_after>package anidb

import (
	"github.com/Kovensky/go-anidb/udp"
	"github.com/Kovensky/go-fscache"
	"strconv"
	"strings"
	"time"
)

func (e *MyListEntry) setCachedTS(ts time.Time) {
	e.Cached = ts
}

func (e *MyListEntry) IsStale() bool {
	if e == nil {
		return true
	}

	max := MyListCacheDuration
	if !e.DateWatched.IsZero() {
		max = MyListWatchedCacheDuration
	}
	return time.Now().Sub(e.Cached) > max
}

var _ cacheable = &MyListEntry{}

func (uid UID) MyList(fid FID) *MyListEntry {
	if f := fid.File(); f == nil {
		return nil
	} else if lid := f.LID[uid]; lid < 1 {
		return nil
	} else {
		return f.LID[uid].MyListEntry()
	}
}

func (lid LID) MyListEntry() *MyListEntry {
	var e MyListEntry
	if CacheGet(&e, "mylist", lid) == nil {
		return &e
	}
	return nil
}

func (adb *AniDB) MyListByFile(f *File) <-chan *MyListEntry {
	ch := make(chan *MyListEntry, 1)

	if f == nil {
		ch <- nil
		close(ch)
		return ch
	}

	go func() {
		user := <-adb.GetCurrentUser()

		var entry *MyListEntry

		if lid := f.LID[user.UID]; lid != 0 {
			entry = <-adb.MyListByLID(lid)
		}
		if entry == nil {
			entry = <-adb.MyListByFID(f.FID)
		}
		ch <- entry
		close(ch)
	}()
	return ch
}

func (adb *AniDB) MyListByLID(lid LID) <-chan *MyListEntry {
	key := []fscache.CacheKey{"mylist", lid}
	ch := make(chan *MyListEntry, 1)

	if lid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	ic := make(chan notification, 1)
	go func() { ch <- (<-ic).(*MyListEntry); close(ch) }()
	if intentMap.Intent(ic, key...) {
		return ch
	}

	if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
		intentMap.NotifyClose((*MyListEntry)(nil), key...)
		return ch
	}

	entry := lid.MyListEntry()
	if !entry.IsStale() {
		intentMap.NotifyClose(entry, key...)
		return ch
	}

	go func() {
		reply := <-adb.udp.SendRecv("MYLIST", paramMap{"lid": lid})

		switch reply.Code() {
		case 221:
			entry = adb.parseMylistReply(reply) // caches
		case 312:
			panic("Multiple MYLIST entries when querying for single LID")
		case 321:
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(entry, key...)
	}()
	return ch
}

func (adb *AniDB) MyListByFID(fid FID) <-chan *MyListEntry {
	ch := make(chan *MyListEntry, 1)

	if fid < 1 {
		ch <- nil
		close(ch)
		return ch
	}

	// This is an odd one: we lack enough data at first to create the cache key
	go func() {
		user := <-adb.GetCurrentUser()
		if user == nil || user.UID < 1 {
			ch <- nil
			close(ch)
			return
		}

		key := []fscache.CacheKey{"mylist", "by-fid", fid, user.UID}

		ic := make(chan notification, 1)
		go func() { ch <- (<-ic).(*MyListEntry); close(ch) }()
		if intentMap.Intent(ic, key...) {
			return
		}

		if !Cache.IsValid(InvalidKeyCacheDuration, key...) {
			intentMap.NotifyClose((*MyListEntry)(nil), key...)
			return
		}

		lid := LID(0)
		switch ts, err := Cache.Get(&lid, key...); {
		case err == nil && time.Now().Sub(ts) < LIDCacheDuration:
			intentMap.NotifyClose(<-adb.MyListByLID(lid), key...)
			return
		}

		reply := <-adb.udp.SendRecv("MYLIST", paramMap{"fid": fid})

		var entry *MyListEntry

		switch reply.Code() {
		case 221:
			entry = adb.parseMylistReply(reply) // caches
		case 312:
			panic("Multiple MYLIST entries when querying for single FID")
		case 321:
			Cache.SetInvalid(key...)
		}

		intentMap.NotifyClose(entry, key...)
	}()
	return ch
}

func (adb *AniDB) parseMylistReply(reply udpapi.APIReply) *MyListEntry {
	// 221: MYLIST ok, 310: MYLISTADD conflict (same return format as 221)
	if reply.Code() != 221 && reply.Code() != 310 {
		return nil
	}

	parts := strings.Split(reply.Lines()[1], "|")
	ints := make([]int64, len(parts))
	for i := range parts {
		ints[i], _ = strconv.ParseInt(parts[i], 10, 64)
	}

	da := time.Unix(ints[5], 0)
	if ints[5] == 0 {
		da = time.Time{}
	}
	dw := time.Unix(ints[7], 0)
	if ints[7] == 0 {
		dw = time.Time{}
	}

	e := &MyListEntry{
		LID: LID(ints[0]),

		FID: FID(ints[1]),
		EID: EID(ints[2]),
		AID: AID(ints[3]),
		GID: GID(ints[4]),

		DateAdded:   da,
		DateWatched: dw,

		State:       FileState(ints[11]),
		MyListState: MyListState(ints[6]),

		Storage: parts[8],
		Source:  parts[9],
		Other:   parts[10],
	}

	user := <-adb.GetCurrentUser()

	if user != nil {
		if f := e.FID.File(); f != nil {
			f.LID[user.UID] = e.LID
			Cache.Set(f, "fid", f.FID)
			Cache.Chtime(f.Cached, "fid", f.FID)

			now := time.Now()
			mla := <-adb.MyListAnime(f.AID)

			key := []fscache.CacheKey{"mylist-anime", user.UID, f.AID}

			intentMap.Intent(nil, key...)

			if mla == nil {
				mla = &MyListAnime{}
			}

			if mla.Cached.Before(now) {
				el := mla.EpisodesWithState[e.MyListState]
				el.Add(f.EpisodeNumber)
				mla.EpisodesWithState[e.MyListState] = el

				if e.DateWatched.IsZero() {
					mla.WatchedEpisodes.Sub(f.EpisodeNumber)
				} else {
					mla.WatchedEpisodes.Add(f.EpisodeNumber)
				}

				eg := mla.EpisodesPerGroup[f.GID]
				eg.Add(f.EpisodeNumber)
				mla.EpisodesPerGroup[f.GID] = eg

				if mla.Cached.IsZero() {
					// as attractive as such an ancient mtime would be,
					// few filesystems can represent it; just make it old enough
					mla.Cached = time.Unix(0, 0)
				}

				Cache.Set(mla, key...)
				Cache.Chtime(mla.Cached, key...)
			}

			// this unfortunately races if Intent returns true:
			// only the first NotifyClose call actually notifies
			go intentMap.NotifyClose(mla, key...)
		}

		CacheSet(e, "mylist", "by-fid", e.FID, user.UID)
	}

	CacheSet(e, "mylist", e.LID)

	return e
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/VividCortex/siesta"

	"github.com/Cistern/catena"
	"github.com/Cistern/cistern/source"
	"github.com/Cistern/cistern/state/series"
)

const (
	responseKey = "response"
	errorKey    = "error"
)

type API struct {
	addr           string
	sourceRegistry *source.Registry
	seriesEngine   *series.Engine
}

type APIResponse struct {
	Data  interface{} `json:"data,omitempty"`
	Error string      `json:"error,omitempty"`
}

func NewAPI(addr string, seriesEngine *series.Engine) *API {
	return &API{
		addr:         addr,
		seriesEngine: seriesEngine,
	}
}

func (api *API) Run() {
	service := siesta.NewService("/")

	service.AddPre(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Headers", r.Header.Get("Access-Control-Request-Headers"))
	})

	service.AddPost(func(c siesta.Context, w http.ResponseWriter, r *http.Request, q func()) {
		resp := c.Get(responseKey)
		err, _ := c.Get(errorKey).(string)

		if resp == nil && err == "" {
			return
		}

		enc := json.NewEncoder(w)
		enc.Encode(APIResponse{
			Data:  resp,
			Error: err,
		})
	})

	service.Route("GET", "/sources", "Lists sources", func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
		var params siesta.Params
		start := params.Int64("start", -3600, "Start timestamp")
		end := params.Int64("start", 0, "End timestamp")
		err := params.Parse(r.Form)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		now := time.Now().Unix()
		if *start < 0 {
			*start += now
		}
		if *end <= 0 {
			*end += now
		}
		sources := api.seriesEngine.Sources(*start, *end)
		c.Set(responseKey, sources)
	})

	service.Route("GET", "/sources/:source/metrics",
		"Lists metrics for a source",
		func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
			var params siesta.Params
			source := params.String("source", "", "Source name")
			start := params.Int64("start", -3600, "Start timestamp")
			end := params.Int64("start", 0, "End timestamp")
			err := params.Parse(r.Form)
			if err != nil {
				c.Set(errorKey, err.Error())
				return
			}

			now := time.Now().Unix()
			if *start < 0 {
				*start += now
			}
			if *end <= 0 {
				*end += now
			}

			metrics := api.seriesEngine.DB.Metrics(*source, *start, *end)
			c.Set(responseKey, metrics)
		})

	service.Route("OPTIONS", "/series/query",
		"Accepts an OPTIONS request",
		func(w http.ResponseWriter, r *http.Request) {
			// Doesn't do anything
		})

	service.Route("POST", "/series/query",
		"Lists metrics for a source",
		api.querySeriesRoute())

	http.ListenAndServe(api.addr, service)
}

// A querySeries is an ordered set of points
// for a source and metric over a range
// of time.
type querySeries struct {
	// First timestamp
	Start int64 `json:"start"`

	// Last timestamp
	End int64 `json:"end"`

	Source string `json:"source"`
	Metric string `json:"metric"`

	Points []catena.Point `json:"points"`
}

// A queryDesc is a description of a
// query. It specifies a source, metric,
// start, and end timestamps.
type queryDesc struct {
	Source string `json:"source"`
	Metric string `json:"metric"`
	Start  int64  `json:"start"`
	End    int64  `json:"end"`
}

// A queryResponse is returned after querying
// the DB with a QueryDesc.
type queryResponse struct {
	Series []querySeries `json:"series"`
}

func (api *API) querySeriesRoute() func(siesta.Context, http.ResponseWriter, *http.Request) {
	return func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
		var params siesta.Params
		pointWidth := params.Int64("pointWidth", 1, "Number of points to average together")
		err := params.Parse(r.Form)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		var descs []queryDesc

		dec := json.NewDecoder(r.Body)
		err = dec.Decode(&descs)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		now := time.Now().Unix()
		for i, desc := range descs {
			if desc.Start <= 0 {
				desc.Start += now
			}

			if desc.End <= 0 {
				desc.End += now
			}

			descs[i] = desc
		}

		resp := queryResponse{}

		for _, desc := range descs {
			log.Println(desc)
			i, err := api.seriesEngine.DB.NewIterator(desc.Source, desc.Metric)
			if err != nil {
				log.Println(err)
				continue
			}

			err = i.Seek(desc.Start)
			if err != nil {
				log.Println(err)
				continue
			}

			s := querySeries{
				Source: desc.Source,
				Metric: desc.Metric,
				Start:  i.Point().Timestamp,
				End:    i.Point().Timestamp,
			}

			pointsSeen := 0

			currentInterval := i.Point().Timestamp / *pointWidth
			currentPoint := catena.Point{
				Timestamp: currentInterval * *pointWidth,
			}

			for {
				p := i.Point()
				if p.Timestamp > desc.End {
					break
				}

				if p.Timestamp / *pointWidth != currentInterval {
					currentPoint.Value /= float64(pointsSeen)
					s.Points = append(s.Points, currentPoint)
					currentInterval = i.Point().Timestamp / *pointWidth
					currentPoint = catena.Point{
						Timestamp: currentInterval * *pointWidth,
						Value:     p.Value,
					}
					pointsSeen = 1
					continue
				}

				currentPoint.Value += p.Value
				pointsSeen++

				err := i.Next()
				if err != nil {
					log.Println(err)
					break
				}
			}

			if pointsSeen > 0 {
				currentPoint.Value /= float64(pointsSeen)
				s.Points = append(s.Points, currentPoint)
			}
			i.Close()

			resp.Series = append(resp.Series, s)
		}

		c.Set(responseKey, resp)
	}
}
<commit_msg>fix params<commit_after>package api

import (
	"encoding/json"
	"log"
	"net/http"
	"time"

	"github.com/VividCortex/siesta"

	"github.com/Cistern/catena"
	"github.com/Cistern/cistern/source"
	"github.com/Cistern/cistern/state/series"
)

const (
	responseKey = "response"
	errorKey    = "error"
)

type API struct {
	addr           string
	sourceRegistry *source.Registry
	seriesEngine   *series.Engine
}

type APIResponse struct {
	Data  interface{} `json:"data,omitempty"`
	Error string      `json:"error,omitempty"`
}

func NewAPI(addr string, seriesEngine *series.Engine) *API {
	return &API{
		addr:         addr,
		seriesEngine: seriesEngine,
	}
}

func (api *API) Run() {
	service := siesta.NewService("/")

	service.AddPre(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Access-Control-Allow-Headers", r.Header.Get("Access-Control-Request-Headers"))
	})

	service.AddPost(func(c siesta.Context, w http.ResponseWriter, r *http.Request, q func()) {
		resp := c.Get(responseKey)
		err, _ := c.Get(errorKey).(string)

		if resp == nil && err == "" {
			return
		}

		enc := json.NewEncoder(w)
		enc.Encode(APIResponse{
			Data:  resp,
			Error: err,
		})
	})

	service.Route("GET", "/sources", "Lists sources", func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
		var params siesta.Params
		start := params.Int64("start", -3600, "Start timestamp")
		end := params.Int64("end", 0, "End timestamp")
		err := params.Parse(r.Form)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		now := time.Now().Unix()
		if *start < 0 {
			*start += now
		}
		if *end <= 0 {
			*end += now
		}
		sources := api.seriesEngine.Sources(*start, *end)
		c.Set(responseKey, sources)
	})

	service.Route("GET", "/sources/:source/metrics",
		"Lists metrics for a source",
		func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
			var params siesta.Params
			source := params.String("source", "", "Source name")
			start := params.Int64("start", -3600, "Start timestamp")
			end := params.Int64("end", 0, "End timestamp")
			err := params.Parse(r.Form)
			if err != nil {
				c.Set(errorKey, err.Error())
				return
			}

			now := time.Now().Unix()
			if *start < 0 {
				*start += now
			}
			if *end <= 0 {
				*end += now
			}

			metrics := api.seriesEngine.DB.Metrics(*source, *start, *end)
			c.Set(responseKey, metrics)
		})

	service.Route("OPTIONS", "/series/query",
		"Accepts an OPTIONS request",
		func(w http.ResponseWriter, r *http.Request) {
			// Doesn't do anything
		})

	service.Route("POST", "/series/query",
		"Lists metrics for a source",
		api.querySeriesRoute())

	http.ListenAndServe(api.addr, service)
}

// A querySeries is an ordered set of points
// for a source and metric over a range
// of time.
type querySeries struct {
	// First timestamp
	Start int64 `json:"start"`

	// Last timestamp
	End int64 `json:"end"`

	Source string `json:"source"`
	Metric string `json:"metric"`

	Points []catena.Point `json:"points"`
}

// A queryDesc is a description of a
// query. It specifies a source, metric,
// start, and end timestamps.
type queryDesc struct {
	Source string `json:"source"`
	Metric string `json:"metric"`
	Start  int64  `json:"start"`
	End    int64  `json:"end"`
}

// A queryResponse is returned after querying
// the DB with a QueryDesc.
type queryResponse struct {
	Series []querySeries `json:"series"`
}

func (api *API) querySeriesRoute() func(siesta.Context, http.ResponseWriter, *http.Request) {
	return func(c siesta.Context, w http.ResponseWriter, r *http.Request) {
		var params siesta.Params
		pointWidth := params.Int64("pointWidth", 1, "Number of points to average together")
		err := params.Parse(r.Form)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		var descs []queryDesc

		dec := json.NewDecoder(r.Body)
		err = dec.Decode(&descs)
		if err != nil {
			c.Set(errorKey, err.Error())
			return
		}

		now := time.Now().Unix()
		for i, desc := range descs {
			if desc.Start <= 0 {
				desc.Start += now
			}

			if desc.End <= 0 {
				desc.End += now
			}

			descs[i] = desc
		}

		resp := queryResponse{}

		for _, desc := range descs {
			log.Println(desc)
			i, err := api.seriesEngine.DB.NewIterator(desc.Source, desc.Metric)
			if err != nil {
				log.Println(err)
				continue
			}

			err = i.Seek(desc.Start)
			if err != nil {
				log.Println(err)
				continue
			}

			s := querySeries{
				Source: desc.Source,
				Metric: desc.Metric,
				Start:  i.Point().Timestamp,
				End:    i.Point().Timestamp,
			}

			pointsSeen := 0

			currentInterval := i.Point().Timestamp / *pointWidth
			currentPoint := catena.Point{
				Timestamp: currentInterval * *pointWidth,
			}

			for {
				p := i.Point()
				if p.Timestamp > desc.End {
					break
				}

				if p.Timestamp / *pointWidth != currentInterval {
					currentPoint.Value /= float64(pointsSeen)
					s.Points = append(s.Points, currentPoint)
					currentInterval = i.Point().Timestamp / *pointWidth
					currentPoint = catena.Point{
						Timestamp: currentInterval * *pointWidth,
						Value:     p.Value,
					}
					pointsSeen = 1
					continue
				}

				currentPoint.Value += p.Value
				pointsSeen++

				err := i.Next()
				if err != nil {
					log.Println(err)
					break
				}
			}

			if pointsSeen > 0 {
				currentPoint.Value /= float64(pointsSeen)
				s.Points = append(s.Points, currentPoint)
			}
			i.Close()

			resp.Series = append(resp.Series, s)
		}

		c.Set(responseKey, resp)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/dominikh/simple-router/conntrack"
	"github.com/dominikh/simple-router/lookup"

	"flag"
	"fmt"
	"os"
	"text/tabwriter"
)

// TODO implement the following flags
//       -p <protocol>        : display connections by protocol
//       -s <source-host>     : display connections by source
//       -d <destination-host>: display connections by destination
//       -x: extended hostnames view
//       -r src | dst | src-port | dst-port | state : sort connections
//       -N: display NAT box connection information (only valid with SNAT & DNAT)
//       -v: print version

var onlySNAT = flag.Bool("S", false, "Display only SNAT connections")
var onlyDNAT = flag.Bool("D", false, "Display only DNAT connections")
var onlyLocal = flag.Bool("L", false, "Display only local connections (originating from or going to the router)")
var onlyRouted = flag.Bool("R", false, "Display only connections routed through the router")
var noResolve = flag.Bool("n", false, "Do not resolve hostnames") // TODO resolve port names as well
var noHeader = flag.Bool("o", false, "Strip output header")

func main() {
	flag.Parse()

	var which conntrack.FilterFlag

	if *onlySNAT {
		which = conntrack.SNATFilter
	}

	if *onlyDNAT {
		which = conntrack.DNATFilter
	}

	if *onlyLocal {
		which = conntrack.LocalFilter
	}

	if *onlyRouted {
		which = conntrack.RoutedFilter
	}

	flows, err := conntrack.Flows()
	if err != nil {
		panic(err)
	}

	tabWriter := &tabwriter.Writer{}
	tabWriter.Init(os.Stdout, 0, 0, 4, ' ', 0)

	if !*noHeader {
		fmt.Fprintln(tabWriter, "Proto\tSource Address\tDestination Address\tState")
	}

	natFlows := flows.Filter(which)
	for _, flow := range natFlows {
		sHostname := lookup.Resolve(flow.Original.Source, *noResolve)
		dHostname := lookup.Resolve(flow.Original.Destination, *noResolve)

		fmt.Fprintf(tabWriter, "%s\t%s:%d\t%s:%d\t%s\n",
			flow.Protocol,
			sHostname,
			flow.Original.SPort,
			dHostname,
			flow.Original.DPort,
			flow.State,
		)
	}
	tabWriter.Flush()
}
<commit_msg>netstat-nat: display SNAT and DNAT by default, instead of nothing<commit_after>package main

import (
	"github.com/dominikh/simple-router/conntrack"
	"github.com/dominikh/simple-router/lookup"

	"flag"
	"fmt"
	"os"
	"text/tabwriter"
)

// TODO implement the following flags
//       -p <protocol>        : display connections by protocol
//       -s <source-host>     : display connections by source
//       -d <destination-host>: display connections by destination
//       -x: extended hostnames view
//       -r src | dst | src-port | dst-port | state : sort connections
//       -N: display NAT box connection information (only valid with SNAT & DNAT)
//       -v: print version

var onlySNAT = flag.Bool("S", false, "Display only SNAT connections")
var onlyDNAT = flag.Bool("D", false, "Display only DNAT connections")
var onlyLocal = flag.Bool("L", false, "Display only local connections (originating from or going to the router)")
var onlyRouted = flag.Bool("R", false, "Display only connections routed through the router")
var noResolve = flag.Bool("n", false, "Do not resolve hostnames") // TODO resolve port names as well
var noHeader = flag.Bool("o", false, "Strip output header")

func main() {
	flag.Parse()

	which := conntrack.SNATFilter | conntrack.DNATFilter

	if *onlySNAT {
		which = conntrack.SNATFilter
	}

	if *onlyDNAT {
		which = conntrack.DNATFilter
	}

	if *onlyLocal {
		which = conntrack.LocalFilter
	}

	if *onlyRouted {
		which = conntrack.RoutedFilter
	}

	flows, err := conntrack.Flows()
	if err != nil {
		panic(err)
	}

	tabWriter := &tabwriter.Writer{}
	tabWriter.Init(os.Stdout, 0, 0, 4, ' ', 0)

	if !*noHeader {
		fmt.Fprintln(tabWriter, "Proto\tSource Address\tDestination Address\tState")
	}

	natFlows := flows.Filter(which)
	for _, flow := range natFlows {
		sHostname := lookup.Resolve(flow.Original.Source, *noResolve)
		dHostname := lookup.Resolve(flow.Original.Destination, *noResolve)

		fmt.Fprintf(tabWriter, "%s\t%s:%d\t%s:%d\t%s\n",
			flow.Protocol,
			sHostname,
			flow.Original.SPort,
			dHostname,
			flow.Original.DPort,
			flow.State,
		)
	}
	tabWriter.Flush()
}
<|endoftext|>


<commit_before>package main

import (
  "path"
  "os"
  "sync"
  "log"
  "sort"
  "io"
  "io/ioutil"
  "path/filepath"
  "os/exec"
)

const (
  CopyPrice = 100
  RenamePrice = 10

  RemoveBackupPrice = 1000
  RemoveFactor = RenamePrice
  UpdateFactor = RenamePrice + CopyPrice
  AddFactor = CopyPrice
)

const (
  BackupExt = ".bak"
)

type BackupPair struct {
  relpath string
  newpath string
}

type ProgressHandler interface {
  HandleSystemMessage(message string)
  HandlePercentChange(percent int)
  HandleFinish()
}

type LogProgressHandler struct {
}

type ProgressReporter struct {
  grandTotal uint64
  currentProgress uint64
  progressChan chan int64
  percent int //0..100
  reportingChan chan bool
  systemMessageChan chan string
  finished chan bool
  progressHandler ProgressHandler
}

type PackageInstaller struct {
  backups map[string]string
  backupsChan chan BackupPair
  progressReporter *ProgressReporter
  installDir string
  packageDir string
  removeSelfPath string // if updating the installer
  failInTheEnd bool // for debugging purposes
}

func (pi *PackageInstaller) Install(filesProvider UpdateFilesProvider) error {
  pi.progressReporter.grandTotal = pi.calculateGrandTotals(filesProvider)
  go pi.progressReporter.reportingLoop()
  defer pi.progressReporter.shutdown()

  pi.beforeInstall()

  err := pi.installPackage(filesProvider)

  if (err == nil) && (!pi.failInTheEnd) {
    pi.afterSuccess()
  } else {
    pi.afterFailure(filesProvider)
  }

  return err
}

func (pi *PackageInstaller) calculateGrandTotals(filesProvider UpdateFilesProvider) uint64 {
  var sum uint64

  for _, fi := range filesProvider.FilesToRemove() {
    sum += uint64(fi.FileSize * RemoveFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToUpdate() {
    sum += uint64(fi.FileSize * UpdateFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToAdd() {
    sum += uint64(fi.FileSize * AddFactor) / 100
  }

  return sum
}

func (pi *PackageInstaller) beforeInstall() {
  pi.removeOldBackups()
}

func (pi *PackageInstaller) installPackage(filesProvider UpdateFilesProvider) (err error) {
  log.Println("Installing package...")

  var wg sync.WaitGroup
  wg.Add(1)
  go func() {
    for bp := range pi.backupsChan {
      pi.backups[bp.relpath] = bp.newpath
    }
    wg.Done()
  }()

  defer func() {
    close(pi.backupsChan)
  }()

  pi.progressReporter.systemMessageChan <- "Removing components"
  err = pi.removeFiles(filesProvider.FilesToRemove())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Updating components"
  err = pi.updateFiles(filesProvider.FilesToUpdate())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Adding components"
  err = pi.addFiles(filesProvider.FilesToAdd())
  if err != nil {
    return err
  }

  wg.Wait()

  return err
}

func (pi *PackageInstaller) afterSuccess() {
  log.Println("After success")
  pi.progressReporter.systemMessageChan <- "Finishing the installation..."
  pi.removeBackups();
  cleanupEmptyDirs(pi.installDir)
}

func (pi *PackageInstaller) afterFailure(filesProvider UpdateFilesProvider) {
  log.Println("After failure")
  pi.progressReporter.systemMessageChan <- "Cleaning up..."
  purgeFiles(pi.installDir, filesProvider.FilesToAdd())
  pi.restoreBackups()
  pi.removeBackups()
  cleanupEmptyDirs(pi.installDir)
}

func copyFile(src, dst string) (err error) {
  log.Printf("About to copy file %v to %v", src, dst)

  fi, err := os.Stat(src)
  if err != nil { return err }
  sourceMode := fi.Mode()

  in, err := os.Open(src)
  if err != nil {
    log.Printf("Failed to open source: %v", err)
    return err
  }

  defer in.Close()

  out, err := os.OpenFile(dst, os.O_RDWR | os.O_TRUNC | os.O_CREATE, sourceMode)
  if err != nil {
    log.Printf("Failed to create destination: %v", err)
    return
  }

  defer func() {
    cerr := out.Close()
    if err == nil {
      err = cerr
    }
  }()

  if _, err = io.Copy(out, in); err != nil {
    return
  }

  err = out.Sync()
  return
}

func (pi *PackageInstaller) backupFile(relpath string) error {
  log.Printf("Backing up %v", relpath)

  oldpath := path.Join(pi.installDir, relpath)
  backupPath := relpath + BackupExt

  newpath := path.Join(pi.installDir, backupPath)
  // remove previous backup if any
  os.Remove(newpath)

  err := os.Rename(oldpath, newpath)

  if err == nil {
    pi.backupsChan <- BackupPair{relpath: relpath, newpath: newpath}
  } else {
    log.Printf("Backup failed: %v", err)
  }

  return err
}

func (pi *PackageInstaller) restoreBackups() {
  log.Printf("Restoring %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for relpath, backuppath := range pi.backups {
    wg.Add(1)

    relativePath := relpath
    pathToRestore := backuppath

    go func() {
      defer wg.Done()

      oldpath := path.Join(pi.installDir, relativePath)
      log.Printf("Restoring %v to %v", pathToRestore, oldpath)
      err := os.Rename(pathToRestore, oldpath)

      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeOldBackups() {
  backeduppath := currentExeFullPath + BackupExt
  err := os.Remove(backeduppath)
  if err == nil {
    log.Println("Old installer backup removed", backeduppath)
  } else if os.IsNotExist(err) {
    log.Println("Old installer backup was not found")
  } else {
    log.Printf("Error while removing old backup: %v", err)
  }
}

func (pi *PackageInstaller) removeBackups() {
  log.Printf("Removing %v backups", len(pi.backups))

  selfpath, err := filepath.Rel(pi.installDir, currentExeFullPath)
  if err == nil {
    if backuppath, ok := pi.backups[selfpath]; ok {
      pi.removeSelfPath = backuppath
      delete(pi.backups, selfpath)
    }
  }

  var wg sync.WaitGroup

  for _, backuppath := range pi.backups {
    wg.Add(1)

    pathToRemove := backuppath

    go func() {
      defer wg.Done()

      err := os.Remove(pathToRemove)
      if err != nil {
        log.Println(err)
      }

      go pi.progressReporter.accountBackupRemove()
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeFiles(files []*UpdateFileInfo) error {
  log.Printf("Removing %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)
    pathToRemove, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      fullpath := filepath.Join(pi.installDir, pathToRemove)
      log.Printf("Removing file %v", fullpath)

      err := pi.backupFile(pathToRemove)

      if err != nil {
        log.Printf("Removing file %v failed", pathToRemove)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountRemove(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) updateFiles(files []*UpdateFileInfo) error {
  log.Printf("Updating %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToUpdate, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToUpdate)
      log.Printf("Updating file %v", oldpath)

      err := pi.backupFile(pathToUpdate)

      if err == nil {
        newpath := path.Join(pi.packageDir, pathToUpdate)
        err = os.Rename(newpath, oldpath)
      }

      if err != nil {
        log.Printf("Updating file %v failed", pathToUpdate)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountUpdate(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) addFiles(files []*UpdateFileInfo) error {
  log.Printf("Adding %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToAdd, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToAdd)
      ensureDirExists(oldpath)

      newpath := path.Join(pi.packageDir, pathToAdd)
      err := os.Rename(newpath, oldpath)

      log.Printf("Adding file %v", pathToAdd)

      if err != nil {
        log.Printf("Adding file %v failed", pathToAdd)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountAdd(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) removeSelfIfNeeded() {
  if len(pi.removeSelfPath) == 0 {
    log.Println("No need to remove itself")
    return
  }

  pathToRemove := filepath.FromSlash(pi.removeSelfPath)
  log.Println("Removing exe backup", pathToRemove)
  cmd := exec.Command("cmd", "/C", "ping localhost -n 2 -w 5000 > nul & del", pathToRemove)
  err := cmd.Start()
  if err != nil {
    log.Println(err)
  }
}

func purgeFiles(root string, files []*UpdateFileInfo) {
  log.Printf("Purging %v files", len(files))

  var wg sync.WaitGroup

  for _, fi := range files {
    wg.Add(1)

    fileToPurge := fi.Filepath

    go func() {
      defer wg.Done()

      fullpath := path.Join(root, fileToPurge)
      err := os.Remove(fullpath)
      if err != nil {
        log.Printf("Error while removing %v: %v", fullpath, err)
      }
    }()
  }

  wg.Wait()
}

func ensureDirExists(fullpath string) (err error) {
  dirpath := path.Dir(fullpath)
  err = os.MkdirAll(dirpath, os.ModeDir)
  if err != nil {
    log.Printf("Failed to create directory %v", dirpath)
  }

  return err
}

type ByLength []string

func (s ByLength) Len() int {
    return len(s)
}
func (s ByLength) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByLength) Less(i, j int) bool {
    return len(s[i]) > len(s[j])
}

func cleanupEmptyDirs(root string) {
  dirs := make([]string, 0, 10)

  err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if info.Mode().IsDir() {
      dirs = append(dirs, path)
    }

    return nil
  })

  if err != nil {
    log.Printf("Error while cleaning up empty dirs: %v", err)
  }

  removeEmptyDirs(dirs)
}

func removeEmptyDirs(dirs []string) {
  sort.Sort(ByLength(dirs))

  for _, dirpath := range dirs {
    entries, err := ioutil.ReadDir(dirpath)
    if err != nil { continue }

    if len(entries) == 0 {
      log.Printf("Removing empty dir %v", dirpath)

      err = os.Remove(dirpath)
      if err != nil {
        log.Println(err)
      }
    }
  }
}

func (pr *ProgressReporter) accountRemove(progress int64) {
  pr.progressChan <- (progress*RemoveFactor)/100
}

func (pr *ProgressReporter) accountUpdate(progress int64) {
  pr.progressChan <- (progress*UpdateFactor)/100
}

func (pr *ProgressReporter) accountAdd(progress int64) {
  pr.progressChan <- (progress*AddFactor)/100
}

func (pr *ProgressReporter) accountBackupRemove() {
  // exact size of files is not known when removeBackups()
  // so using some arbitrary value (fair dice roll)
  pr.progressChan <- RemoveBackupPrice
}

func (pr *ProgressReporter) reportingLoop() {
  var wg sync.WaitGroup

  for chunk := range pr.progressChan {
    pr.currentProgress += uint64(chunk)

    percent := (pr.currentProgress*100) / pr.grandTotal
    pr.percent = int(percent)

    wg.Add(1)
    go func() {
      pr.reportingChan <- true
      wg.Done()
    }()
  }

  go func() {
    wg.Wait()
    close(pr.reportingChan)
  }()
}

func (pr *ProgressReporter) shutdown() {
  log.Println("Shutting down progress reporter...")
  close(pr.progressChan)
  go func() {
    pr.finished <- true
  }()
}

func (pr *ProgressReporter) receiveUpdates() {
  for _ = range pr.reportingChan {
    pr.progressHandler.HandlePercentChange(pr.percent)
  }
}

func (pr *ProgressReporter) receiveSystemMessages() {
  for msg := range pr.systemMessageChan {
    pr.progressHandler.HandleSystemMessage(msg)
  }
}

func (pr *ProgressReporter) receiveFinish() {
  <- pr.finished
  pr.progressHandler.HandleFinish()
}

func (pr *ProgressReporter) handleProgress() {
  go pr.receiveSystemMessages()
  go pr.receiveUpdates()
  go pr.receiveFinish()
}

func (ph *LogProgressHandler) HandlePercentChange(percent int) {
  log.Printf("Completed %v%%", percent)
}

func (ph *LogProgressHandler) HandleSystemMessage(msg string) {
  log.Printf("System message: %v", msg)
}

func (ph *LogProgressHandler) HandleFinish() {
  log.Printf("Finished")
}
<commit_msg>Fix for backups accounting<commit_after>package main

import (
  "path"
  "os"
  "sync"
  "log"
  "sort"
  "io"
  "io/ioutil"
  "path/filepath"
  "os/exec"
)

const (
  CopyPrice = 100
  RenamePrice = 10

  RemoveBackupPrice = 1000
  RemoveFactor = RenamePrice
  UpdateFactor = RenamePrice + CopyPrice
  AddFactor = CopyPrice
)

const (
  BackupExt = ".bak"
)

type BackupPair struct {
  relpath string
  newpath string
}

type ProgressHandler interface {
  HandleSystemMessage(message string)
  HandlePercentChange(percent int)
  HandleFinish()
}

type LogProgressHandler struct {
}

type ProgressReporter struct {
  grandTotal uint64
  currentProgress uint64
  progressChan chan int64
  percent int //0..100
  reportingChan chan bool
  systemMessageChan chan string
  finished chan bool
  progressHandler ProgressHandler
}

type PackageInstaller struct {
  backups map[string]string
  backupsChan chan BackupPair
  backupsWG sync.WaitGroup
  progressReporter *ProgressReporter
  installDir string
  packageDir string
  removeSelfPath string // if updating the installer
  failInTheEnd bool // for debugging purposes
}

func (pi *PackageInstaller) Install(filesProvider UpdateFilesProvider) error {
  pi.progressReporter.grandTotal = pi.calculateGrandTotals(filesProvider)
  go pi.progressReporter.reportingLoop()
  defer pi.progressReporter.shutdown()

  pi.beforeInstall()

  err := pi.installPackage(filesProvider)

  if (err == nil) && (!pi.failInTheEnd) {
    pi.afterSuccess()
  } else {
    pi.afterFailure(filesProvider)
  }

  return err
}

func (pi *PackageInstaller) calculateGrandTotals(filesProvider UpdateFilesProvider) uint64 {
  var sum uint64

  for _, fi := range filesProvider.FilesToRemove() {
    sum += uint64(fi.FileSize * RemoveFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToUpdate() {
    sum += uint64(fi.FileSize * UpdateFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToAdd() {
    sum += uint64(fi.FileSize * AddFactor) / 100
  }

  return sum
}

func (pi *PackageInstaller) beforeInstall() {
  pi.removeOldBackups()
}

func (pi *PackageInstaller) installPackage(filesProvider UpdateFilesProvider) (err error) {
  log.Println("Installing package...")

  go func() {
    for bp := range pi.backupsChan {
      pi.backups[bp.relpath] = bp.newpath
      pi.backupsWG.Done()
    }

    log.Println("Backups accounting finished")
  }()

  defer func() {
    close(pi.backupsChan)
  }()

  pi.progressReporter.systemMessageChan <- "Removing components"
  err = pi.removeFiles(filesProvider.FilesToRemove())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Updating components"
  err = pi.updateFiles(filesProvider.FilesToUpdate())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Adding components"
  err = pi.addFiles(filesProvider.FilesToAdd())
  if err != nil {
    return err
  }

  pi.backupsWG.Wait()

  return err
}

func (pi *PackageInstaller) afterSuccess() {
  log.Println("After success")
  pi.progressReporter.systemMessageChan <- "Finishing the installation..."
  pi.removeBackups();
  cleanupEmptyDirs(pi.installDir)
}

func (pi *PackageInstaller) afterFailure(filesProvider UpdateFilesProvider) {
  log.Println("After failure")
  pi.progressReporter.systemMessageChan <- "Cleaning up..."
  purgeFiles(pi.installDir, filesProvider.FilesToAdd())
  pi.restoreBackups()
  pi.removeBackups()
  cleanupEmptyDirs(pi.installDir)
}

func copyFile(src, dst string) (err error) {
  log.Printf("About to copy file %v to %v", src, dst)

  fi, err := os.Stat(src)
  if err != nil { return err }
  sourceMode := fi.Mode()

  in, err := os.Open(src)
  if err != nil {
    log.Printf("Failed to open source: %v", err)
    return err
  }

  defer in.Close()

  out, err := os.OpenFile(dst, os.O_RDWR | os.O_TRUNC | os.O_CREATE, sourceMode)
  if err != nil {
    log.Printf("Failed to create destination: %v", err)
    return
  }

  defer func() {
    cerr := out.Close()
    if err == nil {
      err = cerr
    }
  }()

  if _, err = io.Copy(out, in); err != nil {
    return
  }

  err = out.Sync()
  return
}

func (pi *PackageInstaller) backupFile(relpath string) error {
  log.Printf("Backing up %v", relpath)

  oldpath := path.Join(pi.installDir, relpath)
  backupPath := relpath + BackupExt

  newpath := path.Join(pi.installDir, backupPath)
  // remove previous backup if any
  os.Remove(newpath)

  err := os.Rename(oldpath, newpath)

  if err == nil {
    pi.backupsWG.Add(1)
    go func() {
      pi.backupsChan <- BackupPair{relpath: relpath, newpath: newpath}
    }()
  } else {
    log.Printf("Backup failed: %v", err)
  }

  return err
}

func (pi *PackageInstaller) restoreBackups() {
  log.Printf("Restoring %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for relpath, backuppath := range pi.backups {
    wg.Add(1)

    relativePath := relpath
    pathToRestore := backuppath

    go func() {
      defer wg.Done()

      oldpath := path.Join(pi.installDir, relativePath)
      log.Printf("Restoring %v to %v", pathToRestore, oldpath)
      err := os.Rename(pathToRestore, oldpath)

      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeOldBackups() {
  backeduppath := currentExeFullPath + BackupExt
  err := os.Remove(backeduppath)
  if err == nil {
    log.Println("Old installer backup removed", backeduppath)
  } else if os.IsNotExist(err) {
    log.Println("Old installer backup was not found")
  } else {
    log.Printf("Error while removing old backup: %v", err)
  }
}

func (pi *PackageInstaller) removeBackups() {
  log.Printf("Removing %v backups", len(pi.backups))

  selfpath, err := filepath.Rel(pi.installDir, currentExeFullPath)
  if err == nil {
    if backuppath, ok := pi.backups[selfpath]; ok {
      pi.removeSelfPath = backuppath
      delete(pi.backups, selfpath)
    }
  }

  var wg sync.WaitGroup

  for _, backuppath := range pi.backups {
    wg.Add(1)

    pathToRemove := backuppath

    go func() {
      defer wg.Done()

      err := os.Remove(pathToRemove)
      if err != nil {
        log.Println(err)
      }

      go pi.progressReporter.accountBackupRemove()
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeFiles(files []*UpdateFileInfo) error {
  log.Printf("Removing %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)
    pathToRemove, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      fullpath := filepath.Join(pi.installDir, pathToRemove)
      log.Printf("Removing file %v", fullpath)

      err := pi.backupFile(pathToRemove)

      if err != nil {
        log.Printf("Removing file %v failed", pathToRemove)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountRemove(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) updateFiles(files []*UpdateFileInfo) error {
  log.Printf("Updating %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToUpdate, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToUpdate)
      log.Printf("Updating file %v", oldpath)

      err := pi.backupFile(pathToUpdate)

      if err == nil {
        newpath := path.Join(pi.packageDir, pathToUpdate)
        err = os.Rename(newpath, oldpath)
      }

      if err != nil {
        log.Printf("Updating file %v failed", pathToUpdate)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountUpdate(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) addFiles(files []*UpdateFileInfo) error {
  log.Printf("Adding %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToAdd, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToAdd)
      ensureDirExists(oldpath)

      newpath := path.Join(pi.packageDir, pathToAdd)
      err := os.Rename(newpath, oldpath)

      log.Printf("Adding file %v", pathToAdd)

      if err != nil {
        log.Printf("Adding file %v failed", pathToAdd)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountAdd(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) removeSelfIfNeeded() {
  if len(pi.removeSelfPath) == 0 {
    log.Println("No need to remove itself")
    return
  }

  pathToRemove := filepath.FromSlash(pi.removeSelfPath)
  log.Println("Removing exe backup", pathToRemove)
  cmd := exec.Command("cmd", "/C", "ping localhost -n 2 -w 5000 > nul & del", pathToRemove)
  err := cmd.Start()
  if err != nil {
    log.Println(err)
  }
}

func purgeFiles(root string, files []*UpdateFileInfo) {
  log.Printf("Purging %v files", len(files))

  var wg sync.WaitGroup

  for _, fi := range files {
    wg.Add(1)

    fileToPurge := fi.Filepath

    go func() {
      defer wg.Done()

      fullpath := path.Join(root, fileToPurge)
      err := os.Remove(fullpath)
      if err != nil {
        log.Printf("Error while removing %v: %v", fullpath, err)
      }
    }()
  }

  wg.Wait()
}

func ensureDirExists(fullpath string) (err error) {
  dirpath := path.Dir(fullpath)
  err = os.MkdirAll(dirpath, os.ModeDir)
  if err != nil {
    log.Printf("Failed to create directory %v", dirpath)
  }

  return err
}

type ByLength []string

func (s ByLength) Len() int {
    return len(s)
}
func (s ByLength) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByLength) Less(i, j int) bool {
    return len(s[i]) > len(s[j])
}

func cleanupEmptyDirs(root string) {
  dirs := make([]string, 0, 10)

  err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if info.Mode().IsDir() {
      dirs = append(dirs, path)
    }

    return nil
  })

  if err != nil {
    log.Printf("Error while cleaning up empty dirs: %v", err)
  }

  removeEmptyDirs(dirs)
}

func removeEmptyDirs(dirs []string) {
  sort.Sort(ByLength(dirs))

  for _, dirpath := range dirs {
    entries, err := ioutil.ReadDir(dirpath)
    if err != nil { continue }

    if len(entries) == 0 {
      log.Printf("Removing empty dir %v", dirpath)

      err = os.Remove(dirpath)
      if err != nil {
        log.Println(err)
      }
    }
  }
}

func (pr *ProgressReporter) accountRemove(progress int64) {
  pr.progressChan <- (progress*RemoveFactor)/100
}

func (pr *ProgressReporter) accountUpdate(progress int64) {
  pr.progressChan <- (progress*UpdateFactor)/100
}

func (pr *ProgressReporter) accountAdd(progress int64) {
  pr.progressChan <- (progress*AddFactor)/100
}

func (pr *ProgressReporter) accountBackupRemove() {
  // exact size of files is not known when removeBackups()
  // so using some arbitrary value (fair dice roll)
  pr.progressChan <- RemoveBackupPrice
}

func (pr *ProgressReporter) reportingLoop() {
  var wg sync.WaitGroup

  for chunk := range pr.progressChan {
    pr.currentProgress += uint64(chunk)

    percent := (pr.currentProgress*100) / pr.grandTotal
    pr.percent = int(percent)

    wg.Add(1)
    go func() {
      pr.reportingChan <- true
      wg.Done()
    }()
  }

  go func() {
    wg.Wait()
    close(pr.reportingChan)
  }()
}

func (pr *ProgressReporter) shutdown() {
  log.Println("Shutting down progress reporter...")
  close(pr.progressChan)
  go func() {
    pr.finished <- true
  }()
}

func (pr *ProgressReporter) receiveUpdates() {
  for _ = range pr.reportingChan {
    pr.progressHandler.HandlePercentChange(pr.percent)
  }
}

func (pr *ProgressReporter) receiveSystemMessages() {
  for msg := range pr.systemMessageChan {
    pr.progressHandler.HandleSystemMessage(msg)
  }
}

func (pr *ProgressReporter) receiveFinish() {
  <- pr.finished
  pr.progressHandler.HandleFinish()
}

func (pr *ProgressReporter) handleProgress() {
  go pr.receiveSystemMessages()
  go pr.receiveUpdates()
  go pr.receiveFinish()
}

func (ph *LogProgressHandler) HandlePercentChange(percent int) {
  log.Printf("Completed %v%%", percent)
}

func (ph *LogProgressHandler) HandleSystemMessage(msg string) {
  log.Printf("System message: %v", msg)
}

func (ph *LogProgressHandler) HandleFinish() {
  log.Printf("Finished")
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	corev1listers "k8s.io/client-go/listers/core/v1"
	extv1beta1listers "k8s.io/client-go/listers/extensions/v1beta1"

	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1alpha2"
	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
	"github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/http/solver"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	pkgutil "github.com/jetstack/cert-manager/pkg/util"
)

const (
	// HTTP01Timeout is the max amount of time to wait for an HTTP01 challenge
	// to succeed
	HTTP01Timeout = time.Minute * 15
	// acmeSolverListenPort is the port acmesolver should listen on
	acmeSolverListenPort = 8089

	domainLabelKey               = "acme.cert-manager.io/http-domain"
	tokenLabelKey                = "acme.cert-manager.io/http-token"
	solverIdentificationLabelKey = "acme.cert-manager.io/http01-solver"
)

var (
	challengeGvk = cmacme.SchemeGroupVersion.WithKind("Challenge")
)

// Solver is an implementation of the acme http-01 challenge solver protocol
type Solver struct {
	*controller.Context

	podLister     corev1listers.PodLister
	serviceLister corev1listers.ServiceLister
	ingressLister extv1beta1listers.IngressLister

	testReachability reachabilityTest
	requiredPasses   int
}

type reachabilityTest func(ctx context.Context, url *url.URL, key string) error

// NewSolver returns a new ACME HTTP01 solver for the given Issuer and client.
// TODO: refactor this to have fewer args
func NewSolver(ctx *controller.Context) *Solver {
	return &Solver{
		Context:          ctx,
		podLister:        ctx.KubeSharedInformerFactory.Core().V1().Pods().Lister(),
		serviceLister:    ctx.KubeSharedInformerFactory.Core().V1().Services().Lister(),
		ingressLister:    ctx.KubeSharedInformerFactory.Extensions().V1beta1().Ingresses().Lister(),
		testReachability: testReachability,
		requiredPasses:   5,
	}
}

func http01LogCtx(ctx context.Context) context.Context {
	return logf.NewContext(ctx, nil, "http01")
}

func httpDomainCfgForChallenge(ch *cmacme.Challenge) (*cmacme.ACMEChallengeSolverHTTP01Ingress, error) {
	if ch.Spec.Solver.HTTP01 == nil || ch.Spec.Solver.HTTP01.Ingress == nil {
		return nil, fmt.Errorf("challenge's 'solver' field is specified but no HTTP01 ingress config provided. " +
			"Ensure solvers[].http01.ingress is specified on your issuer resource")
	}
	return ch.Spec.Solver.HTTP01.Ingress, nil
}

// Present will realise the resources required to solve the given HTTP01
// challenge validation in the apiserver. If those resources already exist, it
// will return nil (i.e. this function is idempotent).
func (s *Solver) Present(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	ctx = http01LogCtx(ctx)

	_, podErr := s.ensurePod(ctx, ch)
	svc, svcErr := s.ensureService(ctx, ch)
	if svcErr != nil {
		return utilerrors.NewAggregate([]error{podErr, svcErr})
	}
	_, ingressErr := s.ensureIngress(ctx, ch, svc.Name)
	return utilerrors.NewAggregate([]error{podErr, svcErr, ingressErr})
}

func (s *Solver) Check(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	ctx = logf.NewContext(http01LogCtx(ctx), nil, "selfCheck")
	log := logf.FromContext(ctx)

	// HTTP Present is idempotent and the state of the system may have
	// changed since present was called by the controllers (killed pods, drained nodes)
	// Call present again to be certain.
	// if the listers are nil, that means we're in the present checks
	// test
	if s.podLister != nil && s.serviceLister != nil && s.ingressLister != nil {
		log.V(logf.DebugLevel).Info("calling Present function before running self check to ensure required resources exist")
		err := s.Present(ctx, issuer, ch)
		if err != nil {
			log.V(logf.DebugLevel).Info("failed to call Present function", "error", err)
			return err
		}
	}

	ctx, cancel := context.WithTimeout(ctx, HTTP01Timeout)
	defer cancel()
	url := s.buildChallengeUrl(ch)
	log = log.WithValues("url", url)
	ctx = logf.NewContext(ctx, log)

	log.V(logf.DebugLevel).Info("running self check multiple times to ensure challenge has propagated", "required_passes", s.requiredPasses)
	for i := 0; i < s.requiredPasses; i++ {
		err := s.testReachability(ctx, url, ch.Spec.Key)
		if err != nil {
			return err
		}
		log.V(logf.DebugLevel).Info("reachability test passed, re-checking in 2s time")
		time.Sleep(time.Second * 2)
	}

	log.V(logf.DebugLevel).Info("self check succeeded")

	return nil
}

// CleanUp will ensure the created service, ingress and pod are clean/deleted of any
// cert-manager created data.
func (s *Solver) CleanUp(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	var errs []error
	errs = append(errs, s.cleanupPods(ctx, ch))
	errs = append(errs, s.cleanupServices(ctx, ch))
	errs = append(errs, s.cleanupIngresses(ctx, ch))
	return utilerrors.NewAggregate(errs)
}

func (s *Solver) buildChallengeUrl(ch *cmacme.Challenge) *url.URL {
	url := &url.URL{}
	url.Scheme = "http"
	url.Host = ch.Spec.DNSName
	url.Path = fmt.Sprintf("%s/%s", solver.HTTPChallengePath, ch.Spec.Token)

	return url
}

// testReachability will attempt to connect to the 'domain' with 'path' and
// check if the returned body equals 'key'
func testReachability(ctx context.Context, url *url.URL, key string) error {
	log := logf.FromContext(ctx)
	log.V(logf.DebugLevel).Info("performing HTTP01 reachability check")

	req := &http.Request{
		Method: http.MethodGet,
		URL:    url,
	}
	req.Header.Set("User-Agent", pkgutil.CertManagerUserAgent)
	req = req.WithContext(ctx)

	// ACME spec says that a verifier should try
	// on http port 80 first, but follow any redirects may be thrown its way
	// The redirects may be HTTPS and its certificate may be invalid (they are trying to get a
	// certificate after all).
	// TODO(dmo): figure out if we need to add a more specific timeout for
	// individual checks
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		// we're only doing 1 request, make the code around this
		// simpler by disabling keepalives
		DisableKeepAlives: true,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	client := http.Client{
		Transport: transport,
	}

	response, err := client.Do(req)
	if err != nil {
		log.V(logf.DebugLevel).Info("failed to perform self check GET request", "error", err)
		return fmt.Errorf("failed to perform self check GET request '%s': %v", url, err)
	}

	if response.StatusCode != http.StatusOK {
		log.V(logf.DebugLevel).Info("received HTTP status code was not StatusOK (200)", "code", response.StatusCode)
		return fmt.Errorf("wrong status code '%d', expected '%d'", response.StatusCode, http.StatusOK)
	}

	defer response.Body.Close()
	presentedKey, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.V(logf.DebugLevel).Info("failed to decode response body", "error", err)
		return fmt.Errorf("failed to read response body: %v", err)
	}

	if string(presentedKey) != key {
		// truncate the response before displaying it to avoid extra long strings
		// being displayed to users
		keyToPrint := string(presentedKey)
		if len(keyToPrint) > 24 {
			// trim spaces to make output look right if it ends with whitespace
			keyToPrint = strings.TrimSpace(keyToPrint[:24]) + "... (truncated)"
		}
		log.V(logf.DebugLevel).Info("key returned by server did not match expected", "actual", keyToPrint, "expected", key)
		return fmt.Errorf("did not get expected response when querying endpoint, expected %q but got: %s", key, keyToPrint)
	}

	log.V(logf.DebugLevel).Info("reachability test succeeded")

	return nil
}
<commit_msg>Use http.NewRequest<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	corev1listers "k8s.io/client-go/listers/core/v1"
	extv1beta1listers "k8s.io/client-go/listers/extensions/v1beta1"

	cmacme "github.com/jetstack/cert-manager/pkg/apis/acme/v1alpha2"
	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha2"
	"github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/issuer/acme/http/solver"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	pkgutil "github.com/jetstack/cert-manager/pkg/util"
)

const (
	// HTTP01Timeout is the max amount of time to wait for an HTTP01 challenge
	// to succeed
	HTTP01Timeout = time.Minute * 15
	// acmeSolverListenPort is the port acmesolver should listen on
	acmeSolverListenPort = 8089

	domainLabelKey               = "acme.cert-manager.io/http-domain"
	tokenLabelKey                = "acme.cert-manager.io/http-token"
	solverIdentificationLabelKey = "acme.cert-manager.io/http01-solver"
)

var (
	challengeGvk = cmacme.SchemeGroupVersion.WithKind("Challenge")
)

// Solver is an implementation of the acme http-01 challenge solver protocol
type Solver struct {
	*controller.Context

	podLister     corev1listers.PodLister
	serviceLister corev1listers.ServiceLister
	ingressLister extv1beta1listers.IngressLister

	testReachability reachabilityTest
	requiredPasses   int
}

type reachabilityTest func(ctx context.Context, url *url.URL, key string) error

// NewSolver returns a new ACME HTTP01 solver for the given Issuer and client.
// TODO: refactor this to have fewer args
func NewSolver(ctx *controller.Context) *Solver {
	return &Solver{
		Context:          ctx,
		podLister:        ctx.KubeSharedInformerFactory.Core().V1().Pods().Lister(),
		serviceLister:    ctx.KubeSharedInformerFactory.Core().V1().Services().Lister(),
		ingressLister:    ctx.KubeSharedInformerFactory.Extensions().V1beta1().Ingresses().Lister(),
		testReachability: testReachability,
		requiredPasses:   5,
	}
}

func http01LogCtx(ctx context.Context) context.Context {
	return logf.NewContext(ctx, nil, "http01")
}

func httpDomainCfgForChallenge(ch *cmacme.Challenge) (*cmacme.ACMEChallengeSolverHTTP01Ingress, error) {
	if ch.Spec.Solver.HTTP01 == nil || ch.Spec.Solver.HTTP01.Ingress == nil {
		return nil, fmt.Errorf("challenge's 'solver' field is specified but no HTTP01 ingress config provided. " +
			"Ensure solvers[].http01.ingress is specified on your issuer resource")
	}
	return ch.Spec.Solver.HTTP01.Ingress, nil
}

// Present will realise the resources required to solve the given HTTP01
// challenge validation in the apiserver. If those resources already exist, it
// will return nil (i.e. this function is idempotent).
func (s *Solver) Present(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	ctx = http01LogCtx(ctx)

	_, podErr := s.ensurePod(ctx, ch)
	svc, svcErr := s.ensureService(ctx, ch)
	if svcErr != nil {
		return utilerrors.NewAggregate([]error{podErr, svcErr})
	}
	_, ingressErr := s.ensureIngress(ctx, ch, svc.Name)
	return utilerrors.NewAggregate([]error{podErr, svcErr, ingressErr})
}

func (s *Solver) Check(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	ctx = logf.NewContext(http01LogCtx(ctx), nil, "selfCheck")
	log := logf.FromContext(ctx)

	// HTTP Present is idempotent and the state of the system may have
	// changed since present was called by the controllers (killed pods, drained nodes)
	// Call present again to be certain.
	// if the listers are nil, that means we're in the present checks
	// test
	if s.podLister != nil && s.serviceLister != nil && s.ingressLister != nil {
		log.V(logf.DebugLevel).Info("calling Present function before running self check to ensure required resources exist")
		err := s.Present(ctx, issuer, ch)
		if err != nil {
			log.V(logf.DebugLevel).Info("failed to call Present function", "error", err)
			return err
		}
	}

	ctx, cancel := context.WithTimeout(ctx, HTTP01Timeout)
	defer cancel()
	url := s.buildChallengeUrl(ch)
	log = log.WithValues("url", url)
	ctx = logf.NewContext(ctx, log)

	log.V(logf.DebugLevel).Info("running self check multiple times to ensure challenge has propagated", "required_passes", s.requiredPasses)
	for i := 0; i < s.requiredPasses; i++ {
		err := s.testReachability(ctx, url, ch.Spec.Key)
		if err != nil {
			return err
		}
		log.V(logf.DebugLevel).Info("reachability test passed, re-checking in 2s time")
		time.Sleep(time.Second * 2)
	}

	log.V(logf.DebugLevel).Info("self check succeeded")

	return nil
}

// CleanUp will ensure the created service, ingress and pod are clean/deleted of any
// cert-manager created data.
func (s *Solver) CleanUp(ctx context.Context, issuer v1alpha2.GenericIssuer, ch *cmacme.Challenge) error {
	var errs []error
	errs = append(errs, s.cleanupPods(ctx, ch))
	errs = append(errs, s.cleanupServices(ctx, ch))
	errs = append(errs, s.cleanupIngresses(ctx, ch))
	return utilerrors.NewAggregate(errs)
}

func (s *Solver) buildChallengeUrl(ch *cmacme.Challenge) *url.URL {
	url := &url.URL{}
	url.Scheme = "http"
	url.Host = ch.Spec.DNSName
	url.Path = fmt.Sprintf("%s/%s", solver.HTTPChallengePath, ch.Spec.Token)

	return url
}

// testReachability will attempt to connect to the 'domain' with 'path' and
// check if the returned body equals 'key'
func testReachability(ctx context.Context, url *url.URL, key string) error {
	log := logf.FromContext(ctx)
	log.V(logf.DebugLevel).Info("performing HTTP01 reachability check")

	req, err := http.NewRequestWithContext(ctx, url.String(), http.MethodGet, nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", pkgutil.CertManagerUserAgent)

	// ACME spec says that a verifier should try
	// on http port 80 first, but follow any redirects may be thrown its way
	// The redirects may be HTTPS and its certificate may be invalid (they are trying to get a
	// certificate after all).
	// TODO(dmo): figure out if we need to add a more specific timeout for
	// individual checks
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		// we're only doing 1 request, make the code around this
		// simpler by disabling keepalives
		DisableKeepAlives: true,
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	client := http.Client{
		Transport: transport,
	}

	response, err := client.Do(req)
	if err != nil {
		log.V(logf.DebugLevel).Info("failed to perform self check GET request", "error", err)
		return fmt.Errorf("failed to perform self check GET request '%s': %v", url, err)
	}

	if response.StatusCode != http.StatusOK {
		log.V(logf.DebugLevel).Info("received HTTP status code was not StatusOK (200)", "code", response.StatusCode)
		return fmt.Errorf("wrong status code '%d', expected '%d'", response.StatusCode, http.StatusOK)
	}

	defer response.Body.Close()
	presentedKey, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.V(logf.DebugLevel).Info("failed to decode response body", "error", err)
		return fmt.Errorf("failed to read response body: %v", err)
	}

	if string(presentedKey) != key {
		// truncate the response before displaying it to avoid extra long strings
		// being displayed to users
		keyToPrint := string(presentedKey)
		if len(keyToPrint) > 24 {
			// trim spaces to make output look right if it ends with whitespace
			keyToPrint = strings.TrimSpace(keyToPrint[:24]) + "... (truncated)"
		}
		log.V(logf.DebugLevel).Info("key returned by server did not match expected", "actual", keyToPrint, "expected", key)
		return fmt.Errorf("did not get expected response when querying endpoint, expected %q but got: %s", key, keyToPrint)
	}

	log.V(logf.DebugLevel).Info("reachability test succeeded")

	return nil
}
<|endoftext|>


<commit_before>package docker

//TODO monitor event and update data
import (
	"sort"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/sapk/sca/pkg/model"
	"github.com/sapk/sca/pkg/tools"
	log "github.com/sirupsen/logrus"
)

const id = "Docker"

//Module retrieve information form executing sca
type Module struct {
	Endpoint string
	Client   *docker.Client
	event    <-chan string
}

//Response describe docker informations
type Response struct {
	Info       *docker.DockerInfo     `json:"Info,omitempty"`
	Containers []docker.APIContainers `json:"Containers,omitempty"`
	Images     []docker.APIImages     `json:"Images,omitempty"`
	Volumes    []docker.Volume        `json:"Volumes,omitempty"`
	Networks   []docker.Network       `json:"Networks,omitempty"`
}

//New constructor for Module
func New(options map[string]string) model.Module {
	log.WithFields(log.Fields{
		"id":      id,
		"options": options,
	}).Debug("Creating new Module")

	client, err := docker.NewClient(options["docker.endpoint"])
	if err != nil {
		log.WithFields(log.Fields{
			"client": client,
			"err":    err,
		}).Warn("Failed to create docker client")
		//return nil
	}
	return &Module{Endpoint: options["docker.endpoint"], Client: client, event: setListener(client)}
}

func setListener(client *docker.Client) <-chan string {
	listener := make(chan *docker.APIEvents)
	out := make(chan string)
	err := client.AddEventListener(listener)
	if err != nil {
		log.WithFields(log.Fields{
			"client": client,
			"err":    err,
		}).Warn("Failed to set docker listener for event")
	}
	go func() {
		for e := range listener {
			//for range listener {
			log.WithFields(log.Fields{
				"event": e,
			}).Debug("Module.Docker Receive event from docker client")
			out <- id
		}
	}()
	/*
		defer func() {
			err = client.RemoveEventListener(listener)
			if err != nil {
				log.Fatal(err)
			}
		}()
	*/
	return out
}

//ID //TODO
func (d *Module) ID() string {
	return id
}

//Event return event chan
func (d *Module) Event() <-chan string {
	return d.event
}

//GetData //TODO
func (d *Module) GetData() interface{} {

	return Response{
		Info:       d.getInfo(),
		Containers: d.getContainers(),
		Networks:   d.getNetworks(),
		Volumes:    d.getVolumes(),
		Images:     d.getImages(),
	}
}
func (d *Module) getInfo() *docker.DockerInfo {
	//Get server info
	info, err := d.Client.Info()
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"info":   info,
			"client": d.Client,
		}).Warn("Failed to get docker host info")
		return nil
	}
	//Clean of . in key info.RegistryConfig.IndexConfigs
	tmp := make(map[string]*docker.IndexInfo, len(info.RegistryConfig.IndexConfigs))
	for id, conf := range info.RegistryConfig.IndexConfigs {
		tmp[strings.Replace(id, ".", "-", -1)] = conf
	}
	info.RegistryConfig.IndexConfigs = tmp

	//Sort Docker/Info/Swarm/RemoteManagers/X to ease optimisation on sync
	sort.Sort(tools.ByPeer(info.Swarm.RemoteManagers))
	sort.Strings(info.Plugins.Network)
	sort.Strings(info.Plugins.Volume)
	return info
}

func (d *Module) getImages() []docker.APIImages {
	//Get images
	imgs, err := d.Client.ListImages(docker.ListImagesOptions{All: true})
	if err != nil {
		panic(err)
	}
	for id, i := range imgs {
		if len(i.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(i.Labels))
			for iid, val := range i.Labels {
				tmp[strings.Replace(iid, ".", "-", -1)] = val
			}
			imgs[id].Labels = tmp
		}
	}
	sort.Sort(tools.ByIID(imgs))
	return imgs
}

func (d *Module) getNetworks() []docker.Network {
	//Get networks
	nets, err := d.Client.ListNetworks()
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"nets":   nets,
			"client": d.Client,
		}).Warn("Failed to get docker network list")
		return nil
	}
	//Clean . in key of options
	for id, n := range nets {
		if len(n.Options) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(n.Options))
			for oid, opt := range n.Options {
				tmp[strings.Replace(oid, ".", "-", -1)] = opt
			}
			nets[id].Options = tmp
		}
		if len(n.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(n.Labels))
			for lid, val := range n.Labels {
				tmp[strings.Replace(lid, ".", "-", -1)] = val
			}
			nets[id].Labels = tmp
		}
	}
	sort.Sort(tools.ByNID(nets))
	return nets
}

func (d *Module) getContainers() []docker.APIContainers {
	//Get container
	cnts, err := d.Client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"cnts":   cnts,
			"client": d.Client,
		}).Warn("Failed to get docker container list")
		return nil
	}
	for id, c := range cnts {
		if len(c.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(c.Labels))
			for vid, val := range c.Labels {
				tmp[strings.Replace(vid, ".", "-", -1)] = val
			}
			cnts[id].Labels = tmp
		}
		//Sort Docker/Containers/X/Mounts/X to ease optimisation on sync
		sort.Sort(tools.ByMount(c.Mounts))
		//Sort Docker/Containers/X/Ports/X to ease optimisation on sync
		sort.Sort(tools.ByPort(c.Ports))
	}
	sort.Sort(tools.ByCID(cnts))
	return cnts
}

func (d *Module) getVolumes() []docker.Volume {
	//Get volumes
	vols, err := d.Client.ListVolumes(docker.ListVolumesOptions{})
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"vols":   vols,
			"client": d.Client,
		}).Warn("Failed to get docker volume list")
		return nil
	}
	sort.Sort(tools.ByVName(vols))
	return vols
}
<commit_msg>Update docker.go<commit_after>package docker

//TODO monitor event and update data
import (
	"sort"
	"strings"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/sapk/sca/pkg/model"
	"github.com/sapk/sca/pkg/tools"
	log "github.com/sirupsen/logrus"
)

const id = "Docker"

//Module retrieve information form executing sca
type Module struct {
	Endpoint string
	Client   *docker.Client
	event    <-chan string
}

//Response describe docker informations
type Response struct {
	Info       *docker.DockerInfo     `json:"Info,omitempty"`
	Containers []docker.APIContainers `json:"Containers,omitempty"`
	Images     []docker.APIImages     `json:"Images,omitempty"`
	Volumes    []docker.Volume        `json:"Volumes,omitempty"`
	Networks   []docker.Network       `json:"Networks,omitempty"`
}

//New constructor for Module
func New(options map[string]string) model.Module {
	log.WithFields(log.Fields{
		"id":      id,
		"options": options,
	}).Debug("Creating new Module")

	client, err := docker.NewClient(options["module.docker.endpoint"])
	if err != nil {
		log.WithFields(log.Fields{
			"client": client,
			"err":    err,
		}).Warn("Failed to create docker client")
		//return nil
	}
	return &Module{Endpoint: options["module.docker.endpoint"], Client: client, event: setListener(client)}
}

func setListener(client *docker.Client) <-chan string {
	listener := make(chan *docker.APIEvents)
	out := make(chan string)
	err := client.AddEventListener(listener)
	if err != nil {
		log.WithFields(log.Fields{
			"client": client,
			"err":    err,
		}).Warn("Failed to set docker listener for event")
	}
	go func() {
		for e := range listener {
			//for range listener {
			log.WithFields(log.Fields{
				"event": e,
			}).Debug("Module.Docker Receive event from docker client")
			out <- id
		}
	}()
	/*
		defer func() {
			err = client.RemoveEventListener(listener)
			if err != nil {
				log.Fatal(err)
			}
		}()
	*/
	return out
}

//ID //TODO
func (d *Module) ID() string {
	return id
}

//Event return event chan
func (d *Module) Event() <-chan string {
	return d.event
}

//GetData //TODO
func (d *Module) GetData() interface{} {

	return Response{
		Info:       d.getInfo(),
		Containers: d.getContainers(),
		Networks:   d.getNetworks(),
		Volumes:    d.getVolumes(),
		Images:     d.getImages(),
	}
}
func (d *Module) getInfo() *docker.DockerInfo {
	//Get server info
	info, err := d.Client.Info()
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"info":   info,
			"client": d.Client,
		}).Warn("Failed to get docker host info")
		return nil
	}
	//Clean of . in key info.RegistryConfig.IndexConfigs
	tmp := make(map[string]*docker.IndexInfo, len(info.RegistryConfig.IndexConfigs))
	for id, conf := range info.RegistryConfig.IndexConfigs {
		tmp[strings.Replace(id, ".", "-", -1)] = conf
	}
	info.RegistryConfig.IndexConfigs = tmp

	//Sort Docker/Info/Swarm/RemoteManagers/X to ease optimisation on sync
	sort.Sort(tools.ByPeer(info.Swarm.RemoteManagers))
	sort.Strings(info.Plugins.Network)
	sort.Strings(info.Plugins.Volume)
	return info
}

func (d *Module) getImages() []docker.APIImages {
	//Get images
	imgs, err := d.Client.ListImages(docker.ListImagesOptions{All: true})
	if err != nil {
		panic(err)
	}
	for id, i := range imgs {
		if len(i.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(i.Labels))
			for iid, val := range i.Labels {
				tmp[strings.Replace(iid, ".", "-", -1)] = val
			}
			imgs[id].Labels = tmp
		}
	}
	sort.Sort(tools.ByIID(imgs))
	return imgs
}

func (d *Module) getNetworks() []docker.Network {
	//Get networks
	nets, err := d.Client.ListNetworks()
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"nets":   nets,
			"client": d.Client,
		}).Warn("Failed to get docker network list")
		return nil
	}
	//Clean . in key of options
	for id, n := range nets {
		if len(n.Options) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(n.Options))
			for oid, opt := range n.Options {
				tmp[strings.Replace(oid, ".", "-", -1)] = opt
			}
			nets[id].Options = tmp
		}
		if len(n.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(n.Labels))
			for lid, val := range n.Labels {
				tmp[strings.Replace(lid, ".", "-", -1)] = val
			}
			nets[id].Labels = tmp
		}
	}
	sort.Sort(tools.ByNID(nets))
	return nets
}

func (d *Module) getContainers() []docker.APIContainers {
	//Get container
	cnts, err := d.Client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"cnts":   cnts,
			"client": d.Client,
		}).Warn("Failed to get docker container list")
		return nil
	}
	for id, c := range cnts {
		if len(c.Labels) > 0 { //Reconstruct map without . in key
			tmp := make(map[string]string, len(c.Labels))
			for vid, val := range c.Labels {
				tmp[strings.Replace(vid, ".", "-", -1)] = val
			}
			cnts[id].Labels = tmp
		}
		//Sort Docker/Containers/X/Mounts/X to ease optimisation on sync
		sort.Sort(tools.ByMount(c.Mounts))
		//Sort Docker/Containers/X/Ports/X to ease optimisation on sync
		sort.Sort(tools.ByPort(c.Ports))
	}
	sort.Sort(tools.ByCID(cnts))
	return cnts
}

func (d *Module) getVolumes() []docker.Volume {
	//Get volumes
	vols, err := d.Client.ListVolumes(docker.ListVolumesOptions{})
	if err != nil {
		log.WithFields(log.Fields{
			"err":    err,
			"vols":   vols,
			"client": d.Client,
		}).Warn("Failed to get docker volume list")
		return nil
	}
	sort.Sort(tools.ByVName(vols))
	return vols
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Juniper Networks, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	"fmt"
	"os/exec"
	"strconv"

	"github.com/docker/libcontainer/netlink"
	"github.com/milosgajdos83/tenus"
)

type NetnsManager interface {
	CreateInterface(dockerId, macAddress, ipAddress, gateway string) (string, error)
	DeleteInterface(dockerId string) error
}

type NetnsManagerImpl struct {
}

func NewNetnsManager() NetnsManager {
	m := new(NetnsManagerImpl)
	return m
}

func (m *NetnsManagerImpl) CreateInterface(dockerId, macAddress, ipAddress, gateway string) (string, error) {
	masterName := fmt.Sprintf("veth-%s", dockerId[0:10])
	veth, err := tenus.NewVethPairWithOptions(masterName, tenus.VethOptions{PeerName: "veth0"})
	if err != nil {
		return "", err
	}
	pid, err := tenus.DockerPidByName(dockerId, "/var/run/docker.sock")
	if err != nil {
		return "", err
	}
	veth.SetPeerLinkNsPid(pid)
	peer := veth.PeerNetInterface()
	netlink.NetworkSetMacAddress(peer, macAddress)
	veth.SetLinkUp()

	cmd := exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid),
		"ip", "link", "set", "veth0", "up")
	err = cmd.Run()
	if err != nil {
		return "", err
	}

	cmd = exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid), "ip", "addr", "add",
		fmt.Sprintf("%s/32", ipAddress), "peer", gateway, "dev", "veth0")
	err = cmd.Run()
	if err != nil {
		return "", err
	}

	cmd = exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid), "ip", "route", "add",
		"default", "via", gateway)
	err = cmd.Run()

	return masterName, nil
}

func (m *NetnsManagerImpl) DeleteInterface(dockerId string) error {
	// masterName := fmt.Sprintf("veth-%s", dockerId[0:10])
	return nil
}
<commit_msg>Attempt to fix the mac-address<commit_after>/*
Copyright 2015 Juniper Networks, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
	"fmt"
	"os/exec"
	"strconv"

	"github.com/docker/libcontainer/netlink"
	"github.com/milosgajdos83/tenus"
)

type NetnsManager interface {
	CreateInterface(dockerId, macAddress, ipAddress, gateway string) (string, error)
	DeleteInterface(dockerId string) error
}

type NetnsManagerImpl struct {
}

func NewNetnsManager() NetnsManager {
	m := new(NetnsManagerImpl)
	return m
}

func (m *NetnsManagerImpl) CreateInterface(dockerId, macAddress, ipAddress, gateway string) (string, error) {
	masterName := fmt.Sprintf("veth-%s", dockerId[0:10])
	veth, err := tenus.NewVethPairWithOptions(masterName, tenus.VethOptions{PeerName: "veth0"})
	if err != nil {
		return "", err
	}
	pid, err := tenus.DockerPidByName(dockerId, "/var/run/docker.sock")
	if err != nil {
		return "", err
	}

	peer := veth.PeerNetInterface()
	netlink.NetworkSetMacAddress(peer, macAddress)
	veth.SetPeerLinkNsPid(pid)
	veth.SetLinkUp()

	cmd := exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid),
		"ip", "link", "set", "veth0", "up")
	err = cmd.Run()
	if err != nil {
		return "", err
	}

	cmd = exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid), "ip", "addr", "add",
		fmt.Sprintf("%s/32", ipAddress), "peer", gateway, "dev", "veth0")
	err = cmd.Run()
	if err != nil {
		return "", err
	}

	cmd = exec.Command("nsenter", "-n", "-t", strconv.Itoa(pid), "ip", "route", "add",
		"default", "via", gateway)
	err = cmd.Run()

	return masterName, nil
}

func (m *NetnsManagerImpl) DeleteInterface(dockerId string) error {
	// masterName := fmt.Sprintf("veth-%s", dockerId[0:10])
	return nil
}
<|endoftext|>