<commit_before>package id3v2

import (
	"bufio"
	"bytes"
	"errors"
	"github.com/bogem/id3v2/frame"
	"github.com/bogem/id3v2/util"
	"io/ioutil"
	"os"
)

type Tag struct {
	Header    *TagHeader
	frames    map[string]frame.Framer
	commonIDs map[string]string

	File         *os.File
	OriginalSize uint32
}

func (t *Tag) SetAttachedPicture(pf frame.PictureFramer) {
	var f frame.APICSequencer
	id := t.commonIDs["Attached Picture"]

	existingFrame := t.frames[id]
	if existingFrame == nil {
		f = frame.NewAPICSequence()
		f.SetID(id)
	} else {
		f = existingFrame.(frame.APICSequencer)
	}

	f.AddPicture(pf)
	t.frames[id] = f
}

func (t *Tag) SetTextFrame(id string, text string) {
	var f frame.TextFramer

	existingFrame := t.frames[id]
	if existingFrame == nil {
		f = new(frame.TextFrame)
		f.SetID(id)
	} else {
		f = existingFrame.(frame.TextFramer)
	}

	f.SetText(text)
	t.frames[id] = f
}

func (t *Tag) SetTitle(title string) {
	t.SetTextFrame(t.commonIDs["Title"], title)
}

func (t *Tag) SetArtist(artist string) {
	t.SetTextFrame(t.commonIDs["Artist"], artist)
}

func (t *Tag) SetAlbum(album string) {
	t.SetTextFrame(t.commonIDs["Album"], album)
}

func (t *Tag) SetYear(year string) {
	t.SetTextFrame(t.commonIDs["Year"], year)
}

func (t *Tag) SetGenre(genre string) {
	t.SetTextFrame(t.commonIDs["Genre"], genre)
}

func NewTag(file *os.File) *Tag {
	header := &TagHeader{
		Version:    4,
		FramesSize: 0,
	}

	return &Tag{
		Header:    header,
		frames:    make(map[string]frame.Framer),
		commonIDs: frame.V24CommonIDs,

		File:         file,
		OriginalSize: 0,
	}
}

func ParseTag(file *os.File) (*Tag, error) {
	header, err := ParseHeader(file)
	if err != nil {
		err = errors.New("Trying to parse tag header: " + err.Error())
		return nil, err
	}
	if header == nil {
		return NewTag(file), nil
	}
	if header.Version < 3 {
		err = errors.New("Unsupported version of ID3 tag")
		return nil, err
	}

	tag := &Tag{
		Header:    header,
		frames:    make(map[string]frame.Framer),
		commonIDs: frame.V24CommonIDs,

		File:         file,
		OriginalSize: TagHeaderSize + header.FramesSize,
	}

	return tag, nil
}

func (t *Tag) Flush() error {
	f := t.File
	defer f.Close()

	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	originalFileBuf := bufio.NewReader(f)
	if _, err := originalFileBuf.Discard(int(t.OriginalSize)); err != nil {
		return err
	}

	newFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	defer os.Remove(newFile.Name())
	newFileBuf := bufio.NewWriter(newFile)

	newTag := bytesBufPool.Get().(*bytes.Buffer)
	newTag.Reset()
	if header, err := FormTagHeader(*t.Header); err != nil {
		return err
	} else {
		newTag.Write(header)
	}
	if frames, err := t.FormFrames(); err != nil {
		return err
	} else {
		newTag.Write(frames)
	}

	tagSize, err := newFileBuf.ReadFrom(newTag)
	if err != nil {
		return err
	}
	bytesBufPool.Put(newTag)
	if _, err = newFileBuf.ReadFrom(originalFileBuf); err != nil {
		return err
	}
	if err = newFileBuf.Flush(); err != nil {
		return err
	}
	setSize(f, uint32(tagSize))

	os.Rename(newFile.Name(), f.Name())
	return nil
}

func setSize(f *os.File, size uint32) (err error) {
	sizeBytes, err := util.FormSize(size)
	if err != nil {
		return
	}

	if _, err = f.WriteAt(sizeBytes, 6); err != nil {
		return
	}

	return
}

func (t Tag) FormFrames() ([]byte, error) {
	b := bytesBufPool.Get().(*bytes.Buffer)
	b.Reset()
	for _, f := range t.frames {
		formedFrame := f.Form()
		frameHeader, err := frame.FormFrameHeader(f, uint32(len(formedFrame)))
		if err != nil {
			return nil, err
		}
		b.Write(frameHeader)
		b.Write(formedFrame)
	}
	bytesBufPool.Put(b)
	return b.Bytes(), nil
}
<commit_msg>Refactoring in tag<commit_after>package id3v2

import (
	"bufio"
	"bytes"
	"errors"
	"github.com/bogem/id3v2/frame"
	"github.com/bogem/id3v2/util"
	"io/ioutil"
	"os"
)

type Tag struct {
	Header    *TagHeader
	frames    map[string]frame.Framer
	commonIDs map[string]string

	File         *os.File
	OriginalSize uint32
}

func (t *Tag) SetAttachedPicture(pf frame.PictureFramer) {
	var f frame.APICSequencer
	id := t.commonIDs["Attached Picture"]

	existingFrame := t.frames[id]
	if existingFrame == nil {
		f = frame.NewAPICSequence()
		f.SetID(id)
	} else {
		f = existingFrame.(frame.APICSequencer)
	}

	f.AddPicture(pf)
	t.frames[id] = f
}

func (t *Tag) SetTextFrame(id string, text string) {
	var f frame.TextFramer

	existingFrame := t.frames[id]
	if existingFrame == nil {
		f = new(frame.TextFrame)
		f.SetID(id)
	} else {
		f = existingFrame.(frame.TextFramer)
	}

	f.SetText(text)
	t.frames[id] = f
}

func (t *Tag) SetTitle(title string) {
	t.SetTextFrame(t.commonIDs["Title"], title)
}

func (t *Tag) SetArtist(artist string) {
	t.SetTextFrame(t.commonIDs["Artist"], artist)
}

func (t *Tag) SetAlbum(album string) {
	t.SetTextFrame(t.commonIDs["Album"], album)
}

func (t *Tag) SetYear(year string) {
	t.SetTextFrame(t.commonIDs["Year"], year)
}

func (t *Tag) SetGenre(genre string) {
	t.SetTextFrame(t.commonIDs["Genre"], genre)
}

func NewTag(file *os.File) *Tag {
	header := &TagHeader{
		Version:    4,
		FramesSize: 0,
	}

	return &Tag{
		Header:    header,
		frames:    make(map[string]frame.Framer),
		commonIDs: frame.V24CommonIDs,

		File:         file,
		OriginalSize: 0,
	}
}

func ParseTag(file *os.File) (*Tag, error) {
	header, err := ParseHeader(file)
	if err != nil {
		err = errors.New("Trying to parse tag header: " + err.Error())
		return nil, err
	}
	if header == nil {
		return NewTag(file), nil
	}
	if header.Version < 3 {
		err = errors.New("Unsupported version of ID3 tag")
		return nil, err
	}

	tag := &Tag{
		Header:    header,
		frames:    make(map[string]frame.Framer),
		commonIDs: frame.V24CommonIDs,

		File:         file,
		OriginalSize: TagHeaderSize + header.FramesSize,
	}

	return tag, nil
}

func (t *Tag) Flush() error {
	// Creating a temp file for mp3 file, which will contain new tag
	newFile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	defer os.Remove(newFile.Name())
	newFileBuf := bufio.NewWriter(newFile)

	// Writing to new file new header
	tagHeader, err := FormTagHeader(*t.Header)
	if err != nil {
		return err
	}
	if _, err := newFileBuf.Write(tagHeader); err != nil {
		return err
	}

	// Writing to new file new frames
	// And getting size of new frames
	frames, err := t.FormFrames()
	if err != nil {
		return err
	}
	framesSize, err := newFileBuf.Write(frames)
	if err != nil {
		return err
	}

	// Getting a music part of mp3
	// (Discarding an original tag of mp3)
	f := t.File
	defer f.Close()
	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	originalFileBuf := bufio.NewReader(f)
	if _, err := originalFileBuf.Discard(int(t.OriginalSize)); err != nil {
		return err
	}

	// Writing to new file the music part
	if _, err = newFileBuf.ReadFrom(originalFileBuf); err != nil {
		return err
	}

	// Flushing the buffered to new file
	if err = newFileBuf.Flush(); err != nil {
		return err
	}

	// Setting size of frames to new file
	setSize(newFile, uint32(framesSize))

	// Replacing original file with new file
	os.Rename(newFile.Name(), f.Name())
	return nil
}

func setSize(f *os.File, size uint32) (err error) {
	if _, err := f.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	sizeBytes, err := util.FormSize(size)
	if err != nil {
		return
	}

	if _, err = f.WriteAt(sizeBytes, 6); err != nil {
		return
	}

	return
}

func (t Tag) FormFrames() ([]byte, error) {
	b := bytesBufPool.Get().(*bytes.Buffer)
	b.Reset()

	for _, f := range t.frames {
		formedFrame := f.Form()
		frameHeader, err := formFrameHeader(f, uint32(len(formedFrame)))
		if err != nil {
			return nil, err
		}
		b.Write(frameHeader)
		b.Write(formedFrame)
	}

	bytesBufPool.Put(b)
	return b.Bytes(), nil
}

func formFrameHeader(f frame.Framer, frameSize uint32) ([]byte, error) {
	b := bytesBufPool.Get().(*bytes.Buffer)
	b.Reset()

	b.WriteString(f.ID())
	if size, err := util.FormSize(frameSize); err != nil {
		return nil, err
	} else {
		b.Write(size)
	}
	b.WriteByte(0)
	b.WriteByte(0)

	bytesBufPool.Put(b)
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcpinfo

// An OptionKind represents an option kind.
type OptionKind int

const (
	KindMaxSegSize    OptionKind = 2
	KindWindowScale   OptionKind = 3
	KindSackPermitted OptionKind = 4
	KindTimestamps    OptionKind = 8
)

var optionKinds = map[OptionKind]string{
	KindMaxSegSize:    "mss",
	KindWindowScale:   "wscale",
	KindSackPermitted: "sack perm",
	KindTimestamps:    "tmstamps",
}

func (k OptionKind) String() string {
	s, ok := optionKinds[k]
	if !ok {
		return "<nil>"
	}
	return s
}

// An Option represents an option.
type Option interface {
	Kind() OptionKind
}

// A MaxSegSize represents a maxiumum sengment size option.
type MaxSegSize uint

// Kind returns an option kind field.
func (mss MaxSegSize) Kind() OptionKind { return KindMaxSegSize }

// A WindowScale represents a windows scale option.
type WindowScale int

// Kind returns an option kind field.
func (ws WindowScale) Kind() OptionKind { return KindWindowScale }

// A SackPermitted reports whether a selective acknowledgment
// permitted option is enabled.
type SackPermitted bool

// Kind returns an option kind field.
func (sp SackPermitted) Kind() OptionKind { return KindSackPermitted }

// A Timestamps reports whether a timestamps option is enabled.
type Timestamps bool

// Kind returns an option kind field.
func (ts Timestamps) Kind() OptionKind { return KindTimestamps }
<commit_msg>tcpinfo: rename SackFoo to SACKFoo<commit_after>// Copyright 2016 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tcpinfo

// An OptionKind represents an option kind.
type OptionKind int

const (
	KindMaxSegSize    OptionKind = 2
	KindWindowScale   OptionKind = 3
	KindSACKPermitted OptionKind = 4
	KindTimestamps    OptionKind = 8
)

var optionKinds = map[OptionKind]string{
	KindMaxSegSize:    "mss",
	KindWindowScale:   "wscale",
	KindSACKPermitted: "sack perm",
	KindTimestamps:    "tmstamps",
}

func (k OptionKind) String() string {
	s, ok := optionKinds[k]
	if !ok {
		return "<nil>"
	}
	return s
}

// An Option represents an option.
type Option interface {
	Kind() OptionKind
}

// A MaxSegSize represents a maxiumum sengment size option.
type MaxSegSize uint

// Kind returns an option kind field.
func (mss MaxSegSize) Kind() OptionKind { return KindMaxSegSize }

// A WindowScale represents a windows scale option.
type WindowScale int

// Kind returns an option kind field.
func (ws WindowScale) Kind() OptionKind { return KindWindowScale }

// A SACKPermitted reports whether a selective acknowledgment
// permitted option is enabled.
type SACKPermitted bool

// Kind returns an option kind field.
func (sp SACKPermitted) Kind() OptionKind { return KindSACKPermitted }

// A Timestamps reports whether a timestamps option is enabled.
type Timestamps bool

// Kind returns an option kind field.
func (ts Timestamps) Kind() OptionKind { return KindTimestamps }
<|endoftext|>


<commit_before>package main

import (
    "os"
    "bufio"
    "fmt"
    "io"
    "math"
    "regexp"
    "runtime"
    "runtime/pprof"
    "sort"
    "strconv"
    "strings"
    "sync"
)

const (
   useProfile = false
   // for Nginx($request_time)
   SCALE = 0
   EFFECTIVE_DIGIT = 3
   // for Apache(%D)
   // SCALE = -6
   // EFFECTIVE_DIGIT = 6
)

var (
    topCount = 10
    urlNormalizes = []string{
      "^GET /memo/[0-9]+$",
      "^GET /stylesheets/",
      "^GET /images/",
    }
)

type Measure struct {
  Url string
  Count int
  Total float64
  Mean float64
  Min float64
  P50 float64
  P90 float64
  P95 float64
  P99 float64
  Max float64
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
  ms := &measureSorter{
    measures: measures,
    by: by,
  }
  sort.Sort(ms)
}

type measureSorter struct {
  measures []*Measure
  by func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
  return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
  s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
  return s.by(s.measures[i], s.measures[j])
}

type Column struct {
  Name string
  Summary string
  Sort By
}

var (
    columns = []*Column{
      &Column{ Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count } },
      &Column{ Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total } },
      &Column{ Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean } },
      &Column{ Name: "Min", Summary: "Minimum(0 Percentile)", Sort: func(a, b *Measure) bool { return a.Min > b.Min } },
      &Column{ Name: "P50", Summary: "Median(50 Percentile)", Sort: func(a, b *Measure) bool { return a.P50 > b.P50 } },
      &Column{ Name: "P90", Summary: "90 Percentile", Sort: func(a, b *Measure) bool { return a.P90 > b.P90 } },
      &Column{ Name: "P95", Summary: "95 Percentile", Sort: func(a, b *Measure) bool { return a.P95 > b.P95 } },
      &Column{ Name: "P99", Summary: "99 Percentile", Sort: func(a, b *Measure) bool { return a.P99 > b.P99 } },
      &Column{ Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max } },
    }
)

type Time struct {
  Url string
  Time float64
}

func showMeasures(measures []*Measure) {
  countWidth := 5 // for title
  totalWidth := 2 + EFFECTIVE_DIGIT
  meanWidth := 2 + EFFECTIVE_DIGIT * 2
  maxWidth := 2 + EFFECTIVE_DIGIT

  for i := 0; i < topCount; i++ {
    if countWidth < int(math.Log10(float64(measures[i].Count)) + 1) {
      countWidth = int(math.Log10(float64(measures[i].Count)) + 1)
    }
    if totalWidth < int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1) {
      totalWidth = int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1)
    }
    if meanWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1) {
      meanWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT * 2 + 1)
    }
    if maxWidth < int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1) {
      maxWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1)
    }
  }

  var format string
  for _, column := range columns {
    switch column.Name {
    case "Count":
      fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
      format += fmt.Sprintf("%%%dd  ", countWidth)
    case "Total":
      fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", totalWidth, EFFECTIVE_DIGIT)
    case "Mean":
      fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", meanWidth, EFFECTIVE_DIGIT * 2)
    default:
      fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
      format += fmt.Sprintf("%%%d.%df  ", maxWidth, EFFECTIVE_DIGIT)
    }
  }
  fmt.Printf("Url/Regexp\n")
  format += "%s\n"

  for i := 0; i < topCount; i++ {
    m := measures[i]
    fmt.Printf(format, m.Count, m.Total, m.Mean, m.Min, m.P50, m.P90, m.P95, m.P99, m.Max, m.Url)
  }
}

func main() {
    runtime.GOMAXPROCS(runtime.NumCPU())
    if (useProfile) {
        f, err := os.Create("/tmp/parse_access_log.prof")
        if err != nil {
            panic(err)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }
    reader := bufio.NewReaderSize(os.Stdin, 4096)
    scale := math.Pow10(SCALE)

    var urlNormalizeRegexps []*regexp.Regexp
    for _, str := range urlNormalizes {
	re := regexp.MustCompile(str)
        urlNormalizeRegexps = append(urlNormalizeRegexps, re)
    }

    ch := make(chan *Time)
    totals := make(map[string]float64)
    times := make(map[string][]float64)
    go func() {
      for time := range ch {
        totals[time.Url] += time.Time
        times[time.Url] = append(times[time.Url], time.Time)
      }
    }()

    var wg sync.WaitGroup
    for {
        line, err := reader.ReadString('\n')
        if err == io.EOF {
          wg.Wait()
          close(ch)
          break
        } else if err != nil {
          panic(err)
        }
        wg.Add(1)
        go func(line string) {
          defer wg.Done()
          s := strings.Split(line, " ")
          if len(s) >= 7 {
            url := strings.TrimLeft(strings.Join(s[5:7], " "), "\"")
            for _, re := range urlNormalizeRegexps {
              if re.MatchString(url) {
                url = re.String()
                break
              }
            }
            time, err := strconv.ParseFloat(strings.Trim(s[len(s)-1], "\r\n"), 10)
            if err == nil {
              time = time * scale
            } else {
              time = 0.000
            }
            ch <- &Time{Url: url, Time: time}
          }
        }(line)
    }

    var measures []*Measure
    for url, total := range totals {
      sorted := times[url]
      sort.Float64s(sorted)
      count := len(sorted)
      measure := &Measure{
        Url: url,
        Count: count,
        Total: total,
        Mean: totals[url]/float64(count),
        Min: sorted[0],
        P50: sorted[int(count*50/100)],
        P90: sorted[int(count*90/100)],
        P95: sorted[int(count*95/100)],
        P99: sorted[int(count*99/100)],
        Max: sorted[count-1],
      }
      measures = append(measures, measure)
    }
    if len(measures) < topCount {
      topCount = len(measures)
    }

    for _, column := range columns {
      fmt.Printf("Sort By %s\n", column.Summary)
      By(column.Sort).Sort(measures)
      showMeasures(measures)
      fmt.Println()
    }
}
<commit_msg>gofmt<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"math"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
)

const (
	useProfile = false
	// for Nginx($request_time)
	SCALE           = 0
	EFFECTIVE_DIGIT = 3
	// for Apache(%D)
	// SCALE = -6
	// EFFECTIVE_DIGIT = 6
)

var (
	topCount      = 10
	urlNormalizes = []string{
		"^GET /memo/[0-9]+$",
		"^GET /stylesheets/",
		"^GET /images/",
	}
)

type Measure struct {
	Url   string
	Count int
	Total float64
	Mean  float64
	Min   float64
	P50   float64
	P90   float64
	P95   float64
	P99   float64
	Max   float64
}

type By func(a, b *Measure) bool

func (by By) Sort(measures []*Measure) {
	ms := &measureSorter{
		measures: measures,
		by:       by,
	}
	sort.Sort(ms)
}

type measureSorter struct {
	measures []*Measure
	by       func(a, b *Measure) bool
}

func (s *measureSorter) Len() int {
	return len(s.measures)
}

func (s *measureSorter) Swap(i, j int) {
	s.measures[i], s.measures[j] = s.measures[j], s.measures[i]
}

func (s *measureSorter) Less(i, j int) bool {
	return s.by(s.measures[i], s.measures[j])
}

type Column struct {
	Name    string
	Summary string
	Sort    By
}

var (
	columns = []*Column{
		&Column{Name: "Count", Summary: "Count", Sort: func(a, b *Measure) bool { return a.Count > b.Count }},
		&Column{Name: "Total", Summary: "Total", Sort: func(a, b *Measure) bool { return a.Total > b.Total }},
		&Column{Name: "Mean", Summary: "Mean", Sort: func(a, b *Measure) bool { return a.Mean > b.Mean }},
		&Column{Name: "Min", Summary: "Minimum(0 Percentile)", Sort: func(a, b *Measure) bool { return a.Min > b.Min }},
		&Column{Name: "P50", Summary: "Median(50 Percentile)", Sort: func(a, b *Measure) bool { return a.P50 > b.P50 }},
		&Column{Name: "P90", Summary: "90 Percentile", Sort: func(a, b *Measure) bool { return a.P90 > b.P90 }},
		&Column{Name: "P95", Summary: "95 Percentile", Sort: func(a, b *Measure) bool { return a.P95 > b.P95 }},
		&Column{Name: "P99", Summary: "99 Percentile", Sort: func(a, b *Measure) bool { return a.P99 > b.P99 }},
		&Column{Name: "Max", Summary: "Maximum(100 Percentile)", Sort: func(a, b *Measure) bool { return a.Max > b.Max }},
	}
)

type Time struct {
	Url  string
	Time float64
}

func showMeasures(measures []*Measure) {
	countWidth := 5 // for title
	totalWidth := 2 + EFFECTIVE_DIGIT
	meanWidth := 2 + EFFECTIVE_DIGIT*2
	maxWidth := 2 + EFFECTIVE_DIGIT

	for i := 0; i < topCount; i++ {
		if countWidth < int(math.Log10(float64(measures[i].Count))+1) {
			countWidth = int(math.Log10(float64(measures[i].Count)) + 1)
		}
		if totalWidth < int(math.Log10(measures[i].Total)+1+EFFECTIVE_DIGIT+1) {
			totalWidth = int(math.Log10(measures[i].Total) + 1 + EFFECTIVE_DIGIT + 1)
		}
		if meanWidth < int(math.Log10(measures[i].Max)+1+EFFECTIVE_DIGIT*2+1) {
			meanWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT*2 + 1)
		}
		if maxWidth < int(math.Log10(measures[i].Max)+1+EFFECTIVE_DIGIT+1) {
			maxWidth = int(math.Log10(measures[i].Max) + 1 + EFFECTIVE_DIGIT + 1)
		}
	}

	var format string
	for _, column := range columns {
		switch column.Name {
		case "Count":
			fmt.Printf(fmt.Sprintf("%%%ds  ", countWidth), column.Name)
			format += fmt.Sprintf("%%%dd  ", countWidth)
		case "Total":
			fmt.Printf(fmt.Sprintf("%%%ds  ", totalWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", totalWidth, EFFECTIVE_DIGIT)
		case "Mean":
			fmt.Printf(fmt.Sprintf("%%%ds  ", meanWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", meanWidth, EFFECTIVE_DIGIT*2)
		default:
			fmt.Printf(fmt.Sprintf("%%%ds  ", maxWidth), column.Name)
			format += fmt.Sprintf("%%%d.%df  ", maxWidth, EFFECTIVE_DIGIT)
		}
	}
	fmt.Printf("Url/Regexp\n")
	format += "%s\n"

	for i := 0; i < topCount; i++ {
		m := measures[i]
		fmt.Printf(format, m.Count, m.Total, m.Mean, m.Min, m.P50, m.P90, m.P95, m.P99, m.Max, m.Url)
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	if useProfile {
		f, err := os.Create("/tmp/parse_access_log.prof")
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	reader := bufio.NewReaderSize(os.Stdin, 4096)
	scale := math.Pow10(SCALE)

	var urlNormalizeRegexps []*regexp.Regexp
	for _, str := range urlNormalizes {
		re := regexp.MustCompile(str)
		urlNormalizeRegexps = append(urlNormalizeRegexps, re)
	}

	ch := make(chan *Time)
	totals := make(map[string]float64)
	times := make(map[string][]float64)
	go func() {
		for time := range ch {
			totals[time.Url] += time.Time
			times[time.Url] = append(times[time.Url], time.Time)
		}
	}()

	var wg sync.WaitGroup
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			wg.Wait()
			close(ch)
			break
		} else if err != nil {
			panic(err)
		}
		wg.Add(1)
		go func(line string) {
			defer wg.Done()
			s := strings.Split(line, " ")
			if len(s) >= 7 {
				url := strings.TrimLeft(strings.Join(s[5:7], " "), "\"")
				for _, re := range urlNormalizeRegexps {
					if re.MatchString(url) {
						url = re.String()
						break
					}
				}
				time, err := strconv.ParseFloat(strings.Trim(s[len(s)-1], "\r\n"), 10)
				if err == nil {
					time = time * scale
				} else {
					time = 0.000
				}
				ch <- &Time{Url: url, Time: time}
			}
		}(line)
	}

	var measures []*Measure
	for url, total := range totals {
		sorted := times[url]
		sort.Float64s(sorted)
		count := len(sorted)
		measure := &Measure{
			Url:   url,
			Count: count,
			Total: total,
			Mean:  totals[url] / float64(count),
			Min:   sorted[0],
			P50:   sorted[int(count*50/100)],
			P90:   sorted[int(count*90/100)],
			P95:   sorted[int(count*95/100)],
			P99:   sorted[int(count*99/100)],
			Max:   sorted[count-1],
		}
		measures = append(measures, measure)
	}
	if len(measures) < topCount {
		topCount = len(measures)
	}

	for _, column := range columns {
		fmt.Printf("Sort By %s\n", column.Summary)
		By(column.Sort).Sort(measures)
		showMeasures(measures)
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>package mdb

/*
#include <stdlib.h>
#include <lmdb.h>
*/
import "C"

import (
	"math"
	"runtime"
	"unsafe"
)

// DBIOpen Database Flags
const (
	REVERSEKEY = C.MDB_REVERSEKEY // use reverse string keys
	DUPSORT    = C.MDB_DUPSORT    // use sorted duplicates
	INTEGERKEY = C.MDB_INTEGERKEY // numeric keys in native byte order. The keys must all be of the same size.
	DUPFIXED   = C.MDB_DUPFIXED   // with DUPSORT, sorted dup items have fixed size
	INTEGERDUP = C.MDB_INTEGERDUP // with DUPSORT, dups are numeric in native byte order
	REVERSEDUP = C.MDB_REVERSEDUP // with DUPSORT, use reverse string dups
	CREATE     = C.MDB_CREATE     // create DB if not already existing
)

// put flags
const (
	NODUPDATA   = C.MDB_NODUPDATA
	NOOVERWRITE = C.MDB_NOOVERWRITE
	RESERVE     = C.MDB_RESERVE
	APPEND      = C.MDB_APPEND
	APPENDDUP   = C.MDB_APPENDDUP
)

// Txn is Opaque structure for a transaction handle.
// All database operations require a transaction handle.
// Transactions may be read-only or read-write.
type Txn struct {
	_txn *C.MDB_txn
}

func (env *Env) BeginTxn(parent *Txn, flags uint) (*Txn, error) {
	var _txn *C.MDB_txn
	var ptxn *C.MDB_txn
	if parent == nil {
		ptxn = nil
	} else {
		ptxn = parent._txn
	}
	if flags&RDONLY == 0 {
		runtime.LockOSThread()
	}
	ret := C.mdb_txn_begin(env._env, ptxn, C.uint(flags), &_txn)
	if ret != SUCCESS {
		runtime.UnlockOSThread()
		return nil, errno(ret)
	}
	return &Txn{_txn}, nil
}

func (txn *Txn) Commit() error {
	ret := C.mdb_txn_commit(txn._txn)
	runtime.UnlockOSThread()
    // The transaction handle is freed if there was no error
    if ret == C.MDB_SUCCESS {
        txn._txn = nil
    }
	return errno(ret)
}

func (txn *Txn) Abort() {
	if txn._txn == nil {
        return
    }
    C.mdb_txn_abort(txn._txn)
	runtime.UnlockOSThread()
    // The transaction handle is always freed.
	txn._txn = nil
}

func (txn *Txn) Reset() {
	C.mdb_txn_reset(txn._txn)
}

func (txn *Txn) Renew() error {
	ret := C.mdb_txn_renew(txn._txn)
	return errno(ret)
}

func (txn *Txn) DBIOpen(name *string, flags uint) (DBI, error) {
	var _dbi C.MDB_dbi
	var cname *C.char
	if name == nil {
		cname = nil
	} else {
		cname = C.CString(*name)
		defer C.free(unsafe.Pointer(cname))
	}
	ret := C.mdb_dbi_open(txn._txn, cname, C.uint(flags), &_dbi)
	if ret != SUCCESS {
		return DBI(math.NaN()), errno(ret)
	}
	return DBI(_dbi), nil
}

func (txn *Txn) Stat(dbi DBI) (*Stat, error) {
	var _stat C.MDB_stat
	ret := C.mdb_stat(txn._txn, C.MDB_dbi(dbi), &_stat)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(_stat.ms_psize),
		Depth:         uint(_stat.ms_depth),
		BranchPages:   uint64(_stat.ms_branch_pages),
		LeafPages:     uint64(_stat.ms_leaf_pages),
		OverflowPages: uint64(_stat.ms_overflow_pages),
		Entries:       uint64(_stat.ms_entries)}
	return &stat, nil
}

func (txn *Txn) Drop(dbi DBI, del int) error {
	ret := C.mdb_drop(txn._txn, C.MDB_dbi(dbi), C.int(del))
	return errno(ret)
}

func (txn *Txn) Get(dbi DBI, key []byte) ([]byte, error) {
	val, err := txn.GetVal(dbi, key)
	if err != nil {
		return nil, err
	}
	return val.Bytes(), nil
}

func (txn *Txn) GetVal(dbi DBI, key []byte) (Val, error) {
	ckey := Wrap(key)
	var cval Val
	ret := C.mdb_get(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval))
	return cval, errno(ret)
}

func (txn *Txn) Put(dbi DBI, key []byte, val []byte, flags uint) error {
	ckey := Wrap(key)
	cval := Wrap(val)
	ret := C.mdb_put(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval), C.uint(flags))
	return errno(ret)
}

func (txn *Txn) Del(dbi DBI, key, val []byte) error {
	ckey := Wrap(key)
	if val == nil {
		ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), nil)
		return errno(ret)
	}
	cval := Wrap(val)
	ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval))
	return errno(ret)
}

type Cursor struct {
	_cursor *C.MDB_cursor
}

func (txn *Txn) CursorOpen(dbi DBI) (*Cursor, error) {
	var _cursor *C.MDB_cursor
	ret := C.mdb_cursor_open(txn._txn, C.MDB_dbi(dbi), &_cursor)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	return &Cursor{_cursor}, nil
}

func (txn *Txn) CursorRenew(cursor *Cursor) error {
	ret := C.mdb_cursor_renew(txn._txn, cursor._cursor)
	return errno(ret)
}

/*
type CmpFunc func(a, b []byte) int

func (txn *Txn) SetCompare(dbi DBI, cmp CmpFunc) error {
    f := func(a, b *C.MDB_val) C.int {
        ga := C.GoBytes(a.mv_data, C.int(a.mv_size))
        gb := C.GoBytes(a.mv_data, C.int(a.mv_size))
        return C.int(cmp(ga, gb))
    }
    ret := C.mdb_set_compare(txn._txn, C.MDB_dbi(dbi), *unsafe.Pointer(&f))
    return errno(ret)
}
*/
// func (txn *Txn) SetDupSort(dbi DBI, comp *C.MDB_comp_func) error
// func (txn *Txn) SetRelFunc(dbi DBI, rel *C.MDB_rel_func) error
// func (txn *Txn) SetRelCtx(dbi DBI, void *) error
<commit_msg>Rip out use of runtime.(Un)lockOSThread().<commit_after>package mdb

/*
#include <stdlib.h>
#include <lmdb.h>
*/
import "C"

import (
	"math"
	"unsafe"
)

// DBIOpen Database Flags
const (
	REVERSEKEY = C.MDB_REVERSEKEY // use reverse string keys
	DUPSORT    = C.MDB_DUPSORT    // use sorted duplicates
	INTEGERKEY = C.MDB_INTEGERKEY // numeric keys in native byte order. The keys must all be of the same size.
	DUPFIXED   = C.MDB_DUPFIXED   // with DUPSORT, sorted dup items have fixed size
	INTEGERDUP = C.MDB_INTEGERDUP // with DUPSORT, dups are numeric in native byte order
	REVERSEDUP = C.MDB_REVERSEDUP // with DUPSORT, use reverse string dups
	CREATE     = C.MDB_CREATE     // create DB if not already existing
)

// put flags
const (
	NODUPDATA   = C.MDB_NODUPDATA
	NOOVERWRITE = C.MDB_NOOVERWRITE
	RESERVE     = C.MDB_RESERVE
	APPEND      = C.MDB_APPEND
	APPENDDUP   = C.MDB_APPENDDUP
)

// Txn is Opaque structure for a transaction handle.
// All database operations require a transaction handle.
// Transactions may be read-only or read-write.
type Txn struct {
	_txn *C.MDB_txn
}

func (env *Env) BeginTxn(parent *Txn, flags uint) (*Txn, error) {
	var _txn *C.MDB_txn
	var ptxn *C.MDB_txn
	if parent == nil {
		ptxn = nil
	} else {
		ptxn = parent._txn
	}
	ret := C.mdb_txn_begin(env._env, ptxn, C.uint(flags), &_txn)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	return &Txn{_txn}, nil
}

func (txn *Txn) Commit() error {
	ret := C.mdb_txn_commit(txn._txn)
    // The transaction handle is freed if there was no error
    if ret == C.MDB_SUCCESS {
        txn._txn = nil
    }
	return errno(ret)
}

func (txn *Txn) Abort() {
	if txn._txn == nil {
        return
    }
    C.mdb_txn_abort(txn._txn)
    // The transaction handle is always freed.
	txn._txn = nil
}

func (txn *Txn) Reset() {
	C.mdb_txn_reset(txn._txn)
}

func (txn *Txn) Renew() error {
	ret := C.mdb_txn_renew(txn._txn)
	return errno(ret)
}

func (txn *Txn) DBIOpen(name *string, flags uint) (DBI, error) {
	var _dbi C.MDB_dbi
	var cname *C.char
	if name == nil {
		cname = nil
	} else {
		cname = C.CString(*name)
		defer C.free(unsafe.Pointer(cname))
	}
	ret := C.mdb_dbi_open(txn._txn, cname, C.uint(flags), &_dbi)
	if ret != SUCCESS {
		return DBI(math.NaN()), errno(ret)
	}
	return DBI(_dbi), nil
}

func (txn *Txn) Stat(dbi DBI) (*Stat, error) {
	var _stat C.MDB_stat
	ret := C.mdb_stat(txn._txn, C.MDB_dbi(dbi), &_stat)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(_stat.ms_psize),
		Depth:         uint(_stat.ms_depth),
		BranchPages:   uint64(_stat.ms_branch_pages),
		LeafPages:     uint64(_stat.ms_leaf_pages),
		OverflowPages: uint64(_stat.ms_overflow_pages),
		Entries:       uint64(_stat.ms_entries)}
	return &stat, nil
}

func (txn *Txn) Drop(dbi DBI, del int) error {
	ret := C.mdb_drop(txn._txn, C.MDB_dbi(dbi), C.int(del))
	return errno(ret)
}

func (txn *Txn) Get(dbi DBI, key []byte) ([]byte, error) {
	val, err := txn.GetVal(dbi, key)
	if err != nil {
		return nil, err
	}
	return val.Bytes(), nil
}

func (txn *Txn) GetVal(dbi DBI, key []byte) (Val, error) {
	ckey := Wrap(key)
	var cval Val
	ret := C.mdb_get(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval))
	return cval, errno(ret)
}

func (txn *Txn) Put(dbi DBI, key []byte, val []byte, flags uint) error {
	ckey := Wrap(key)
	cval := Wrap(val)
	ret := C.mdb_put(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval), C.uint(flags))
	return errno(ret)
}

func (txn *Txn) Del(dbi DBI, key, val []byte) error {
	ckey := Wrap(key)
	if val == nil {
		ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), nil)
		return errno(ret)
	}
	cval := Wrap(val)
	ret := C.mdb_del(txn._txn, C.MDB_dbi(dbi), (*C.MDB_val)(&ckey), (*C.MDB_val)(&cval))
	return errno(ret)
}

type Cursor struct {
	_cursor *C.MDB_cursor
}

func (txn *Txn) CursorOpen(dbi DBI) (*Cursor, error) {
	var _cursor *C.MDB_cursor
	ret := C.mdb_cursor_open(txn._txn, C.MDB_dbi(dbi), &_cursor)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	return &Cursor{_cursor}, nil
}

func (txn *Txn) CursorRenew(cursor *Cursor) error {
	ret := C.mdb_cursor_renew(txn._txn, cursor._cursor)
	return errno(ret)
}

/*
type CmpFunc func(a, b []byte) int

func (txn *Txn) SetCompare(dbi DBI, cmp CmpFunc) error {
    f := func(a, b *C.MDB_val) C.int {
        ga := C.GoBytes(a.mv_data, C.int(a.mv_size))
        gb := C.GoBytes(a.mv_data, C.int(a.mv_size))
        return C.int(cmp(ga, gb))
    }
    ret := C.mdb_set_compare(txn._txn, C.MDB_dbi(dbi), *unsafe.Pointer(&f))
    return errno(ret)
}
*/
// func (txn *Txn) SetDupSort(dbi DBI, comp *C.MDB_comp_func) error
// func (txn *Txn) SetRelFunc(dbi DBI, rel *C.MDB_rel_func) error
// func (txn *Txn) SetRelCtx(dbi DBI, void *) error
<|endoftext|>


<commit_before>package db

import (
	"labix.org/v2/mgo"
	. "launchpad.net/gocheck"
	"reflect"
	"testing"
)

type hasUniqueIndexChecker struct{}

func (c *hasUniqueIndexChecker) Info() *CheckerInfo {
	return &CheckerInfo{Name: "HasUniqueField", Params: []string{"collection", "key"}}
}

func (c *hasUniqueIndexChecker) Check(params []interface{}, names []string) (bool, string) {
	collection, ok := params[0].(*mgo.Collection)
	if !ok {
		return false, "first parameter should be a mgo collection"
	}
	key, ok := params[1].([]string)
	if !ok {
		return false, "second parameter should be the key, as used for mgo index declaration (slice of strings)"
	}
	indexes, err := collection.Indexes()
	if err != nil {
		return false, "failed to get collection indexes: " + err.Error()
	}
	for _, index := range indexes {
		if reflect.DeepEqual(index.Key, key) {
			return index.Unique, ""
		}
	}
	return false, ""
}

var HasUniqueIndex Checker = &hasUniqueIndexChecker{}

func Test(t *testing.T) { TestingT(t) }

type S struct {
	storage *Storage
}

var _ = Suite(&S{})

func (s *S) SetUpSuite(c *C) {
	s.storage, _ = Open("127.0.0.1:27017", "tsuru_storage_test")
}

func (s *S) TearDownSuite(c *C) {
	defer s.storage.Close()
	s.storage.session.DB("tsuru").DropDatabase()
}

func (s *S) TestShouldProvideMethodToOpenAConnection(c *C) {
	storage, _ := Open("127.0.0.1:27017", "tsuru_storage_test")
	c.Assert(storage.session.Ping(), IsNil)
	storage.Close()
}

func (s *S) TestMethodCloseSholdCloseTheConnectionWithMongoDB(c *C) {
	defer func() {
		if r := recover(); r == nil {
			c.Errorf("Should close the connection, but did not!")
		}
	}()
	storage, _ := Open("127.0.0.1:27017", "tsuru_storage_test")
	storage.Close()
	storage.session.Ping()
}

func (s *S) TestShouldProvidePrivateMethodToGetACollection(c *C) {
	collection := s.storage.getCollection("users")
	c.Assert(collection.FullName, Equals, s.storage.dbname+".users")
}

func (s *S) TestShouldCacheCollection(c *C) {
	collection := s.storage.getCollection("users")
	c.Assert(collection, DeepEquals, s.storage.collections["users"])
}

func (s *S) TestMethodUsersShouldReturnUsersCollection(c *C) {
	users := s.storage.Users()
	usersc := s.storage.getCollection("users")
	c.Assert(users, DeepEquals, usersc)
}

func (s *S) TestMethodUserShouldReturnUsersCollectionWithUniqueIndexForEmail(c *C) {
	users := s.storage.Users()
	c.Assert(users, HasUniqueIndex, []string{"email"})
}

func (s *S) TestMethodAppsShouldReturnAppsCollection(c *C) {
	apps := s.storage.Apps()
	appsc := s.storage.getCollection("apps")
	c.Assert(apps, DeepEquals, appsc)
}

func (s *S) TestMethodAppsShouldReturnAppsCollectionWithUniqueIndexForName(c *C) {
	apps := s.storage.Apps()
	c.Assert(apps, HasUniqueIndex, []string{"name"})
}

func (s *S) TestMethodServicesShouldReturnServicesCollection(c *C) {
	services := s.storage.Services()
	servicesc := s.storage.getCollection("services")
	c.Assert(services, DeepEquals, servicesc)
}

func (s *S) TestMethodServicesShouldReturnServicesCollectionWithUniqueIndexForName(c *C) {
	services := s.storage.Services()
	c.Assert(services, HasUniqueIndex, []string{"name"})
}

func (s *S) TestMethodServiceInstancesShouldReturnServiceInstancesCollection(c *C) {
	serviceApps := s.storage.ServiceInstances()
	serviceAppsc := s.storage.getCollection("service_apps")
	c.Assert(serviceApps, DeepEquals, serviceAppsc)
}

func (s *S) TestMethodUnitsShouldReturnUnitsCollection(c *C) {
	units := s.storage.Units()
	unitsc := s.storage.getCollection("units")
	c.Assert(units, DeepEquals, unitsc)
}

func (s *S) TestMethodTeamsShouldReturnTeamsCollection(c *C) {
	teams := s.storage.Teams()
	teamsc := s.storage.getCollection("teams")
	c.Assert(teams, DeepEquals, teamsc)
}

func (s *S) TestMethodTeamsShouldReturnTeamsCollectionWithUniqueIndexForName(c *C) {
	teams := s.storage.Teams()
	c.Assert(teams, HasUniqueIndex, []string{"name"})
}
<commit_msg>db: removed broken test<commit_after>package db

import (
	"labix.org/v2/mgo"
	. "launchpad.net/gocheck"
	"reflect"
	"testing"
)

type hasUniqueIndexChecker struct{}

func (c *hasUniqueIndexChecker) Info() *CheckerInfo {
	return &CheckerInfo{Name: "HasUniqueField", Params: []string{"collection", "key"}}
}

func (c *hasUniqueIndexChecker) Check(params []interface{}, names []string) (bool, string) {
	collection, ok := params[0].(*mgo.Collection)
	if !ok {
		return false, "first parameter should be a mgo collection"
	}
	key, ok := params[1].([]string)
	if !ok {
		return false, "second parameter should be the key, as used for mgo index declaration (slice of strings)"
	}
	indexes, err := collection.Indexes()
	if err != nil {
		return false, "failed to get collection indexes: " + err.Error()
	}
	for _, index := range indexes {
		if reflect.DeepEqual(index.Key, key) {
			return index.Unique, ""
		}
	}
	return false, ""
}

var HasUniqueIndex Checker = &hasUniqueIndexChecker{}

func Test(t *testing.T) { TestingT(t) }

type S struct {
	storage *Storage
}

var _ = Suite(&S{})

func (s *S) SetUpSuite(c *C) {
	s.storage, _ = Open("127.0.0.1:27017", "tsuru_storage_test")
}

func (s *S) TearDownSuite(c *C) {
	defer s.storage.Close()
	s.storage.session.DB("tsuru_storage_test").DropDatabase()
}

func (s *S) TestShouldProvideMethodToOpenAConnection(c *C) {
	storage, _ := Open("127.0.0.1:27017", "tsuru_storage_test")
	c.Assert(storage.session.Ping(), IsNil)
	storage.Close()
}

func (s *S) TestMethodCloseSholdCloseTheConnectionWithMongoDB(c *C) {
	defer func() {
		if r := recover(); r == nil {
			c.Errorf("Should close the connection, but did not!")
		}
	}()
	storage, _ := Open("127.0.0.1:27017", "tsuru_storage_test")
	storage.Close()
	storage.session.Ping()
}

func (s *S) TestShouldProvidePrivateMethodToGetACollection(c *C) {
	collection := s.storage.getCollection("users")
	c.Assert(collection.FullName, Equals, s.storage.dbname+".users")
}

func (s *S) TestShouldCacheCollection(c *C) {
	collection := s.storage.getCollection("users")
	c.Assert(collection, DeepEquals, s.storage.collections["users"])
}

func (s *S) TestMethodUsersShouldReturnUsersCollection(c *C) {
	users := s.storage.Users()
	usersc := s.storage.getCollection("users")
	c.Assert(users, DeepEquals, usersc)
}

func (s *S) TestMethodUserShouldReturnUsersCollectionWithUniqueIndexForEmail(c *C) {
	users := s.storage.Users()
	c.Assert(users, HasUniqueIndex, []string{"email"})
}

func (s *S) TestMethodAppsShouldReturnAppsCollection(c *C) {
	apps := s.storage.Apps()
	appsc := s.storage.getCollection("apps")
	c.Assert(apps, DeepEquals, appsc)
}

func (s *S) TestMethodAppsShouldReturnAppsCollectionWithUniqueIndexForName(c *C) {
	apps := s.storage.Apps()
	c.Assert(apps, HasUniqueIndex, []string{"name"})
}

func (s *S) TestMethodServicesShouldReturnServicesCollection(c *C) {
	services := s.storage.Services()
	servicesc := s.storage.getCollection("services")
	c.Assert(services, DeepEquals, servicesc)
}

func (s *S) TestMethodServiceInstancesShouldReturnServiceInstancesCollection(c *C) {
	serviceApps := s.storage.ServiceInstances()
	serviceAppsc := s.storage.getCollection("service_apps")
	c.Assert(serviceApps, DeepEquals, serviceAppsc)
}

func (s *S) TestMethodUnitsShouldReturnUnitsCollection(c *C) {
	units := s.storage.Units()
	unitsc := s.storage.getCollection("units")
	c.Assert(units, DeepEquals, unitsc)
}

func (s *S) TestMethodTeamsShouldReturnTeamsCollection(c *C) {
	teams := s.storage.Teams()
	teamsc := s.storage.getCollection("teams")
	c.Assert(teams, DeepEquals, teamsc)
}

func (s *S) TestMethodTeamsShouldReturnTeamsCollectionWithUniqueIndexForName(c *C) {
	teams := s.storage.Teams()
	c.Assert(teams, HasUniqueIndex, []string{"name"})
}
<|endoftext|>


<commit_before>package dns

import (
	"context"
	"errors"
	"fmt"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	dns2 "github.com/miekg/dns"

	"github.com/datawire/dlib/dexec"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/datawire/dlib/dtime"
	"github.com/telepresenceio/telepresence/v2/pkg/vif"
)

var errResolveDNotConfigured = errors.New("resolved not configured")

func (s *Server) Worker(c context.Context, dev *vif.Device, configureDNS func(net.IP, *net.UDPAddr)) error {
	if runningInDocker() {
		// Don't bother with systemd-resolved when running in a docker container
		return s.runOverridingServer(dgroup.WithGoroutineName(c, "/docker"), dev)
	}

	err := s.tryResolveD(dgroup.WithGoroutineName(c, "/resolved"), dev, configureDNS)
	if err == errResolveDNotConfigured {
		err = nil
		if c.Err() == nil {
			dlog.Info(c, "Unable to use systemd-resolved, falling back to local server")
			err = s.runOverridingServer(dgroup.WithGoroutineName(c, "/legacy"), dev)
		}
	}
	return err
}

// shouldApplySearch returns true if search path should be applied
func (s *Server) shouldApplySearch(query string) bool {
	if len(s.search) == 0 {
		return false
	}

	if query == "localhost." {
		return false
	}

	// Don't apply search paths to the kubernetes zone
	if strings.HasSuffix(query, "."+s.clusterDomain) {
		return false
	}

	// Don't apply search paths if one is already there
	for _, s := range s.search {
		if strings.HasSuffix(query, s) {
			return false
		}
	}

	// Don't apply search path to namespaces or "svc".
	query = query[:len(query)-1]
	if lastDot := strings.LastIndexByte(query, '.'); lastDot >= 0 {
		tld := query[lastDot+1:]
		if _, ok := s.namespaces[tld]; ok || tld == "svc" {
			return false
		}
	}
	return true
}

// resolveInSearch is only used by the overriding resolver. It is needed because unlike other resolvers, this
// resolver does not hook into a DNS system that handles search paths prior to the arrival of the request.
//
// TODO: With the DNS lookups now being done in the cluster, there's only one reason left to have a search path,
// and that's the local-only intercepts which means that using search-paths really should be limited to that
// use-case.
func (s *Server) resolveInSearch(c context.Context, query string) ([]net.IP, error) {
	query = strings.ToLower(query)
	query = strings.TrimSuffix(query, tel2SubDomainDot)

	if !s.shouldDoClusterLookup(query) {
		return nil, nil
	}

	if s.shouldApplySearch(query) {
		for _, sp := range s.search {
			if ips, err := s.resolveInCluster(c, query+sp); err != nil || len(ips) > 0 {
				return ips, err
			}
		}
	}
	return s.resolveInCluster(c, query)
}

func (s *Server) runOverridingServer(c context.Context, dev *vif.Device) error {
	if s.config.LocalIp == nil {
		dat, err := os.ReadFile("/etc/resolv.conf")
		if err != nil {
			return err
		}
		for _, line := range strings.Split(string(dat), "\n") {
			if strings.HasPrefix(strings.TrimSpace(line), "nameserver") {
				fields := strings.Fields(line)
				ip := net.ParseIP(fields[1])
				if ip.To4() != nil {
					s.config.LocalIp = ip.To4()
					dlog.Infof(c, "Automatically set -dns=%s", net.IP(s.config.LocalIp))
					break
				}
			}

			// The search entry in /etc/resolv.conf is not intended for this resolver so
			// ensure that we just forward such queries without sending them to the cluster
			// by adding corresponding entries to excludeSuffixes
			if strings.HasPrefix(strings.TrimSpace(line), "search") {
				fields := strings.Fields(line)
				for _, field := range fields[1:] {
					s.config.ExcludeSuffixes = append(s.config.ExcludeSuffixes, "."+field)
				}
			}
		}
	}
	if s.config.LocalIp == nil {
		return errors.New("couldn't determine dns ip from /etc/resolv.conf")
	}

	listeners, err := s.dnsListeners(c)
	if err != nil {
		return err
	}
	dnsResolverAddr, err := splitToUDPAddr(listeners[0].LocalAddr())
	if err != nil {
		return err
	}
	dlog.Debugf(c, "Bootstrapping local DNS server on port %d", dnsResolverAddr.Port)

	// Create the connection later used for fallback. We need to create this before the firewall
	// rule because the rule must exclude the local address of this connection in order to
	// let it reach the original destination and not cause an endless loop.
	conn, err := dns2.Dial("udp", net.JoinHostPort(net.IP(s.config.LocalIp).String(), "53"))
	if err != nil {
		return err
	}
	defer func() {
		_ = conn.Close()
	}()

	serverStarted := make(chan struct{})
	serverDone := make(chan struct{})
	g := dgroup.NewGroup(c, dgroup.GroupConfig{})
	g.Go("Server", func(c context.Context) error {
		defer close(serverDone)
		// Server will close the listener, so no need to close it here.
		s.processSearchPaths(g, func(c context.Context, paths []string, _ *vif.Device) error {
			namespaces := make(map[string]struct{})
			search := make([]string, 0)
			for _, path := range paths {
				if strings.ContainsRune(path, '.') {
					search = append(search, path)
				} else if path != "" {
					namespaces[path] = struct{}{}
				}
			}
			s.domainsLock.Lock()
			s.namespaces = namespaces
			s.search = search
			s.domainsLock.Unlock()
			s.flushDNS()
			return nil
		}, dev)
		return s.Run(c, serverStarted, listeners, conn, s.resolveInSearch)
	})

	g.Go("NAT-redirect", func(c context.Context) error {
		select {
		case <-c.Done():
		case <-serverStarted:
			// Give DNS server time to start before rerouting NAT
			dtime.SleepWithContext(c, time.Millisecond)

			err := routeDNS(c, s.config.LocalIp, dnsResolverAddr, conn.LocalAddr().(*net.UDPAddr))
			if err != nil {
				return err
			}
			defer func() {
				c := context.Background()
				unrouteDNS(c)
				s.flushDNS()
			}()
			s.flushDNS()
			<-serverDone // Stay alive until DNS server is done
		}
		return nil
	})
	return g.Wait()
}

func (s *Server) dnsListeners(c context.Context) ([]net.PacketConn, error) {
	listener, err := newLocalUDPListener(c)
	if err != nil {
		return nil, err
	}
	listeners := []net.PacketConn{listener}
	if runningInDocker() {
		// Inside docker. Don't add docker bridge
		return listeners, nil
	}

	// This is the default docker bridge. We need to listen here because the nat logic we use to intercept
	// dns packets will divert the packet to the interface it originates from, which in the case of
	// containers is the docker bridge. Without this dns won't work from inside containers.
	output, err := dexec.CommandContext(c, "docker", "inspect", "bridge",
		"-f", "{{(index .IPAM.Config 0).Gateway}}").Output()
	if err != nil {
		dlog.Info(c, "not listening on docker bridge")
		return listeners, nil
	}

	localAddr, err := splitToUDPAddr(listener.LocalAddr())
	if err != nil {
		return nil, err
	}

	dockerGatewayIP := net.ParseIP(strings.TrimSpace(string(output)))
	if dockerGatewayIP == nil || dockerGatewayIP.Equal(localAddr.IP) {
		return listeners, nil
	}

	// Check that the dockerGatewayIP is registered as an interface on this machine. When running WSL2 on
	// a Windows box, the gateway is managed by Windows and never visible to the Linux host and hence
	// will not be affected by the nat logic. Also, any attempt to listen to it will fail.
	found := false
	ifAddrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, ifAddr := range ifAddrs {
		_, network, err := net.ParseCIDR(ifAddr.String())
		if err != nil {
			continue
		}
		if network.Contains(dockerGatewayIP) {
			found = true
			break
		}
	}

	if !found {
		dlog.Infof(c, "docker gateway %s is not visible as a network interface", dockerGatewayIP)
		return listeners, nil
	}

	for {
		extraAddr := &net.UDPAddr{IP: dockerGatewayIP, Port: localAddr.Port}
		ls, err := net.ListenPacket("udp", extraAddr.String())
		if err == nil {
			dlog.Infof(c, "listening to docker bridge at %s", dockerGatewayIP)
			return append(listeners, ls), nil
		}

		// the extraAddr was busy, try next available port
		for localAddr.Port++; localAddr.Port <= math.MaxUint16; localAddr.Port++ {
			if ls, err = net.ListenPacket("udp", localAddr.String()); err == nil {
				if localAddr, err = splitToUDPAddr(ls.LocalAddr()); err != nil {
					ls.Close()
					return nil, err
				}
				_ = listeners[0].Close()
				listeners = []net.PacketConn{ls}
				break
			}
		}
		if localAddr.Port > math.MaxUint16 {
			return nil, fmt.Errorf("unable to find a free port for both %s and %s", localAddr.IP, extraAddr.IP)
		}
	}
}

func runningInDocker() bool {
	_, err := os.Stat("/.dockerenv")
	return err == nil
}

// runNatTableCmd runs "iptables -t nat ..."
func runNatTableCmd(c context.Context, args ...string) error {
	// We specifically don't want to use the cancellation of 'ctx' here, because we don't ever
	// want to leave things in a half-cleaned-up state.
	return dexec.CommandContext(c, "iptables", append([]string{"-t", "nat"}, args...)...).Run()
}

const tpDNSChain = "TELEPRESENCE_DNS"

// routeDNS creates a new chain in the "nat" table with two rules in it. One rule ensures
// that all packets sent to the currently configured DNS service are rerouted to our local
// DNS service. Another rule ensures that when our local DNS service cannot resolve and
// uses a fallback, that fallback reaches the original DNS service.
func routeDNS(c context.Context, dnsIP net.IP, toAddr *net.UDPAddr, localDNS *net.UDPAddr) (err error) {
	// create the chain
	unrouteDNS(c)

	// Create the TELEPRESENCE_DNS chain
	if err = runNatTableCmd(c, "-N", tpDNSChain); err != nil {
		return err
	}

	// This rule prevents that any rules in this table applies to the localDNS address when
	// used as a source. I.e. we let the local DNS server reach the original DNS server
	if err = runNatTableCmd(c, "-A", tpDNSChain,
		"-p", "udp",
		"--source", localDNS.IP.String(),
		"--sport", strconv.Itoa(localDNS.Port),
		"-j", "RETURN",
	); err != nil {
		return err
	}

	// This rule redirects all packets intended for the DNS service to our local DNS service
	if err = runNatTableCmd(c, "-A", tpDNSChain,
		"-p", "udp",
		"--dest", dnsIP.String()+"/32",
		"--dport", "53",
		"-j", "DNAT",
		"--to-destination", toAddr.String(),
	); err != nil {
		return err
	}

	// Alter locally generated packets before routing
	return runNatTableCmd(c, "-I", "OUTPUT", "1", "-j", tpDNSChain)
}

// unrouteDNS removes the chain installed by routeDNS.
func unrouteDNS(c context.Context) {
	// The errors returned by these commands aren't of any interest besides logging. And they
	// are already logged since dexec is used.
	_ = runNatTableCmd(c, "-D", "OUTPUT", "-j", tpDNSChain)
	_ = runNatTableCmd(c, "-F", tpDNSChain)
	_ = runNatTableCmd(c, "-X", tpDNSChain)
}
<commit_msg>pr feedback<commit_after>package dns

import (
	"context"
	"errors"
	"fmt"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	dns2 "github.com/miekg/dns"

	"github.com/datawire/dlib/dexec"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/datawire/dlib/dtime"
	"github.com/telepresenceio/telepresence/v2/pkg/vif"
)

var errResolveDNotConfigured = errors.New("resolved not configured")

func (s *Server) Worker(c context.Context, dev *vif.Device, configureDNS func(net.IP, *net.UDPAddr)) error {
	if runningInDocker() {
		// Don't bother with systemd-resolved when running in a docker container
		return s.runOverridingServer(dgroup.WithGoroutineName(c, "/docker"), dev)
	}

	err := s.tryResolveD(dgroup.WithGoroutineName(c, "/resolved"), dev, configureDNS)
	if err == errResolveDNotConfigured {
		err = nil
		if c.Err() == nil {
			dlog.Info(c, "Unable to use systemd-resolved, falling back to local server")
			err = s.runOverridingServer(dgroup.WithGoroutineName(c, "/legacy"), dev)
		}
	}
	return err
}

// shouldApplySearch returns true if search path should be applied
func (s *Server) shouldApplySearch(query string) bool {
	if len(s.search) == 0 {
		return false
	}

	if query == "localhost." {
		return false
	}

	// Don't apply search paths to the kubernetes zone
	if strings.HasSuffix(query, "."+s.clusterDomain) {
		return false
	}

	// Don't apply search paths if one is already there
	for _, s := range s.search {
		if strings.HasSuffix(query, s) {
			return false
		}
	}

	// Don't apply search path to namespaces or "svc".
	query = query[:len(query)-1]
	if lastDot := strings.LastIndexByte(query, '.'); lastDot >= 0 {
		tld := query[lastDot+1:]
		if _, ok := s.namespaces[tld]; ok || tld == "svc" {
			return false
		}
	}
	return true
}

// resolveInSearch is only used by the overriding resolver. It is needed because unlike other resolvers, this
// resolver does not hook into a DNS system that handles search paths prior to the arrival of the request.
//
// TODO: With the DNS lookups now being done in the cluster, there's only one reason left to have a search path,
// and that's the local-only intercepts which means that using search-paths really should be limited to that
// use-case.
func (s *Server) resolveInSearch(c context.Context, query string) ([]net.IP, error) {
	query = strings.ToLower(query)
	query = strings.TrimSuffix(query, tel2SubDomainDot)

	if !s.shouldDoClusterLookup(query) {
		return nil, nil
	}

	if s.shouldApplySearch(query) {
		for _, sp := range s.search {
			if ips, err := s.resolveInCluster(c, query+sp); err != nil || len(ips) > 0 {
				return ips, err
			}
		}
	}
	return s.resolveInCluster(c, query)
}

func (s *Server) runOverridingServer(c context.Context, dev *vif.Device) error {
	if s.config.LocalIp == nil {
		dat, err := os.ReadFile("/etc/resolv.conf")
		if err != nil {
			return err
		}
		for _, line := range strings.Split(string(dat), "\n") {
			if strings.HasPrefix(strings.TrimSpace(line), "nameserver") {
				fields := strings.Fields(line)
				ip := net.ParseIP(fields[1])
				if ip.To4() != nil {
					s.config.LocalIp = ip.To4()
					dlog.Infof(c, "Automatically set -dns=%s", net.IP(s.config.LocalIp))
				}
			}

			// The search entry in /etc/resolv.conf is not intended for this resolver so
			// ensure that we just forward such queries without sending them to the cluster
			// by adding corresponding entries to excludeSuffixes
			if strings.HasPrefix(strings.TrimSpace(line), "search") {
				fields := strings.Fields(line)
				for _, field := range fields[1:] {
					s.config.ExcludeSuffixes = append(s.config.ExcludeSuffixes, "."+field)
				}
			}
		}
	}
	if s.config.LocalIp == nil {
		return errors.New("couldn't determine dns ip from /etc/resolv.conf")
	}

	listeners, err := s.dnsListeners(c)
	if err != nil {
		return err
	}
	dnsResolverAddr, err := splitToUDPAddr(listeners[0].LocalAddr())
	if err != nil {
		return err
	}
	dlog.Debugf(c, "Bootstrapping local DNS server on port %d", dnsResolverAddr.Port)

	// Create the connection later used for fallback. We need to create this before the firewall
	// rule because the rule must exclude the local address of this connection in order to
	// let it reach the original destination and not cause an endless loop.
	conn, err := dns2.Dial("udp", net.JoinHostPort(net.IP(s.config.LocalIp).String(), "53"))
	if err != nil {
		return err
	}
	defer func() {
		_ = conn.Close()
	}()

	serverStarted := make(chan struct{})
	serverDone := make(chan struct{})
	g := dgroup.NewGroup(c, dgroup.GroupConfig{})
	g.Go("Server", func(c context.Context) error {
		defer close(serverDone)
		// Server will close the listener, so no need to close it here.
		s.processSearchPaths(g, func(c context.Context, paths []string, _ *vif.Device) error {
			namespaces := make(map[string]struct{})
			search := make([]string, 0)
			for _, path := range paths {
				if strings.ContainsRune(path, '.') {
					search = append(search, path)
				} else if path != "" {
					namespaces[path] = struct{}{}
				}
			}
			s.domainsLock.Lock()
			s.namespaces = namespaces
			s.search = search
			s.domainsLock.Unlock()
			s.flushDNS()
			return nil
		}, dev)
		return s.Run(c, serverStarted, listeners, conn, s.resolveInSearch)
	})

	g.Go("NAT-redirect", func(c context.Context) error {
		select {
		case <-c.Done():
		case <-serverStarted:
			// Give DNS server time to start before rerouting NAT
			dtime.SleepWithContext(c, time.Millisecond)

			err := routeDNS(c, s.config.LocalIp, dnsResolverAddr, conn.LocalAddr().(*net.UDPAddr))
			if err != nil {
				return err
			}
			defer func() {
				c := context.Background()
				unrouteDNS(c)
				s.flushDNS()
			}()
			s.flushDNS()
			<-serverDone // Stay alive until DNS server is done
		}
		return nil
	})
	return g.Wait()
}

func (s *Server) dnsListeners(c context.Context) ([]net.PacketConn, error) {
	listener, err := newLocalUDPListener(c)
	if err != nil {
		return nil, err
	}
	listeners := []net.PacketConn{listener}
	if runningInDocker() {
		// Inside docker. Don't add docker bridge
		return listeners, nil
	}

	// This is the default docker bridge. We need to listen here because the nat logic we use to intercept
	// dns packets will divert the packet to the interface it originates from, which in the case of
	// containers is the docker bridge. Without this dns won't work from inside containers.
	output, err := dexec.CommandContext(c, "docker", "inspect", "bridge",
		"-f", "{{(index .IPAM.Config 0).Gateway}}").Output()
	if err != nil {
		dlog.Info(c, "not listening on docker bridge")
		return listeners, nil
	}

	localAddr, err := splitToUDPAddr(listener.LocalAddr())
	if err != nil {
		return nil, err
	}

	dockerGatewayIP := net.ParseIP(strings.TrimSpace(string(output)))
	if dockerGatewayIP == nil || dockerGatewayIP.Equal(localAddr.IP) {
		return listeners, nil
	}

	// Check that the dockerGatewayIP is registered as an interface on this machine. When running WSL2 on
	// a Windows box, the gateway is managed by Windows and never visible to the Linux host and hence
	// will not be affected by the nat logic. Also, any attempt to listen to it will fail.
	found := false
	ifAddrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, ifAddr := range ifAddrs {
		_, network, err := net.ParseCIDR(ifAddr.String())
		if err != nil {
			continue
		}
		if network.Contains(dockerGatewayIP) {
			found = true
			break
		}
	}

	if !found {
		dlog.Infof(c, "docker gateway %s is not visible as a network interface", dockerGatewayIP)
		return listeners, nil
	}

	for {
		extraAddr := &net.UDPAddr{IP: dockerGatewayIP, Port: localAddr.Port}
		ls, err := net.ListenPacket("udp", extraAddr.String())
		if err == nil {
			dlog.Infof(c, "listening to docker bridge at %s", dockerGatewayIP)
			return append(listeners, ls), nil
		}

		// the extraAddr was busy, try next available port
		for localAddr.Port++; localAddr.Port <= math.MaxUint16; localAddr.Port++ {
			if ls, err = net.ListenPacket("udp", localAddr.String()); err == nil {
				if localAddr, err = splitToUDPAddr(ls.LocalAddr()); err != nil {
					ls.Close()
					return nil, err
				}
				_ = listeners[0].Close()
				listeners = []net.PacketConn{ls}
				break
			}
		}
		if localAddr.Port > math.MaxUint16 {
			return nil, fmt.Errorf("unable to find a free port for both %s and %s", localAddr.IP, extraAddr.IP)
		}
	}
}

func runningInDocker() bool {
	_, err := os.Stat("/.dockerenv")
	return err == nil
}

// runNatTableCmd runs "iptables -t nat ..."
func runNatTableCmd(c context.Context, args ...string) error {
	// We specifically don't want to use the cancellation of 'ctx' here, because we don't ever
	// want to leave things in a half-cleaned-up state.
	return dexec.CommandContext(c, "iptables", append([]string{"-t", "nat"}, args...)...).Run()
}

const tpDNSChain = "TELEPRESENCE_DNS"

// routeDNS creates a new chain in the "nat" table with two rules in it. One rule ensures
// that all packets sent to the currently configured DNS service are rerouted to our local
// DNS service. Another rule ensures that when our local DNS service cannot resolve and
// uses a fallback, that fallback reaches the original DNS service.
func routeDNS(c context.Context, dnsIP net.IP, toAddr *net.UDPAddr, localDNS *net.UDPAddr) (err error) {
	// create the chain
	unrouteDNS(c)

	// Create the TELEPRESENCE_DNS chain
	if err = runNatTableCmd(c, "-N", tpDNSChain); err != nil {
		return err
	}

	// This rule prevents that any rules in this table applies to the localDNS address when
	// used as a source. I.e. we let the local DNS server reach the original DNS server
	if err = runNatTableCmd(c, "-A", tpDNSChain,
		"-p", "udp",
		"--source", localDNS.IP.String(),
		"--sport", strconv.Itoa(localDNS.Port),
		"-j", "RETURN",
	); err != nil {
		return err
	}

	// This rule redirects all packets intended for the DNS service to our local DNS service
	if err = runNatTableCmd(c, "-A", tpDNSChain,
		"-p", "udp",
		"--dest", dnsIP.String()+"/32",
		"--dport", "53",
		"-j", "DNAT",
		"--to-destination", toAddr.String(),
	); err != nil {
		return err
	}

	// Alter locally generated packets before routing
	return runNatTableCmd(c, "-I", "OUTPUT", "1", "-j", tpDNSChain)
}

// unrouteDNS removes the chain installed by routeDNS.
func unrouteDNS(c context.Context) {
	// The errors returned by these commands aren't of any interest besides logging. And they
	// are already logged since dexec is used.
	_ = runNatTableCmd(c, "-D", "OUTPUT", "-j", tpDNSChain)
	_ = runNatTableCmd(c, "-F", tpDNSChain)
	_ = runNatTableCmd(c, "-X", tpDNSChain)
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package container

import (
	"sync"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	cio "github.com/containerd/containerd/pkg/cri/io"
	"github.com/containerd/containerd/pkg/cri/store"
	"github.com/containerd/containerd/pkg/cri/store/label"
	"github.com/containerd/containerd/pkg/cri/store/stats"
	"github.com/containerd/containerd/pkg/cri/store/truncindex"

	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)

// Container contains all resources associated with the container. All methods to
// mutate the internal state are thread-safe.
type Container struct {
	// Metadata is the metadata of the container, it is **immutable** after created.
	Metadata
	// Status stores the status of the container.
	Status StatusStorage
	// Container is the containerd container client.
	Container containerd.Container
	// Container IO.
	// IO could only be nil when the container is in unknown state.
	IO *cio.ContainerIO
	// StopCh is used to propagate the stop information of the container.
	*store.StopCh
	// IsStopSignaledWithTimeout the default is 0, and it is set to 1 after sending
	// the signal once to avoid repeated sending of the signal.
	IsStopSignaledWithTimeout *uint32
	// Stats contains (mutable) stats for the container
	Stats *stats.ContainerStats
}

// Opts sets specific information to newly created Container.
type Opts func(*Container) error

// WithContainer adds the containerd Container to the internal data store.
func WithContainer(cntr containerd.Container) Opts {
	return func(c *Container) error {
		c.Container = cntr
		return nil
	}
}

// WithContainerIO adds IO into the container.
func WithContainerIO(io *cio.ContainerIO) Opts {
	return func(c *Container) error {
		c.IO = io
		return nil
	}
}

// WithStatus adds status to the container.
func WithStatus(status Status, root string) Opts {
	return func(c *Container) error {
		s, err := StoreStatus(root, c.ID, status)
		if err != nil {
			return err
		}
		c.Status = s
		if s.Get().State() == runtime.ContainerState_CONTAINER_EXITED {
			c.Stop()
		}
		return nil
	}
}

// NewContainer creates an internally used container type.
func NewContainer(metadata Metadata, opts ...Opts) (Container, error) {
	c := Container{
		Metadata:                  metadata,
		StopCh:                    store.NewStopCh(),
		IsStopSignaledWithTimeout: new(uint32),
	}
	for _, o := range opts {
		if err := o(&c); err != nil {
			return Container{}, err
		}
	}
	return c, nil
}

// Delete deletes checkpoint for the container.
func (c *Container) Delete() error {
	return c.Status.Delete()
}

// Store stores all Containers.
type Store struct {
	lock       sync.RWMutex
	containers map[string]Container
	idIndex    *truncindex.TruncIndex
	labels     *label.Store
}

// NewStore creates a container store.
func NewStore(labels *label.Store) *Store {
	return &Store{
		containers: make(map[string]Container),
		idIndex:    truncindex.NewTruncIndex([]string{}),
		labels:     labels,
	}
}

// Add a container into the store. Returns store.ErrAlreadyExist if the
// container already exists.
func (s *Store) Add(c Container) error {
	s.lock.Lock()
	defer s.lock.Unlock()
	if _, ok := s.containers[c.ID]; ok {
		return errdefs.ErrAlreadyExists
	}
	if err := s.labels.Reserve(c.ProcessLabel); err != nil {
		return err
	}
	if err := s.idIndex.Add(c.ID); err != nil {
		return err
	}
	s.containers[c.ID] = c
	return nil
}

// Get returns the container with specified id. Returns store.ErrNotExist
// if the container doesn't exist.
func (s *Store) Get(id string) (Container, error) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		if err == truncindex.ErrNotExist {
			err = errdefs.ErrNotFound
		}
		return Container{}, err
	}
	if c, ok := s.containers[id]; ok {
		return c, nil
	}
	return Container{}, errdefs.ErrNotFound
}

// List lists all containers.
func (s *Store) List() []Container {
	s.lock.RLock()
	defer s.lock.RUnlock()
	var containers []Container
	for _, c := range s.containers {
		containers = append(containers, c)
	}
	return containers
}

func (s *Store) UpdateContainerStats(id string, newContainerStats *stats.ContainerStats) error {
	s.lock.RLock()
	defer s.lock.RUnlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		if err == truncindex.ErrNotExist {
			err = errdefs.ErrNotFound
		}
		return err
	}

	if _, ok := s.containers[id]; !ok {
		return errdefs.ErrNotFound
	}

	c := s.containers[id]
	c.Stats = newContainerStats
	s.containers[id] = c
	return nil
}

// Delete deletes the container from store with specified id.
func (s *Store) Delete(id string) {
	s.lock.Lock()
	defer s.lock.Unlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		// Note: The idIndex.Delete and delete doesn't handle truncated index.
		// So we need to return if there are error.
		return
	}
	s.labels.Release(s.containers[id].ProcessLabel)
	s.idIndex.Delete(id) // nolint: errcheck
	delete(s.containers, id)
}
<commit_msg>use write lock when updating container stats<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package container

import (
	"sync"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	cio "github.com/containerd/containerd/pkg/cri/io"
	"github.com/containerd/containerd/pkg/cri/store"
	"github.com/containerd/containerd/pkg/cri/store/label"
	"github.com/containerd/containerd/pkg/cri/store/stats"
	"github.com/containerd/containerd/pkg/cri/store/truncindex"

	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)

// Container contains all resources associated with the container. All methods to
// mutate the internal state are thread-safe.
type Container struct {
	// Metadata is the metadata of the container, it is **immutable** after created.
	Metadata
	// Status stores the status of the container.
	Status StatusStorage
	// Container is the containerd container client.
	Container containerd.Container
	// Container IO.
	// IO could only be nil when the container is in unknown state.
	IO *cio.ContainerIO
	// StopCh is used to propagate the stop information of the container.
	*store.StopCh
	// IsStopSignaledWithTimeout the default is 0, and it is set to 1 after sending
	// the signal once to avoid repeated sending of the signal.
	IsStopSignaledWithTimeout *uint32
	// Stats contains (mutable) stats for the container
	Stats *stats.ContainerStats
}

// Opts sets specific information to newly created Container.
type Opts func(*Container) error

// WithContainer adds the containerd Container to the internal data store.
func WithContainer(cntr containerd.Container) Opts {
	return func(c *Container) error {
		c.Container = cntr
		return nil
	}
}

// WithContainerIO adds IO into the container.
func WithContainerIO(io *cio.ContainerIO) Opts {
	return func(c *Container) error {
		c.IO = io
		return nil
	}
}

// WithStatus adds status to the container.
func WithStatus(status Status, root string) Opts {
	return func(c *Container) error {
		s, err := StoreStatus(root, c.ID, status)
		if err != nil {
			return err
		}
		c.Status = s
		if s.Get().State() == runtime.ContainerState_CONTAINER_EXITED {
			c.Stop()
		}
		return nil
	}
}

// NewContainer creates an internally used container type.
func NewContainer(metadata Metadata, opts ...Opts) (Container, error) {
	c := Container{
		Metadata:                  metadata,
		StopCh:                    store.NewStopCh(),
		IsStopSignaledWithTimeout: new(uint32),
	}
	for _, o := range opts {
		if err := o(&c); err != nil {
			return Container{}, err
		}
	}
	return c, nil
}

// Delete deletes checkpoint for the container.
func (c *Container) Delete() error {
	return c.Status.Delete()
}

// Store stores all Containers.
type Store struct {
	lock       sync.RWMutex
	containers map[string]Container
	idIndex    *truncindex.TruncIndex
	labels     *label.Store
}

// NewStore creates a container store.
func NewStore(labels *label.Store) *Store {
	return &Store{
		containers: make(map[string]Container),
		idIndex:    truncindex.NewTruncIndex([]string{}),
		labels:     labels,
	}
}

// Add a container into the store. Returns store.ErrAlreadyExist if the
// container already exists.
func (s *Store) Add(c Container) error {
	s.lock.Lock()
	defer s.lock.Unlock()
	if _, ok := s.containers[c.ID]; ok {
		return errdefs.ErrAlreadyExists
	}
	if err := s.labels.Reserve(c.ProcessLabel); err != nil {
		return err
	}
	if err := s.idIndex.Add(c.ID); err != nil {
		return err
	}
	s.containers[c.ID] = c
	return nil
}

// Get returns the container with specified id. Returns store.ErrNotExist
// if the container doesn't exist.
func (s *Store) Get(id string) (Container, error) {
	s.lock.RLock()
	defer s.lock.RUnlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		if err == truncindex.ErrNotExist {
			err = errdefs.ErrNotFound
		}
		return Container{}, err
	}
	if c, ok := s.containers[id]; ok {
		return c, nil
	}
	return Container{}, errdefs.ErrNotFound
}

// List lists all containers.
func (s *Store) List() []Container {
	s.lock.RLock()
	defer s.lock.RUnlock()
	var containers []Container
	for _, c := range s.containers {
		containers = append(containers, c)
	}
	return containers
}

func (s *Store) UpdateContainerStats(id string, newContainerStats *stats.ContainerStats) error {
	s.lock.Lock()
	defer s.lock.Unlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		if err == truncindex.ErrNotExist {
			err = errdefs.ErrNotFound
		}
		return err
	}

	if _, ok := s.containers[id]; !ok {
		return errdefs.ErrNotFound
	}

	c := s.containers[id]
	c.Stats = newContainerStats
	s.containers[id] = c
	return nil
}

// Delete deletes the container from store with specified id.
func (s *Store) Delete(id string) {
	s.lock.Lock()
	defer s.lock.Unlock()
	id, err := s.idIndex.Get(id)
	if err != nil {
		// Note: The idIndex.Delete and delete doesn't handle truncated index.
		// So we need to return if there are error.
		return
	}
	s.labels.Release(s.containers[id].ProcessLabel)
	s.idIndex.Delete(id) // nolint: errcheck
	delete(s.containers, id)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-aggregator/pkg/registry/apiservice"
)

// rest implements a RESTStorage for API services against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against API services.
func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) *REST {
	strategy := apiservice.NewStrategy(scheme)
	store := &genericregistry.Store{
		Copier:      scheme,
		NewFunc:     func() runtime.Object { return &apiregistration.APIService{} },
		NewListFunc: func() runtime.Object { return &apiregistration.APIServiceList{} },
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*apiregistration.APIService).Name, nil
		},
		PredicateFunc:     apiservice.MatchAPIService,
		QualifiedResource: apiregistration.Resource("apiservices"),
		WatchCacheSize:    100,

		CreateStrategy: strategy,
		UpdateStrategy: strategy,
		DeleteStrategy: strategy,
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: apiservice.GetAttrs}
	if err := store.CompleteWithOptions(options); err != nil {
		panic(err) // TODO: Propagate error up
	}
	return &REST{store}
}
<commit_msg>Default ObjectNameFunc for all REST Stores<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/registry/generic"
	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
	"k8s.io/kube-aggregator/pkg/registry/apiservice"
)

// rest implements a RESTStorage for API services against etcd
type REST struct {
	*genericregistry.Store
}

// NewREST returns a RESTStorage object that will work against API services.
func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) *REST {
	strategy := apiservice.NewStrategy(scheme)
	store := &genericregistry.Store{
		Copier:            scheme,
		NewFunc:           func() runtime.Object { return &apiregistration.APIService{} },
		NewListFunc:       func() runtime.Object { return &apiregistration.APIServiceList{} },
		PredicateFunc:     apiservice.MatchAPIService,
		QualifiedResource: apiregistration.Resource("apiservices"),
		WatchCacheSize:    100,

		CreateStrategy: strategy,
		UpdateStrategy: strategy,
		DeleteStrategy: strategy,
	}
	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: apiservice.GetAttrs}
	if err := store.CompleteWithOptions(options); err != nil {
		panic(err) // TODO: Propagate error up
	}
	return &REST{store}
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/facette/facette/pkg/library"
	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/facette/facette/thirdparty/github.com/fatih/set"
)

func (server *Server) serveCollection(writer http.ResponseWriter, request *http.Request) {
	type tmpCollection struct {
		*library.Collection
		Parent string `json:"parent"`
	}

	if request.Method != "GET" && request.Method != "HEAD" && server.Config.ReadOnly {
		server.serveResponse(writer, serverResponse{mesgReadOnlyMode}, http.StatusForbidden)
		return
	}

	collectionID := strings.TrimPrefix(request.URL.Path, urlLibraryPath+"collections/")

	switch request.Method {
	case "DELETE":
		if collectionID == "" {
			server.serveResponse(writer, serverResponse{mesgMethodNotAllowed}, http.StatusMethodNotAllowed)
			return
		}

		err := server.Library.DeleteItem(collectionID, library.LibraryItemCollection)
		if os.IsNotExist(err) {
			server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
			return
		} else if err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
			return
		}

		server.serveResponse(writer, nil, http.StatusOK)

	case "GET", "HEAD":
		if collectionID == "" {
			server.serveCollectionList(writer, request)
			return
		}

		item, err := server.Library.GetItem(collectionID, library.LibraryItemCollection)
		if os.IsNotExist(err) {
			server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
			return
		} else if err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
			return
		}

		server.serveResponse(writer, item, http.StatusOK)

	case "POST", "PUT":
		if response, status := server.parseStoreRequest(writer, request, collectionID); status != http.StatusOK {
			server.serveResponse(writer, response, status)
			return
		}

		collectionTemp := &tmpCollection{
			Collection: &library.Collection{
				Item: library.Item{ID: collectionID},
			},
		}

		if request.Method == "POST" && request.FormValue("inherit") != "" {
			// Get collection from library
			item, err := server.Library.GetItem(request.FormValue("inherit"), library.LibraryItemCollection)
			if os.IsNotExist(err) {
				server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
				return
			} else if err != nil {
				logger.Log(logger.LevelError, "server", "%s", err)
				server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
				return
			}

			utils.Clone(item.(*library.Collection), collectionTemp.Collection)

			collectionTemp.Collection.ID = ""
			collectionTemp.Collection.Children = nil
		}

		collectionTemp.Collection.Modified = time.Now()

		// Parse input JSON for collection data
		body, _ := ioutil.ReadAll(request.Body)

		if err := json.Unmarshal(body, &collectionTemp); err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgResourceInvalid}, http.StatusBadRequest)
			return
		}

		// Update parent relation
		if item, _ := server.Library.GetItem(collectionTemp.Parent, library.LibraryItemCollection); item != nil {
			collection := item.(*library.Collection)

			// Register parent relation
			collectionTemp.Collection.Parent = collection
			collectionTemp.Collection.ParentID = collectionTemp.Collection.Parent.ID
			collection.Children = append(collection.Children, collectionTemp.Collection)
		} else {
			// Remove existing parent relation
			if item, _ := server.Library.GetItem(collectionTemp.Collection.ID,
				library.LibraryItemCollection); item != nil {
				collection := item.(*library.Collection)

				if collection.Parent != nil {
					for index, child := range collection.Parent.Children {
						if reflect.DeepEqual(child, collection) {
							collection.Parent.Children = append(collection.Parent.Children[:index],
								collection.Parent.Children[index+1:]...)
							break
						}
					}
				}
			}
		}

		// Keep current children list
		if item, _ := server.Library.GetItem(collectionTemp.Collection.ID, library.LibraryItemCollection); item != nil {
			collectionTemp.Collection.Children = item.(*library.Collection).Children
		}

		// Store collection data
		err := server.Library.StoreItem(collectionTemp.Collection, library.LibraryItemCollection)
		if response, status := server.parseError(writer, request, err); status != http.StatusOK {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, response, status)
			return
		}

		if request.Method == "POST" {
			writer.Header().Add("Location", strings.TrimRight(request.URL.Path, "/")+"/"+collectionTemp.Collection.ID)
			server.serveResponse(writer, nil, http.StatusCreated)
		} else {
			server.serveResponse(writer, nil, http.StatusOK)
		}

	default:
		server.serveResponse(writer, serverResponse{mesgMethodNotAllowed}, http.StatusMethodNotAllowed)
	}
}

func (server *Server) serveCollectionList(writer http.ResponseWriter, request *http.Request) {
	var (
		collection      *library.Collection
		collectionStack []*library.Collection
		items           CollectionListResponse
		offset, limit   int
	)

	if response, status := server.parseListRequest(writer, request, &offset, &limit); status != http.StatusOK {
		server.serveResponse(writer, response, status)
		return
	}

	// Check for item exclusion
	excludeSet := set.New(set.ThreadSafe)

	if request.FormValue("exclude") != "" {
		if item, err := server.Library.GetItem(request.FormValue("exclude"), library.LibraryItemCollection); err == nil {
			collectionStack = append(collectionStack, item.(*library.Collection))
		}

		for len(collectionStack) > 0 {
			collection, collectionStack = collectionStack[0], collectionStack[1:]
			excludeSet.Add(collection.ID)
			collectionStack = append(collectionStack, collection.Children...)
		}
	}

	// Fill collections list
	items = make(CollectionListResponse, 0)

	for _, collection := range server.Library.Collections {
		if request.FormValue("parent") != "" && (request.FormValue("parent") == "" &&
			collection.Parent != nil || request.FormValue("parent") != "" &&
			(collection.Parent == nil || collection.Parent.ID != request.FormValue("parent"))) {
			continue
		}

		if request.FormValue("filter") != "" && !utils.FilterMatch(request.FormValue("filter"), collection.Name) {
			continue
		}

		// Skip excluded items
		if excludeSet.Has(collection.ID) {
			continue
		}

		collectionItem := &CollectionResponse{
			ItemResponse: ItemResponse{
				ID:          collection.ID,
				Name:        collection.Name,
				Description: collection.Description,
				Modified:    collection.Modified.Format(time.RFC3339),
			},
			Options:     collection.Options,
			HasChildren: len(collection.Children) > 0,
		}

		if collection.Parent != nil {
			collectionItem.Parent = &collection.Parent.ID
		}

		items = append(items, collectionItem)
	}

	response := &listResponse{
		list:   items,
		offset: offset,
		limit:  limit,
	}

	server.applyResponseLimit(writer, request, response)

	server.serveResponse(writer, response.list, http.StatusOK)
}
<commit_msg>Add top-level collections retrieval in library API<commit_after>package server

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"reflect"
	"strings"
	"time"

	"github.com/facette/facette/pkg/library"
	"github.com/facette/facette/pkg/logger"
	"github.com/facette/facette/pkg/utils"
	"github.com/facette/facette/thirdparty/github.com/fatih/set"
)

func (server *Server) serveCollection(writer http.ResponseWriter, request *http.Request) {
	type tmpCollection struct {
		*library.Collection
		Parent string `json:"parent"`
	}

	if request.Method != "GET" && request.Method != "HEAD" && server.Config.ReadOnly {
		server.serveResponse(writer, serverResponse{mesgReadOnlyMode}, http.StatusForbidden)
		return
	}

	collectionID := strings.TrimPrefix(request.URL.Path, urlLibraryPath+"collections/")

	switch request.Method {
	case "DELETE":
		if collectionID == "" {
			server.serveResponse(writer, serverResponse{mesgMethodNotAllowed}, http.StatusMethodNotAllowed)
			return
		}

		err := server.Library.DeleteItem(collectionID, library.LibraryItemCollection)
		if os.IsNotExist(err) {
			server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
			return
		} else if err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
			return
		}

		server.serveResponse(writer, nil, http.StatusOK)

	case "GET", "HEAD":
		if collectionID == "" {
			server.serveCollectionList(writer, request)
			return
		}

		item, err := server.Library.GetItem(collectionID, library.LibraryItemCollection)
		if os.IsNotExist(err) {
			server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
			return
		} else if err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
			return
		}

		server.serveResponse(writer, item, http.StatusOK)

	case "POST", "PUT":
		if response, status := server.parseStoreRequest(writer, request, collectionID); status != http.StatusOK {
			server.serveResponse(writer, response, status)
			return
		}

		collectionTemp := &tmpCollection{
			Collection: &library.Collection{
				Item: library.Item{ID: collectionID},
			},
		}

		if request.Method == "POST" && request.FormValue("inherit") != "" {
			// Get collection from library
			item, err := server.Library.GetItem(request.FormValue("inherit"), library.LibraryItemCollection)
			if os.IsNotExist(err) {
				server.serveResponse(writer, serverResponse{mesgResourceNotFound}, http.StatusNotFound)
				return
			} else if err != nil {
				logger.Log(logger.LevelError, "server", "%s", err)
				server.serveResponse(writer, serverResponse{mesgUnhandledError}, http.StatusInternalServerError)
				return
			}

			utils.Clone(item.(*library.Collection), collectionTemp.Collection)

			collectionTemp.Collection.ID = ""
			collectionTemp.Collection.Children = nil
		}

		collectionTemp.Collection.Modified = time.Now()

		// Parse input JSON for collection data
		body, _ := ioutil.ReadAll(request.Body)

		if err := json.Unmarshal(body, &collectionTemp); err != nil {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, serverResponse{mesgResourceInvalid}, http.StatusBadRequest)
			return
		}

		// Update parent relation
		if item, _ := server.Library.GetItem(collectionTemp.Parent, library.LibraryItemCollection); item != nil {
			collection := item.(*library.Collection)

			// Register parent relation
			collectionTemp.Collection.Parent = collection
			collectionTemp.Collection.ParentID = collectionTemp.Collection.Parent.ID
			collection.Children = append(collection.Children, collectionTemp.Collection)
		} else {
			// Remove existing parent relation
			if item, _ := server.Library.GetItem(collectionTemp.Collection.ID,
				library.LibraryItemCollection); item != nil {
				collection := item.(*library.Collection)

				if collection.Parent != nil {
					for index, child := range collection.Parent.Children {
						if reflect.DeepEqual(child, collection) {
							collection.Parent.Children = append(collection.Parent.Children[:index],
								collection.Parent.Children[index+1:]...)
							break
						}
					}
				}
			}
		}

		// Keep current children list
		if item, _ := server.Library.GetItem(collectionTemp.Collection.ID, library.LibraryItemCollection); item != nil {
			collectionTemp.Collection.Children = item.(*library.Collection).Children
		}

		// Store collection data
		err := server.Library.StoreItem(collectionTemp.Collection, library.LibraryItemCollection)
		if response, status := server.parseError(writer, request, err); status != http.StatusOK {
			logger.Log(logger.LevelError, "server", "%s", err)
			server.serveResponse(writer, response, status)
			return
		}

		if request.Method == "POST" {
			writer.Header().Add("Location", strings.TrimRight(request.URL.Path, "/")+"/"+collectionTemp.Collection.ID)
			server.serveResponse(writer, nil, http.StatusCreated)
		} else {
			server.serveResponse(writer, nil, http.StatusOK)
		}

	default:
		server.serveResponse(writer, serverResponse{mesgMethodNotAllowed}, http.StatusMethodNotAllowed)
	}
}

func (server *Server) serveCollectionList(writer http.ResponseWriter, request *http.Request) {
	var (
		collection      *library.Collection
		collectionStack []*library.Collection
		items           CollectionListResponse
		offset, limit   int
	)

	if response, status := server.parseListRequest(writer, request, &offset, &limit); status != http.StatusOK {
		server.serveResponse(writer, response, status)
		return
	}

	// Check for item exclusion
	excludeSet := set.New(set.ThreadSafe)

	if request.FormValue("exclude") != "" {
		if item, err := server.Library.GetItem(request.FormValue("exclude"), library.LibraryItemCollection); err == nil {
			collectionStack = append(collectionStack, item.(*library.Collection))
		}

		for len(collectionStack) > 0 {
			collection, collectionStack = collectionStack[0], collectionStack[1:]
			excludeSet.Add(collection.ID)
			collectionStack = append(collectionStack, collection.Children...)
		}
	}

	// Fill collections list
	items = make(CollectionListResponse, 0)

	for _, collection := range server.Library.Collections {
		if request.FormValue("parent") == "null" && collection.Parent != nil || request.FormValue("parent") != "" &&
			request.FormValue("parent") != "null" && (collection.Parent == nil ||
			collection.Parent.ID != request.FormValue("parent")) {

			continue
		}

		if request.FormValue("filter") != "" && !utils.FilterMatch(request.FormValue("filter"), collection.Name) {
			continue
		}

		// Skip excluded items
		if excludeSet.Has(collection.ID) {
			continue
		}

		collectionItem := &CollectionResponse{
			ItemResponse: ItemResponse{
				ID:          collection.ID,
				Name:        collection.Name,
				Description: collection.Description,
				Modified:    collection.Modified.Format(time.RFC3339),
			},
			Options:     collection.Options,
			HasChildren: len(collection.Children) > 0,
		}

		if collection.Parent != nil {
			collectionItem.Parent = &collection.Parent.ID
		}

		items = append(items, collectionItem)
	}

	response := &listResponse{
		list:   items,
		offset: offset,
		limit:  limit,
	}

	server.applyResponseLimit(writer, request, response)

	server.serveResponse(writer, response.list, http.StatusOK)
}
<|endoftext|>


<commit_before>package accesscontrol

import (
	"errors"
	"time"
)

var (
	ErrRoleNotFound                = errors.New("role not found")
	ErrTeamRoleAlreadyAdded        = errors.New("role is already added to this team")
	ErrUserRoleAlreadyAdded        = errors.New("role is already added to this user")
	ErrTeamRoleNotFound            = errors.New("team role not found")
	ErrUserRoleNotFound            = errors.New("user role not found")
	ErrTeamNotFound                = errors.New("team not found")
	ErrPermissionNotFound          = errors.New("permission not found")
	ErrRoleFailedGenerateUniqueUID = errors.New("failed to generate role definition UID")
	ErrVersionLE                   = errors.New("the provided role version is smaller than or equal to stored role")
)

// Role is the model for Role in RBAC.
type Role struct {
	ID          int64  `json:"id" xorm:"pk autoincr 'id'"`
	OrgID       int64  `json:"orgId" xorm:"org_id"`
	Version     int64  `json:"version"`
	UID         string `xorm:"uid" json:"uid"`
	Name        string `json:"name"`
	Description string `json:"description"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

type RoleDTO struct {
	ID          int64        `json:"id" xorm:"pk autoincr 'id'"`
	OrgID       int64        `json:"orgId" xorm:"org_id"`
	Version     int64        `json:"version"`
	UID         string       `xorm:"uid" json:"uid"`
	Name        string       `json:"name"`
	Description string       `json:"description"`
	Permissions []Permission `json:"permissions,omitempty"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

// Permission is the model for Permission in RBAC.
type Permission struct {
	ID         int64  `json:"id" xorm:"pk autoincr 'id'"`
	RoleID     int64  `json:"-" xorm:"role_id"`
	Permission string `json:"permission"`
	Scope      string `json:"scope"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

type GetUserPermissionsQuery struct {
	OrgID  int64 `json:"-"`
	UserID int64 `json:"userId"`
	Roles  []string
}

type EvaluationResult struct {
	HasAccess bool
	Meta      interface{}
}

func (p RoleDTO) Role() Role {
	return Role{
		ID:          p.ID,
		OrgID:       p.OrgID,
		Name:        p.Name,
		Description: p.Description,
		Updated:     p.Updated,
		Created:     p.Created,
	}
}
<commit_msg>Chore: move errors to enterprise (#32753)<commit_after>package accesscontrol

import (
	"time"
)

// Role is the model for Role in RBAC.
type Role struct {
	ID          int64  `json:"id" xorm:"pk autoincr 'id'"`
	OrgID       int64  `json:"orgId" xorm:"org_id"`
	Version     int64  `json:"version"`
	UID         string `xorm:"uid" json:"uid"`
	Name        string `json:"name"`
	Description string `json:"description"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

type RoleDTO struct {
	ID          int64        `json:"id" xorm:"pk autoincr 'id'"`
	OrgID       int64        `json:"orgId" xorm:"org_id"`
	Version     int64        `json:"version"`
	UID         string       `xorm:"uid" json:"uid"`
	Name        string       `json:"name"`
	Description string       `json:"description"`
	Permissions []Permission `json:"permissions,omitempty"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

// Permission is the model for Permission in RBAC.
type Permission struct {
	ID         int64  `json:"id" xorm:"pk autoincr 'id'"`
	RoleID     int64  `json:"-" xorm:"role_id"`
	Permission string `json:"permission"`
	Scope      string `json:"scope"`

	Updated time.Time `json:"updated"`
	Created time.Time `json:"created"`
}

type GetUserPermissionsQuery struct {
	OrgID  int64 `json:"-"`
	UserID int64 `json:"userId"`
	Roles  []string
}

type EvaluationResult struct {
	HasAccess bool
	Meta      interface{}
}

func (p RoleDTO) Role() Role {
	return Role{
		ID:          p.ID,
		OrgID:       p.OrgID,
		Name:        p.Name,
		Description: p.Description,
		Updated:     p.Updated,
		Created:     p.Created,
	}
}
<|endoftext|>