<commit_before>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

type OptionStruct struct {
	NoKokushi   *bool
	NoChitoitsu *bool
	NoNormal    *bool
}

var option OptionStruct

func Start(o OptionStruct) {
	option = o

	start := time.Now().UnixNano()
	// 処理
	var i float64
	i = 0
	for {
		i++
		end := time.Now().UnixNano()
		diff := float64(end-start) / 1000000000
		m := i / diff
		out := 0
		if m >= 0 {
			out = int(m)
		}
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		if int(i)%10000 == 0 {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		}
		if ok {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
			break
		}
		//fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		//if i > 100000 {
		//	break
		//}
	}
	fmt.Printf("\n")
}

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := hand.Solve()
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

const HandSize = 14
const MahjongSetSize = 136

var defaultSet []int

func GetMahjongSet() []int {
	if defaultSet == nil {
		size := MahjongSetSize
		defaultSet = make([]int, size, size)
		for i := 0; i < size; i++ {
			defaultSet[i] = i / 4
		}
	}
	return defaultSet
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	hand := make(Hand, MahjongSetSize, MahjongSetSize)
	copy(hand, GetMahjongSet())
	hand2 := make(Hand, 0, 0)
	var j int

	for k := MahjongSetSize; k > MahjongSetSize-HandSize; k-- {
		j = rand.Intn(k)
		hand2 = append(hand2, hand[j])
		hand = append(hand[:j], hand[j+1:]...)
	}

	return hand2
}

type Hand []int

// 牌文字への変換(スペース区切り)
func (hand Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < HandSize; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(hand[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// 七対子判定
func (hand Hand) solveChitoitsu() bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}

// 国士無双判定
func (hand Hand) solveKokushi() bool {
	sort.Ints(hand)
	//比較するために配列にする
	var a [HandSize]int
	for i := 0; i < HandSize; i++ {
		a[i] = hand[i]
	}

	//あがりパターン列挙
	agaris := [13][14]int{
		[14]int{0, 0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33, 33},
	}
	for _, v := range agaris {
		if v == a {
			return true
		}
	}
	return false
}

// あがり判定する
func (hand Hand) Solve() bool {
	return (!*(option.NoKokushi) && hand.solveKokushi()) || (!*(option.NoChitoitsu) && hand.solveChitoitsu()) || (!*(option.NoNormal) && hand.GroupSuit().Solve())
}

type SuitGroup struct {
	innerSuitGroup
	color int
}

type innerSuitGroup []int

func NewSuitGroup(color int) *SuitGroup {
	s := SuitGroup{innerSuitGroup{}, color}
	return &s
}

type SuitsGroupedHand map[int]SuitGroup

const (
	Jihai = iota
	Manzu
	Sozu
	Pinzu
)

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{
		Jihai: *NewSuitGroup(Jihai),
		Manzu: *NewSuitGroup(Manzu),
		Sozu:  *NewSuitGroup(Sozu),
		Pinzu: *NewSuitGroup(Pinzu),
	}
	for _, i := range hand {
		quo := (i - 7 + 9) / 9
		var mod int
		if i-7 >= 0 {
			mod = (i - 7) % 9
		} else {
			mod = i
		}
		s := m[quo]
		s.append(mod)
		m[quo] = s
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	return m.a_pair_existible() && m.valid_33332()
}
func (m SuitsGroupedHand) a_pair_existible() bool {
	//スートのサイズを3で割った時
	//あまりが2であるスートグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a.list()) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !m[i].valid_suit_group(i) {
			return false
		}
	}
	return true
}

func (a innerSuitGroup) list() innerSuitGroup {
	return a
}

func (a *innerSuitGroup) SetSuitGroup(b innerSuitGroup) {
	*a = b
}

func (a *innerSuitGroup) append(w int) {
	*a = append(*a, w)
}

// 33332形を形成するスートグループがどうかを判定
func (a SuitGroup) valid_suit_group(i int) bool {
	// 対子が含まれているスートグループがただ1つある前提

	//ソート
	sort.Ints(a.list())
	if len(a.list())%3 == 2 {
		//ペアを探す
		pair_numbers := a.pairable_numbers()
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := NewSuitGroup(i)
			c := 2
			for _, w := range a.list() {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest.SetSuitGroup(append(rest.list(), w))
				}
				if w == v {
					c--
				}
			}
			if rest.valid_3cards() {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a.list())%3 == 0 {
		return a.valid_3cards()
	}
	// 到達しないはず
	panic("到達しないはず")
}

func (a SuitGroup) valid_3cards() bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	for {
		if a.remove_kotsu() {
			continue
		}
		break
	}

	// 字牌でなければ順子チェック
	if a.color != Jihai {
		for {
			if a.remove_shuntsu() {
				continue
			}
			break
		}
	}
	return len(a.list()) == 0
}

func (a *innerSuitGroup) remove_kotsu() bool {
	// 刻子を除去できればtrue
	// a is sorted
	x := *a
	if len(x) < 3 {
		return false
	}
	if x[0] == x[1] && x[0] == x[2] {
		*a = x[3:]
		return true
	}
	return false
}

func (a *innerSuitGroup) remove_shuntsu() bool {
	// 順子を除去できればtrue
	// a is sorted
	rest := innerSuitGroup{}
	first := -1
	second := -1
	found := false
	for _, v := range *a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	*a = rest
	return found
}

func (a innerSuitGroup) pairable_numbers() innerSuitGroup {
	// a is sorted
	counter := []int{}
	var x, y int // 2つ前と1つ前
	for _, v := range a {
		if y == v && x != v {
			counter = append(counter, v)
		} else {
			y = v
		}
	}
	return counter
}
<commit_msg>Remove a comment<commit_after>package tenho

import (
	"fmt"
	"math/rand"
	"sort"
	"time"
)

type OptionStruct struct {
	NoKokushi   *bool
	NoChitoitsu *bool
	NoNormal    *bool
}

var option OptionStruct

func Start(o OptionStruct) {
	option = o

	start := time.Now().UnixNano()
	var i float64
	i = 0
	for {
		i++
		end := time.Now().UnixNano()
		diff := float64(end-start) / 1000000000
		m := i / diff
		out := 0
		if m >= 0 {
			out = int(m)
		}
		seed := time.Now().UnixNano()
		var hai string
		var ok bool
		hai, ok = tryOnce(seed)
		if int(i)%10000 == 0 {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		}
		if ok {
			fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
			break
		}
		//fmt.Printf("\r%v回試行  %v秒経過 %v回/秒 %v", i, diff, out, hai)
		//if i > 100000 {
		//	break
		//}
	}
	fmt.Printf("\n")
}

func tryOnce(seed int64) (string, bool) {
	hand := ShuffledHand(seed)
	hai := hand.HaiString()
	ok := hand.Solve()
	return hai, ok
}

// http://d.hatena.ne.jp/hake/20150930/p1
func shuffle(hand Hand) {
	for i := len(hand); i > 1; i-- {
		j := rand.Intn(i) // 0 .. i-1 の乱数発生
		hand[i-1], hand[j] = hand[j], hand[i-1]
	}
}

const HandSize = 14
const MahjongSetSize = 136

var defaultSet []int

func GetMahjongSet() []int {
	if defaultSet == nil {
		size := MahjongSetSize
		defaultSet = make([]int, size, size)
		for i := 0; i < size; i++ {
			defaultSet[i] = i / 4
		}
	}
	return defaultSet
}

func ShuffledHand(seed int64) Hand {
	rand.Seed(seed)

	hand := make(Hand, MahjongSetSize, MahjongSetSize)
	copy(hand, GetMahjongSet())
	hand2 := make(Hand, 0, 0)
	var j int

	for k := MahjongSetSize; k > MahjongSetSize-HandSize; k-- {
		j = rand.Intn(k)
		hand2 = append(hand2, hand[j])
		hand = append(hand[:j], hand[j+1:]...)
	}

	return hand2
}

type Hand []int

// 牌文字への変換(スペース区切り)
func (hand Hand) HaiString() string {
	// http://qiita.com/ruiu/items/2bb83b29baeae2433a79
	// サイズ0、内部バッファの長さ69の[]byteの値を割り当てる
	b := make([]byte, 0, 70)

	// bに文字列を追加
	for j := 0; j < HandSize; j++ {
		// コードポイント上、普通の麻雀牌はU+1F000からの34個。
		// U+1F000 is 'MAHJONG TILE EAST WIND' ('東')
		// https://codepoints.net/U+1F000
		b = append(b, string(hand[j]+0x1F000)...) // appendするには...が必要
		// 自分のMacではスペース区切りでないとうまく表示されないためスペースを挿入する
		// U+0020 is 'SPACE'
		// https://codepoints.net/U+0020
		b = append(b, string(0x20)...) // appendするには...が必要
	}
	return string(b)
}

// 七対子判定
func (hand Hand) solveChitoitsu() bool {
	//カウンタ
	c := map[int]int{}

	for _, v := range hand {
		count, ok := c[v]
		if ok {
			if count == 1 {
				c[v] = 2
			} else {
				// c[v] == 2
				return false
			}
		} else {
			c[v] = 1
		}
		//8個チェック
		if len(c) >= 8 {
			return false
		}
	}
	return true
}

// 国士無双判定
func (hand Hand) solveKokushi() bool {
	sort.Ints(hand)
	//比較するために配列にする
	var a [HandSize]int
	for i := 0; i < HandSize; i++ {
		a[i] = hand[i]
	}

	//あがりパターン列挙
	agaris := [13][14]int{
		[14]int{0, 0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 4, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 5, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 6, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 7, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 15, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 16, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 24, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 25, 33},
		[14]int{0, 1, 2, 3, 4, 5, 6, 7, 15, 16, 24, 25, 33, 33},
	}
	for _, v := range agaris {
		if v == a {
			return true
		}
	}
	return false
}

// あがり判定する
func (hand Hand) Solve() bool {
	return (!*(option.NoKokushi) && hand.solveKokushi()) || (!*(option.NoChitoitsu) && hand.solveChitoitsu()) || (!*(option.NoNormal) && hand.GroupSuit().Solve())
}

type SuitGroup struct {
	innerSuitGroup
	color int
}

type innerSuitGroup []int

func NewSuitGroup(color int) *SuitGroup {
	s := SuitGroup{innerSuitGroup{}, color}
	return &s
}

type SuitsGroupedHand map[int]SuitGroup

const (
	Jihai = iota
	Manzu
	Sozu
	Pinzu
)

// スート分類
func (hand Hand) GroupSuit() SuitsGroupedHand {
	m := SuitsGroupedHand{
		Jihai: *NewSuitGroup(Jihai),
		Manzu: *NewSuitGroup(Manzu),
		Sozu:  *NewSuitGroup(Sozu),
		Pinzu: *NewSuitGroup(Pinzu),
	}
	for _, i := range hand {
		quo := (i - 7 + 9) / 9
		var mod int
		if i-7 >= 0 {
			mod = (i - 7) % 9
		} else {
			mod = i
		}
		s := m[quo]
		s.append(mod)
		m[quo] = s
	}
	return m
}

func (m SuitsGroupedHand) Solve() bool {
	return m.a_pair_existible() && m.valid_33332()
}
func (m SuitsGroupedHand) a_pair_existible() bool {
	//スートのサイズを3で割った時
	//あまりが2であるスートグループが1つであること
	c := 0
	for _, a := range m {
		switch len(a.list()) % 3 {
		case 0:
			// noop
		case 1:
			return false
		case 2:
			c++
		}
	}
	return c == 1
}

func (m SuitsGroupedHand) valid_33332() bool {
	for i := 0; i < 4; i++ {
		if !m[i].valid_suit_group(i) {
			return false
		}
	}
	return true
}

func (a innerSuitGroup) list() innerSuitGroup {
	return a
}

func (a *innerSuitGroup) SetSuitGroup(b innerSuitGroup) {
	*a = b
}

func (a *innerSuitGroup) append(w int) {
	*a = append(*a, w)
}

// 33332形を形成するスートグループがどうかを判定
func (a SuitGroup) valid_suit_group(i int) bool {
	// 対子が含まれているスートグループがただ1つある前提

	//ソート
	sort.Ints(a.list())
	if len(a.list())%3 == 2 {
		//ペアを探す
		pair_numbers := a.pairable_numbers()
		//ペア候補がなかったらぬける
		if len(pair_numbers) == 0 {
			return false
		}
		//ペア候補毎に繰り返し処理
		for _, v := range pair_numbers {
			//ペアとなる２枚を除去
			rest := NewSuitGroup(i)
			c := 2
			for _, w := range a.list() {
				// ペア候補以外は新スライスに入れる
				// ペア候補は３枚目以降は新スライスに入れる
				if w != v || c <= 0 {
					rest.SetSuitGroup(append(rest.list(), w))
				}
				if w == v {
					c--
				}
			}
			if rest.valid_3cards() {
				return true
			} else {
				continue
			}
		}
		return false
	} else if len(a.list())%3 == 0 {
		return a.valid_3cards()
	}
	// 到達しないはず
	panic("到達しないはず")
}

func (a SuitGroup) valid_3cards() bool {
	// 刻子や順子のみで構成されている場合true
	// a is sorted
	// a.size % 3 is0
	for {
		if a.remove_kotsu() {
			continue
		}
		break
	}

	// 字牌でなければ順子チェック
	if a.color != Jihai {
		for {
			if a.remove_shuntsu() {
				continue
			}
			break
		}
	}
	return len(a.list()) == 0
}

func (a *innerSuitGroup) remove_kotsu() bool {
	// 刻子を除去できればtrue
	// a is sorted
	x := *a
	if len(x) < 3 {
		return false
	}
	if x[0] == x[1] && x[0] == x[2] {
		*a = x[3:]
		return true
	}
	return false
}

func (a *innerSuitGroup) remove_shuntsu() bool {
	// 順子を除去できればtrue
	// a is sorted
	rest := innerSuitGroup{}
	first := -1
	second := -1
	found := false
	for _, v := range *a {
		if found {
			rest = append(rest, v)
			continue
		}
		if first == -1 {
			first = v
		} else if second == -1 && first+1 == v {
			second = v
		} else if second != -1 && first+2 == v {
			//flush
			first = -1
			second = -1
			found = true
		} else {
			rest = append(rest, v)
		}
	}
	*a = rest
	return found
}

func (a innerSuitGroup) pairable_numbers() innerSuitGroup {
	// a is sorted
	counter := []int{}
	var x, y int // 2つ前と1つ前
	for _, v := range a {
		if y == v && x != v {
			counter = append(counter, v)
		} else {
			y = v
		}
	}
	return counter
}
<|endoftext|>


<commit_before>package test

import (
	"bytes"
	"github.com/boltdb/bolt"
	"interfaces"
	"io"
	"io/ioutil"
	//"log"
	"os"
	"path/filepath"
	boltStore "store/boltdb"
	"sync"

	uuid "github.com/satori/go.uuid"
)

var (
	pool = sync.Pool{
		New: func() interface{} {
			return new(bytes.Buffer)
		},
	}
	mu sync.Mutex
)

func ImportWorkspace(db *bolt.DB, workspaceRoot string) error {

	buckets, err := ioutil.ReadDir(workspaceRoot)
	if err != nil {
		return err
	}
	for _, bucket := range buckets {
		if err = ImportBucket(db, workspaceRoot, bucket.Name()); err != nil {
			return err
		}
	}
	return nil
}

func ImportBucket(db *bolt.DB, workspaceRoot, bucketName string) (err error) {
	var (
		bucketPath = filepath.Join(workspaceRoot, bucketName)
	)

	if _, _, err = createOrGetBucket(db, bucketName); err != nil {
		return err
	}

	files, err := ioutil.ReadDir(bucketPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		if err = ImportFsVirtualFile(db, workspaceRoot, bucketName, file.Name()); err != nil {
			return err
		}
	}

	return nil
}

// ImportFsVirtualFile set folder items (file_name,config.json,meta.json,structural.json) as file in boltdb
// it return erro if file not exists en db
func ImportFsVirtualFile(db *bolt.DB, workspaceRoot, bucketName, fileName string) (err error) {

	// todo check file folder contains max 4 files

	var (
		fm   = boltStore.NewFileManager(db)
		file *interfaces.File

		buffer *bytes.Buffer = pool.Get().(*bytes.Buffer)
	)

	{
		// return buffer to pool buffer
		defer func() {
			pool.Put(buffer)
		}()
	}

	if file, _, err = createOrGetFile(db, bucketName, fileName); err != nil {
		return
	}

	{
		// empty file data
		// it need for deleting some data case
		// all values will be overwire from fs
		file.LuaScript = nil
		file.RawData = nil
		file.MetaData = make(map[string]interface{})
		file.StructuralData = make(map[string]interface{})
	}

	filePath := filepath.Join(workspaceRoot, bucketName, fileName)
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return
	}
	for _, dataFile := range files {
		buffer.Reset()
		f, err := os.OpenFile(filepath.Join(filePath, dataFile.Name()), os.O_RDONLY, FilesPermission)
		if err != nil {
			return err
		}

		// todo test file is no folder

		// put data in buffer here
		_, err = io.Copy(buffer, f)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}

		tused := detectUsedType(dataFile.Name())
		err = setDataToFile(file, tused, buffer.Bytes())
		if err != nil {
			return err
		}
	}

	err = fm.UpdateFileFrom(file, interfaces.DataFile)
	return
}

func ImportFsDataFile(db *bolt.DB, workspaceRoot, bucketName, fileName, dataName string) (err error) {

	mu.Lock()

	var (
		fileManager   = boltStore.NewFileManager(db)
		bucketManager = boltStore.NewBucketManager(db)

		filePath = filepath.Join(workspaceRoot, bucketName, fileName, dataName)
		has      bool
		used     interfaces.DataUsed
		file     *interfaces.File
		bucket   *interfaces.Bucket

		buffer *bytes.Buffer = pool.Get().(*bytes.Buffer)
	)

	defer func() {
		mu.Unlock()
		pool.Put(buffer)
	}()

	// copy file data to buffer
	{
		// empty buffer
		buffer.Reset()

		f, err := os.OpenFile(filePath, os.O_RDONLY, FilesPermission)
		if err != nil {
			return err
		}

		// todo test file is no folder

		// put data in buffer here
		_, err = io.Copy(buffer, f)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
	}

	if file, err = fileManager.FindFileByName(bucketName, fileName, interfaces.FullFile); err != nil && err != interfaces.ErrNotFound {
		return
	} else if err == interfaces.ErrNotFound {
		has = false
		bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket)
		if err != nil {
			return err
		}
	} else {
		has = true
	}

	if file == nil {
		file = interfaces.NewFile()
	}

	// detect content type
	{
		used = detectUsedType(dataName)
		err = setDataToFile(file, used, buffer.Bytes())
		if err != nil {
			return err
		}
	}

	// put data to database
	if has {
		err = fileManager.UpdateFileFrom(file, used)
	} else {
		file.FileID = uuid.NewV4()
		file.BucketID = bucket.BucketID
		file.FileName = fileName
		//file.LuaScript = []byte{}
		//file.ContentType = "text/plain"

		err = fileManager.CreateFile(file)
	}
	if err != nil {
		return err
	}

	return nil
}

func createOrGetBucket(db *bolt.DB, bucketName string) (bucket *interfaces.Bucket, isNew bool, err error) {
	var (
		bucketManager = boltStore.NewBucketManager(db)
	)

	if bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket); err != nil {
		if err != interfaces.ErrNotFound {
			return
		}
	} else {
		return
	}

	isNew = true

	bucket.BucketID = uuid.NewV4()
	bucket.BucketName = bucketName

	err = bucketManager.CreateBucket(bucket)
	return
}

func createOrGetFile(db *bolt.DB, bucketName, fileName string) (file *interfaces.File, isNew bool, err error) {

	var (
		fileManager   = boltStore.NewFileManager(db)
		bucketManager = boltStore.NewBucketManager(db)

		bucket *interfaces.Bucket
	)

	if file, err = fileManager.FindFileByName(bucketName, fileName, interfaces.FullFile); err != nil && err != interfaces.ErrNotFound {
		return
	} else if err == interfaces.ErrNotFound {
		isNew = true
		bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket)
		if err != nil {
			return
		}
	} else {
		isNew = false
		return
	}

	if file == nil {
		file = interfaces.NewFile()
	}
	file.FileID = uuid.NewV4()
	file.BucketID = bucket.BucketID
	file.FileName = fileName

	err = fileManager.CreateFile(file)
	return
}
<commit_msg>add zip importer (without tests)<commit_after>package test

import (
	"archive/zip"
	"bytes"
	"github.com/boltdb/bolt"
	"interfaces"
	"io"
	"io/ioutil"
	"strings"
	//"log"
	"os"
	"path/filepath"
	boltStore "store/boltdb"
	"sync"

	uuid "github.com/satori/go.uuid"
)

var (
	pool = sync.Pool{
		New: func() interface{} {
			return new(bytes.Buffer)
		},
	}
	mu sync.Mutex
)

func ImportWorkspace(db *bolt.DB, workspaceRoot string) error {

	buckets, err := ioutil.ReadDir(workspaceRoot)
	if err != nil {
		return err
	}
	for _, bucket := range buckets {
		if err = ImportBucket(db, workspaceRoot, bucket.Name()); err != nil {
			return err
		}
	}
	return nil
}

func ImportWorkspaceZip(db *bolt.DB, workspaceRoot string) error {

	var (
		buffer *bytes.Buffer = pool.Get().(*bytes.Buffer)

		buckets = make(map[string]*interfaces.Bucket)
	)

	r, err := zip.OpenReader(workspaceRoot)
	if err != nil {
		return err
	}

	defer func() {
		r.Close()
		pool.Put(buffer)
	}()

	for _, f := range r.File {
		buffer.Reset()

		arr := strings.SplitN(f.Name, os.PathSeparator, 3)
		if len(arr) != 3 {
			fmt.Printf("Skip file %s:\n", f.Name)
			continue
		}

		var (
			bucketName string = arr[0]
			fileName   string = arr[1]
			dataName   string = arr[2]
		)

		// todo. create other way to kreatin buckets
		if bucket, ok := buckets[bucketName]; !ok {
			bucket, err = createOrGetBucket(db, bucketName)
			if err != nil {
				return err
			}
			buckets[bucket.BucketName] = bucket
		}

		rc, err := f.Open()
		if err != nil {
			return err
		}
		_, err = io.Copy(buffer, rc)
		if err != nil {
			return err
		}
		rc.Close()

		err = importFsDataFileData(db, bucketName, fileName, dataName, buffer.Bytes())
		if err != nil {
			return err
		}
	}
}

func ImportBucket(db *bolt.DB, workspaceRoot, bucketName string) (err error) {
	var (
		bucketPath = filepath.Join(workspaceRoot, bucketName)
	)

	if _, _, err = createOrGetBucket(db, bucketName); err != nil {
		return err
	}

	files, err := ioutil.ReadDir(bucketPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		if err = ImportFsVirtualFile(db, workspaceRoot, bucketName, file.Name()); err != nil {
			return err
		}
	}

	return nil
}

// ImportFsVirtualFile set folder items (file_name,config.json,meta.json,structural.json) as file in boltdb
// it return erro if file not exists en db
func ImportFsVirtualFile(db *bolt.DB, workspaceRoot, bucketName, fileName string) (err error) {

	// todo check file folder contains max 4 files

	var (
		fm   = boltStore.NewFileManager(db)
		file *interfaces.File

		buffer *bytes.Buffer = pool.Get().(*bytes.Buffer)
	)

	{
		// return buffer to pool buffer
		defer func() {
			pool.Put(buffer)
		}()
	}

	if file, _, err = createOrGetFile(db, bucketName, fileName); err != nil {
		return
	}

	{
		// empty file data
		// it need for deleting some data case
		// all values will be overwire from fs
		file.LuaScript = nil
		file.RawData = nil
		file.MetaData = make(map[string]interface{})
		file.StructuralData = make(map[string]interface{})
	}

	filePath := filepath.Join(workspaceRoot, bucketName, fileName)
	files, err := ioutil.ReadDir(filePath)
	if err != nil {
		return
	}
	for _, dataFile := range files {
		buffer.Reset()
		f, err := os.OpenFile(filepath.Join(filePath, dataFile.Name()), os.O_RDONLY, FilesPermission)
		if err != nil {
			return err
		}

		// todo test file is no folder

		// put data in buffer here
		_, err = io.Copy(buffer, f)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}

		tused := detectUsedType(dataFile.Name())
		err = setDataToFile(file, tused, buffer.Bytes())
		if err != nil {
			return err
		}
	}

	err = fm.UpdateFileFrom(file, interfaces.DataFile)
	return
}

func ImportFsDataFile(db *bolt.DB, workspaceRoot, bucketName, fileName, dataName string) (err error) {

	var (
		filePath = filepath.Join(workspaceRoot, bucketName, fileName, dataName)

		buffer *bytes.Buffer = pool.Get().(*bytes.Buffer)
	)

	defer func() {
		pool.Put(buffer)
	}()

	// copy file data to buffer
	{
		// empty buffer
		buffer.Reset()

		f, err := os.OpenFile(filePath, os.O_RDONLY, FilesPermission)
		if err != nil {
			return err
		}

		// todo test file is no folder

		// put data in buffer here
		_, err = io.Copy(buffer, f)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
	}

	return importFsDataFileData(db, bucketName, fileName, dataName, buffer.Bytes())
}

func importFsDataFileData(db *bolt.DB, bucketName, fileName, dataName string, data []byte) (err error) {

	mu.Lock()

	var (
		fileManager   = boltStore.NewFileManager(db)
		bucketManager = boltStore.NewBucketManager(db)

		filePath = filepath.Join(workspaceRoot, bucketName, fileName, dataName)
		has      bool
		used     interfaces.DataUsed
		file     *interfaces.File
		bucket   *interfaces.Bucket
	)

	defer func() {
		mu.Unlock()
	}()

	if file, err = fileManager.FindFileByName(bucketName, fileName, interfaces.FullFile); err != nil && err != interfaces.ErrNotFound {
		return
	} else if err == interfaces.ErrNotFound {
		has = false
		bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket)
		if err != nil {
			return err
		}
	} else {
		has = true
	}

	if file == nil {
		file = interfaces.NewFile()
	}

	// detect content type
	{
		used = detectUsedType(dataName)
		err = setDataToFile(file, used, data)
		if err != nil {
			return err
		}
	}

	// put data to database
	if has {
		err = fileManager.UpdateFileFrom(file, used)
	} else {
		file.FileID = uuid.NewV4()
		file.BucketID = bucket.BucketID
		file.FileName = fileName
		//file.LuaScript = []byte{}
		//file.ContentType = "text/plain"

		err = fileManager.CreateFile(file)
	}
	if err != nil {
		return err
	}

	return nil
}
func createOrGetBucket(db *bolt.DB, bucketName string) (bucket *interfaces.Bucket, isNew bool, err error) {
	var (
		bucketManager = boltStore.NewBucketManager(db)
	)

	if bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket); err != nil {
		if err != interfaces.ErrNotFound {
			return
		}
	} else {
		return
	}

	isNew = true

	bucket.BucketID = uuid.NewV4()
	bucket.BucketName = bucketName

	err = bucketManager.CreateBucket(bucket)
	return
}

func createOrGetFile(db *bolt.DB, bucketName, fileName string) (file *interfaces.File, isNew bool, err error) {

	var (
		fileManager   = boltStore.NewFileManager(db)
		bucketManager = boltStore.NewBucketManager(db)

		bucket *interfaces.Bucket
	)

	if file, err = fileManager.FindFileByName(bucketName, fileName, interfaces.FullFile); err != nil && err != interfaces.ErrNotFound {
		return
	} else if err == interfaces.ErrNotFound {
		isNew = true
		bucket, err = bucketManager.FindBucketByName(bucketName, interfaces.FullBucket)
		if err != nil {
			return
		}
	} else {
		isNew = false
		return
	}

	if file == nil {
		file = interfaces.NewFile()
	}
	file.FileID = uuid.NewV4()
	file.BucketID = bucket.BucketID
	file.FileName = fileName

	err = fileManager.CreateFile(file)
	return
}
<|endoftext|>


<commit_before>package relay

import (
	"net"

	pb "github.com/libp2p/go-libp2p-circuit/pb"

	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var _ manet.Listener = (*RelayListener)(nil)

type RelayListener Relay

func (l *RelayListener) Relay() *Relay {
	return (*Relay)(l)
}

func (r *Relay) Listener() *RelayListener {
	// TODO: Only allow one!
	return (*RelayListener)(r)
}

func (l *RelayListener) Accept() (manet.Conn, error) {
	select {
	case c := <-l.incoming:
		err := l.Relay().writeResponse(c.stream, pb.CircuitRelay_SUCCESS)
		if err != nil {
			log.Debugf("error writing relay response: %s", err.Error())
			c.stream.Reset()
			return nil, err
		}

		// TODO: Pretty print.
		log.Infof("accepted relay connection: %q", c)

		c.tagHop()
		return c, nil
	case <-l.ctx.Done():
		return nil, l.ctx.Err()
	}
}

func (l *RelayListener) Addr() net.Addr {
	return &NetAddr{
		Relay:  "any",
		Remote: "any",
	}
}

func (l *RelayListener) Multiaddr() ma.Multiaddr {
	return circuitAddr
}

func (l *RelayListener) Close() error {
	// TODO: noop?
	return nil
}
<commit_msg>fix: don't abort accept when accepting a single connection fails<commit_after>package relay

import (
	"net"

	pb "github.com/libp2p/go-libp2p-circuit/pb"

	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var _ manet.Listener = (*RelayListener)(nil)

type RelayListener Relay

func (l *RelayListener) Relay() *Relay {
	return (*Relay)(l)
}

func (r *Relay) Listener() *RelayListener {
	// TODO: Only allow one!
	return (*RelayListener)(r)
}

func (l *RelayListener) Accept() (manet.Conn, error) {
	for {
		select {
		case c := <-l.incoming:
			err := l.Relay().writeResponse(c.stream, pb.CircuitRelay_SUCCESS)
			if err != nil {
				log.Debugf("error writing relay response: %s", err.Error())
				c.stream.Reset()
				continue
			}

			// TODO: Pretty print.
			log.Infof("accepted relay connection: %q", c)

			c.tagHop()
			return c, nil
		case <-l.ctx.Done():
			return nil, l.ctx.Err()
		}
	}
}

func (l *RelayListener) Addr() net.Addr {
	return &NetAddr{
		Relay:  "any",
		Remote: "any",
	}
}

func (l *RelayListener) Multiaddr() ma.Multiaddr {
	return circuitAddr
}

func (l *RelayListener) Close() error {
	// TODO: noop?
	return nil
}
<|endoftext|>


<commit_before>package idx

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"unsafe"

	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

//go:generate msgp

type mType uint8

// MType represented by a uint8
const (
	MTypeUndefined mType = iota
	MTypeGauge
	MTypeRate
	MTypeCount
	MTypeCounter
	MTypeTimestamp
)

// MetricName stores the name as a []uintptr to strings interned in an object store.
// Each word is stored as a separate interned string without the '.'. The whole name
// can be retrieved by calling MetricName.String()
type MetricName struct {
	nodes []uintptr
}

// Nodes returns the []uintptr of interned string addresses
// for the MetricName
func (mn *MetricName) Nodes() []uintptr {
	return mn.nodes
}

// String returns the full MetricName as a string
// using data interned in the object store
func (mn *MetricName) String() string {
	if len(mn.nodes) == 0 {
		return ""
	}

	bld := strings.Builder{}
	return mn.string(&bld)
}

func (mn *MetricName) string(bld *strings.Builder) string {
	// get []int of the lengths of all of the mn.Nodes
	lns, ok := IdxIntern.LenNoCprsn(mn.nodes)
	if !ok {
		// this should never happen, do what now?
		return ""
	}

	// should be faster than calling IdxIntern.SetStringNoCprsn in a tight loop
	var tmpSz string
	szHeader := (*reflect.StringHeader)(unsafe.Pointer(&tmpSz))
	first, _ := IdxIntern.ObjString(mn.nodes[0])
	bld.WriteString(first)
	for idx, nodePtr := range mn.nodes[1:] {
		szHeader.Data = nodePtr
		szHeader.Len = lns[idx+1]
		bld.WriteString(".")
		bld.WriteString(tmpSz)
	}

	return bld.String()
}

// setMetricName interns the MetricName in an
// object store and stores the addresses of those strings
// in MetricName.nodes
func (mn *MetricName) setMetricName(name string) {
	nodes := strings.Split(name, ".")
	mn.nodes = make([]uintptr, len(nodes))
	for i, node := range nodes {
		// TODO: add error checking? Fail somehow
		nodePtr, err := IdxIntern.AddOrGet([]byte(node))
		if err != nil {
			log.Error(err)
		}
		mn.nodes[i] = nodePtr
	}
}

// ExtensionType is required to use custom marshaling as an extension
// with msgp
func (mn *MetricName) ExtensionType() int8 {
	return 95
}

// Len is required to use custom marshaling as an extension
// with msgp
func (mn *MetricName) Len() int {
	return len(mn.String())
}

// MarshalBinaryTo is required to use custom marshaling as an extension
// in msgp
func (mn *MetricName) MarshalBinaryTo(b []byte) error {
	copy(b, []byte(mn.String()))
	return nil
}

// UnmarshalBinary is required to use custom marshaling as an extension
// in msgp
func (mn *MetricName) UnmarshalBinary(b []byte) error {
	mn.setMetricName(string(b))
	return nil
}

// TagKeyValue stores a Key/Value pair. The strings
// are interned in an object store before they are assigned.
type TagKeyValue struct {
	Key   string
	Value string
}

// String returns a Key/Value pair in the form of
// 'key=value'
func (t *TagKeyValue) String() string {
	bld := strings.Builder{}

	bld.WriteString(t.Key)
	bld.WriteString("=")
	bld.WriteString(t.Value)

	return bld.String()
}

// TagKeyValues stores a slice of all of the Tag Key/Value pair combinations for a MetricDefinition
type TagKeyValues []TagKeyValue

// Strings returns a slice containing all of the Tag Key/Value pair combinations for a MetricDefinition.
// Each item in the slice is in the form of 'key=value'
func (t TagKeyValues) Strings() []string {
	tags := make([]string, len(t))
	for i, tag := range t {
		tags[i] = tag.String()
	}
	return tags
}

// Helper functions to sort TagKeyValues
func (t TagKeyValues) Len() int           { return len(t) }
func (t TagKeyValues) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t TagKeyValues) Less(i, j int) bool { return t[i].Key < t[j].Key }

// MetricDefinition stores information which identifies a single metric
type MetricDefinition struct {
	Id    schema.MKey
	OrgId uint32
	// using custom marshalling for MetricName
	// if there is another way we should explore that
	Name       MetricName `msg:"name,extension"`
	Interval   int
	Unit       string
	mtype      mType
	Tags       TagKeyValues
	LastUpdate int64
	Partition  int32
}

// NameWithTags returns a string version of the MetricDefinition's name with
// all of its tagsin the form of 'name;key1=value1;key2=value2;key3=value3'
func (md *MetricDefinition) NameWithTags() string {
	bld := strings.Builder{}

	md.Name.string(&bld)
	sort.Sort(TagKeyValues(md.Tags))
	for _, tag := range md.Tags {
		if tag.Key == "name" {
			continue
		}
		bld.WriteString(";")
		bld.WriteString(tag.String())
	}
	return bld.String()
}

// SetMType translates a string into a uint8 which is used to store
// the actual metric type. Valid values are 'gauge', 'rate', 'count',
// 'counter', and 'timestamp'.
func (md *MetricDefinition) SetMType(mtype string) {
	switch mtype {
	case "gauge":
		md.mtype = MTypeGauge
	case "rate":
		md.mtype = MTypeRate
	case "count":
		md.mtype = MTypeCount
	case "counter":
		md.mtype = MTypeCounter
	case "timestamp":
		md.mtype = MTypeTimestamp
	default:
		// for values "" and other unknown/corrupted values
		md.mtype = MTypeUndefined
	}
}

// Mtype returns a string version of the current MType
func (md *MetricDefinition) Mtype() string {
	switch md.mtype {
	case MTypeGauge:
		return "gauge"
	case MTypeRate:
		return "rate"
	case MTypeCount:
		return "count"
	case MTypeCounter:
		return "counter"
	case MTypeTimestamp:
		return "timestamp"
	default:
		// case of MTypeUndefined and also default for unknown/corrupted values
		return ""
	}
}

// SetUnit takes a string, interns it in an object store
// and then uses it to store the unit.
func (md *MetricDefinition) SetUnit(unit string) {
	sz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(unit))
	if err != nil {
		log.Errorf("idx: Failed to intern Unit %v. %v", unit, err)
		md.Unit = unit
	}
	md.Unit = sz
}

// SetMetricName interns the MetricName in an
// object store and stores the addresses of those strings
// in MetricName.nodes
func (md *MetricDefinition) SetMetricName(name string) {
	nodes := strings.Split(name, ".")
	md.Name.nodes = make([]uintptr, len(nodes))
	for i, node := range nodes {
		// TODO: add error checking? Fail somehow
		nodePtr, err := IdxIntern.AddOrGet([]byte(node))
		if err != nil {
			log.Errorf("idx: Failed to intern word in MetricName: %v, %v", node, err)
		}
		md.Name.nodes[i] = nodePtr
	}
}

// SetTags takes a []string which should contain Key/Value pairs
// in the form of 'key=value'. It splits up the Key and Value for each
// item, interns them in the object store, and creates a TagKeyValue
// for them. It then stores all of these in Tags.
//
// The items in the input argument should not contain ';'. Each item
// is a separate Key/Value pair. Do not combine multiple Key/Value pairs
// into a single index in the []string.
func (md *MetricDefinition) SetTags(tags []string) {
	md.Tags = make([]TagKeyValue, len(tags))
	sort.Strings(tags)
	for i, tag := range tags {
		if strings.Contains(tag, ";") {
			invalidTag.Inc()
			log.Errorf("idx: Tag %q has an invalid format, ignoring", tag)
			continue
		}
		splits := strings.Split(tag, "=")
		if len(splits) < 2 {
			invalidTag.Inc()
			log.Errorf("idx: Tag %q has an invalid format, ignoring", tag)
			continue
		}
		keySz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(splits[0]))
		if err != nil {
			log.Errorf("idx: Failed to intern tag %q, %v", tag, err)
			keyTmpSz := splits[0]
			md.Tags[i].Key = keyTmpSz
		} else {
			md.Tags[i].Key = keySz
		}

		valueSz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(splits[1]))
		if err != nil {
			log.Errorf("idx: Failed to intern tag %q, %v", tag, err)
			valueTmpSz := splits[1]
			md.Tags[i].Value = valueTmpSz
		} else {
			md.Tags[i].Value = valueSz
		}
	}
}

// SetId creates and sets the MKey which identifies a metric
func (md *MetricDefinition) SetId() {
	sort.Sort(TagKeyValues(md.Tags))
	buffer := bytes.NewBufferString(md.Name.String())
	buffer.WriteByte(0)
	buffer.WriteString(md.Unit)
	buffer.WriteByte(0)
	buffer.WriteString(md.Mtype())
	buffer.WriteByte(0)
	fmt.Fprintf(buffer, "%d", md.Interval)

	for _, t := range md.Tags {
		if t.Key == "name" {
			continue
		}

		buffer.WriteByte(0)
		buffer.WriteString(t.String())
	}

	md.Id = schema.MKey{
		Key: md5.Sum(buffer.Bytes()),
		Org: uint32(md.OrgId),
	}
}

// MetricDefinitionFromMetricDataWithMkey takes an MKey and MetricData and returns a MetricDefinition
// based on them.
func MetricDefinitionFromMetricDataWithMkey(mkey schema.MKey, d *schema.MetricData) *MetricDefinition {
	md := &MetricDefinition{
		Id:         mkey,
		OrgId:      uint32(d.OrgId),
		Interval:   d.Interval,
		LastUpdate: d.Time,
	}

	md.SetMetricName(d.Name)
	md.SetMType(d.Mtype)
	md.SetTags(d.Tags)
	md.SetUnit(d.Unit)

	return md
}

// MetricDefinitionFromMetricData takes a MetricData, attempts to generate an MKey for it,
// and returns a MetricDefinition upon success. On failure it returns an error
func MetricDefinitionFromMetricData(d *schema.MetricData) (*MetricDefinition, error) {
	mkey, err := schema.MKeyFromString(d.Id)
	if err != nil {
		return nil, fmt.Errorf("idx: Error parsing ID: %s", err)
	}

	return MetricDefinitionFromMetricDataWithMkey(mkey, d), nil
}
<commit_msg>honor field order<commit_after>package idx

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"unsafe"

	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

//go:generate msgp

type mType uint8

// MType represented by a uint8
const (
	MTypeUndefined mType = iota
	MTypeGauge
	MTypeRate
	MTypeCount
	MTypeCounter
	MTypeTimestamp
)

// MetricName stores the name as a []uintptr to strings interned in an object store.
// Each word is stored as a separate interned string without the '.'. The whole name
// can be retrieved by calling MetricName.String()
type MetricName struct {
	nodes []uintptr
}

// Nodes returns the []uintptr of interned string addresses
// for the MetricName
func (mn *MetricName) Nodes() []uintptr {
	return mn.nodes
}

// String returns the full MetricName as a string
// using data interned in the object store
func (mn *MetricName) String() string {
	if len(mn.nodes) == 0 {
		return ""
	}

	bld := strings.Builder{}
	return mn.string(&bld)
}

func (mn *MetricName) string(bld *strings.Builder) string {
	// get []int of the lengths of all of the mn.Nodes
	lns, ok := IdxIntern.LenNoCprsn(mn.nodes)
	if !ok {
		// this should never happen, do what now?
		return ""
	}

	// should be faster than calling IdxIntern.SetStringNoCprsn in a tight loop
	var tmpSz string
	szHeader := (*reflect.StringHeader)(unsafe.Pointer(&tmpSz))
	first, _ := IdxIntern.ObjString(mn.nodes[0])
	bld.WriteString(first)
	for idx, nodePtr := range mn.nodes[1:] {
		szHeader.Data = nodePtr
		szHeader.Len = lns[idx+1]
		bld.WriteString(".")
		bld.WriteString(tmpSz)
	}

	return bld.String()
}

// setMetricName interns the MetricName in an
// object store and stores the addresses of those strings
// in MetricName.nodes
func (mn *MetricName) setMetricName(name string) {
	nodes := strings.Split(name, ".")
	mn.nodes = make([]uintptr, len(nodes))
	for i, node := range nodes {
		// TODO: add error checking? Fail somehow
		nodePtr, err := IdxIntern.AddOrGet([]byte(node))
		if err != nil {
			log.Error(err)
		}
		mn.nodes[i] = nodePtr
	}
}

// ExtensionType is required to use custom marshaling as an extension
// with msgp
func (mn *MetricName) ExtensionType() int8 {
	return 95
}

// Len is required to use custom marshaling as an extension
// with msgp
func (mn *MetricName) Len() int {
	return len(mn.String())
}

// MarshalBinaryTo is required to use custom marshaling as an extension
// in msgp
func (mn *MetricName) MarshalBinaryTo(b []byte) error {
	copy(b, []byte(mn.String()))
	return nil
}

// UnmarshalBinary is required to use custom marshaling as an extension
// in msgp
func (mn *MetricName) UnmarshalBinary(b []byte) error {
	mn.setMetricName(string(b))
	return nil
}

// TagKeyValue stores a Key/Value pair. The strings
// are interned in an object store before they are assigned.
type TagKeyValue struct {
	Key   string
	Value string
}

// String returns a Key/Value pair in the form of
// 'key=value'
func (t *TagKeyValue) String() string {
	bld := strings.Builder{}

	bld.WriteString(t.Key)
	bld.WriteString("=")
	bld.WriteString(t.Value)

	return bld.String()
}

// TagKeyValues stores a slice of all of the Tag Key/Value pair combinations for a MetricDefinition
type TagKeyValues []TagKeyValue

// Strings returns a slice containing all of the Tag Key/Value pair combinations for a MetricDefinition.
// Each item in the slice is in the form of 'key=value'
func (t TagKeyValues) Strings() []string {
	tags := make([]string, len(t))
	for i, tag := range t {
		tags[i] = tag.String()
	}
	return tags
}

// Helper functions to sort TagKeyValues
func (t TagKeyValues) Len() int           { return len(t) }
func (t TagKeyValues) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t TagKeyValues) Less(i, j int) bool { return t[i].Key < t[j].Key }

// MetricDefinition stores information which identifies a single metric
type MetricDefinition struct {
	Id    schema.MKey
	OrgId uint32
	// using custom marshalling for MetricName
	// if there is another way we should explore that
	Name       MetricName `msg:"name,extension"`
	Interval   int
	Unit       string
	mtype      mType
	Tags       TagKeyValues
	LastUpdate int64
	Partition  int32
}

// NameWithTags returns a string version of the MetricDefinition's name with
// all of its tagsin the form of 'name;key1=value1;key2=value2;key3=value3'
func (md *MetricDefinition) NameWithTags() string {
	bld := strings.Builder{}

	md.Name.string(&bld)
	sort.Sort(TagKeyValues(md.Tags))
	for _, tag := range md.Tags {
		if tag.Key == "name" {
			continue
		}
		bld.WriteString(";")
		bld.WriteString(tag.String())
	}
	return bld.String()
}

// SetMType translates a string into a uint8 which is used to store
// the actual metric type. Valid values are 'gauge', 'rate', 'count',
// 'counter', and 'timestamp'.
func (md *MetricDefinition) SetMType(mtype string) {
	switch mtype {
	case "gauge":
		md.mtype = MTypeGauge
	case "rate":
		md.mtype = MTypeRate
	case "count":
		md.mtype = MTypeCount
	case "counter":
		md.mtype = MTypeCounter
	case "timestamp":
		md.mtype = MTypeTimestamp
	default:
		// for values "" and other unknown/corrupted values
		md.mtype = MTypeUndefined
	}
}

// Mtype returns a string version of the current MType
func (md *MetricDefinition) Mtype() string {
	switch md.mtype {
	case MTypeGauge:
		return "gauge"
	case MTypeRate:
		return "rate"
	case MTypeCount:
		return "count"
	case MTypeCounter:
		return "counter"
	case MTypeTimestamp:
		return "timestamp"
	default:
		// case of MTypeUndefined and also default for unknown/corrupted values
		return ""
	}
}

// SetUnit takes a string, interns it in an object store
// and then uses it to store the unit.
func (md *MetricDefinition) SetUnit(unit string) {
	sz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(unit))
	if err != nil {
		log.Errorf("idx: Failed to intern Unit %v. %v", unit, err)
		md.Unit = unit
	}
	md.Unit = sz
}

// SetMetricName interns the MetricName in an
// object store and stores the addresses of those strings
// in MetricName.nodes
func (md *MetricDefinition) SetMetricName(name string) {
	nodes := strings.Split(name, ".")
	md.Name.nodes = make([]uintptr, len(nodes))
	for i, node := range nodes {
		// TODO: add error checking? Fail somehow
		nodePtr, err := IdxIntern.AddOrGet([]byte(node))
		if err != nil {
			log.Errorf("idx: Failed to intern word in MetricName: %v, %v", node, err)
		}
		md.Name.nodes[i] = nodePtr
	}
}

// SetTags takes a []string which should contain Key/Value pairs
// in the form of 'key=value'. It splits up the Key and Value for each
// item, interns them in the object store, and creates a TagKeyValue
// for them. It then stores all of these in Tags.
//
// The items in the input argument should not contain ';'. Each item
// is a separate Key/Value pair. Do not combine multiple Key/Value pairs
// into a single index in the []string.
func (md *MetricDefinition) SetTags(tags []string) {
	md.Tags = make([]TagKeyValue, len(tags))
	sort.Strings(tags)
	for i, tag := range tags {
		if strings.Contains(tag, ";") {
			invalidTag.Inc()
			log.Errorf("idx: Tag %q has an invalid format, ignoring", tag)
			continue
		}
		splits := strings.Split(tag, "=")
		if len(splits) < 2 {
			invalidTag.Inc()
			log.Errorf("idx: Tag %q has an invalid format, ignoring", tag)
			continue
		}
		keySz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(splits[0]))
		if err != nil {
			log.Errorf("idx: Failed to intern tag %q, %v", tag, err)
			keyTmpSz := splits[0]
			md.Tags[i].Key = keyTmpSz
		} else {
			md.Tags[i].Key = keySz
		}

		valueSz, err := IdxIntern.AddOrGetSzNoCprsn([]byte(splits[1]))
		if err != nil {
			log.Errorf("idx: Failed to intern tag %q, %v", tag, err)
			valueTmpSz := splits[1]
			md.Tags[i].Value = valueTmpSz
		} else {
			md.Tags[i].Value = valueSz
		}
	}
}

// SetId creates and sets the MKey which identifies a metric
func (md *MetricDefinition) SetId() {
	sort.Sort(TagKeyValues(md.Tags))
	buffer := bytes.NewBufferString(md.Name.String())
	buffer.WriteByte(0)
	buffer.WriteString(md.Unit)
	buffer.WriteByte(0)
	buffer.WriteString(md.Mtype())
	buffer.WriteByte(0)
	fmt.Fprintf(buffer, "%d", md.Interval)

	for _, t := range md.Tags {
		if t.Key == "name" {
			continue
		}

		buffer.WriteByte(0)
		buffer.WriteString(t.String())
	}

	md.Id = schema.MKey{
		Key: md5.Sum(buffer.Bytes()),
		Org: uint32(md.OrgId),
	}
}

// MetricDefinitionFromMetricDataWithMkey takes an MKey and MetricData and returns a MetricDefinition
// based on them.
func MetricDefinitionFromMetricDataWithMkey(mkey schema.MKey, d *schema.MetricData) *MetricDefinition {
	md := &MetricDefinition{
		Id:         mkey,
		OrgId:      uint32(d.OrgId),
		Interval:   d.Interval,
		LastUpdate: d.Time,
	}

	md.SetMetricName(d.Name)
	md.SetUnit(d.Unit)
	md.SetMType(d.Mtype)
	md.SetTags(d.Tags)

	return md
}

// MetricDefinitionFromMetricData takes a MetricData, attempts to generate an MKey for it,
// and returns a MetricDefinition upon success. On failure it returns an error
func MetricDefinitionFromMetricData(d *schema.MetricData) (*MetricDefinition, error) {
	mkey, err := schema.MKeyFromString(d.Id)
	if err != nil {
		return nil, fmt.Errorf("idx: Error parsing ID: %s", err)
	}

	return MetricDefinitionFromMetricDataWithMkey(mkey, d), nil
}
<|endoftext|>


<commit_before>package httpd

import (
	"bufio"
	"fmt"
	"net/http"

	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/srpc"
)

func statusHandler(w http.ResponseWriter, req *http.Request) {
	if req.URL.Path != "/" {
		http.NotFound(w, req)
		return
	}
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	fmt.Fprintln(writer, "<title>subd status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
subd status page
")
	if !srpc.CheckTlsRequired() {
		fmt.Fprintln(writer,
			`<h1><font color="red">Running in insecure mode. You can get pwned!!!</font></h1>`)
	}
	fmt.Fprintln(writer, "")
	html.WriteHeaderWithRequest(writer, req)
	fmt.Fprintln(writer, "
")
	for _, htmlWriter := range htmlWriters {
		htmlWriter.WriteHtml(writer)
	}
	fmt.Fprintln(writer, "
")
	fmt.Fprintln(writer, "
")
	html.WriteFooter(writer)
	fmt.Fprintln(writer, "")
}
<commit_msg>Add security headers to subd HTTPD handler.<commit_after>package httpd

import (
	"bufio"
	"fmt"
	"net/http"

	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/srpc"
)

func statusHandler(w http.ResponseWriter, req *http.Request) {
	html.SetSecurityHeaders(w) // Compliance checkbox.
	if req.URL.Path != "/" {
		http.NotFound(w, req)
		return
	}
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	fmt.Fprintln(writer, "<title>subd status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
subd status page
")
	if !srpc.CheckTlsRequired() {
		fmt.Fprintln(writer,
			`<h1><font color="red">Running in insecure mode. You can get pwned!!!</font></h1>`)
	}
	fmt.Fprintln(writer, "")
	html.WriteHeaderWithRequest(writer, req)
	fmt.Fprintln(writer, "
")
	for _, htmlWriter := range htmlWriters {
		htmlWriter.WriteHtml(writer)
	}
	fmt.Fprintln(writer, "
")
	fmt.Fprintln(writer, "
")
	html.WriteFooter(writer)
	fmt.Fprintln(writer, "")
}
<|endoftext|>


<commit_before>package validator

import (
	"context"
	"fmt"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/square/go-jose.v2"
	"gopkg.in/square/go-jose.v2/jwt"
)

// Validator to use with the jose v2 package.
type Validator struct {
	keyFunc            func(context.Context) (interface{}, error) // Required.
	signatureAlgorithm jose.SignatureAlgorithm                    // Required.
	expectedClaims     jwt.Expected                               // Optional.
	customClaims       CustomClaims                               // Optional.
	allowedClockSkew   time.Duration                              // Optional.
}

// New sets up a new Validator with the required keyFunc
// and signatureAlgorithm as well as custom options.
func New(
	keyFunc func(context.Context) (interface{}, error),
	signatureAlgorithm string,
	issuerURL string,
	audience []string,
	opts ...Option,
) (*Validator, error) {
	if keyFunc == nil {
		return nil, errors.New("keyFunc is required but was nil")
	}
	if signatureAlgorithm == "" {
		return nil, errors.New("signature algorithm is required but was empty")
	}
	if issuerURL == "" {
		return nil, errors.New("issuer url is required but was empty")
	}
	if audience == nil {
		return nil, errors.New("audience is required but was nil")
	}

	v := &Validator{
		keyFunc:            keyFunc,
		signatureAlgorithm: jose.SignatureAlgorithm(signatureAlgorithm),
		expectedClaims: jwt.Expected{
			Issuer:   issuerURL,
			Audience: audience,
			Time:     time.Now(),
		},
	}

	for _, opt := range opts {
		opt(v)
	}

	return v, nil
}

// ValidateToken validates the passed in JWT using the jose v2 package.
func (v *Validator) ValidateToken(ctx context.Context, tokenString string) (interface{}, error) {
	token, err := jwt.ParseSigned(tokenString)
	if err != nil {
		return nil, fmt.Errorf("could not parse the token: %w", err)
	}

	if string(v.signatureAlgorithm) != token.Headers[0].Algorithm {
		return nil, fmt.Errorf(
			"expected %q signing algorithm but token specified %q",
			v.signatureAlgorithm,
			token.Headers[0].Algorithm,
		)
	}

	key, err := v.keyFunc(ctx)
	if err != nil {
		return nil, fmt.Errorf("error getting the keys from the key func: %w", err)
	}

	claimDest := []interface{}{&jwt.Claims{}}
	if v.customClaims != nil {
		claimDest = append(claimDest, v.customClaims)
	}

	if err = token.Claims(key, claimDest...); err != nil {
		return nil, fmt.Errorf("could not get token claims: %w", err)
	}

	registeredClaims := *claimDest[0].(*jwt.Claims)
	if err = registeredClaims.ValidateWithLeeway(v.expectedClaims, v.allowedClockSkew); err != nil {
		return nil, fmt.Errorf("expected claims not validated: %w", err)
	}

	validatedClaims := &ValidatedClaims{
		RegisteredClaims: RegisteredClaims{
			Issuer:   registeredClaims.Issuer,
			Subject:  registeredClaims.Subject,
			Audience: registeredClaims.Audience,
			ID:       registeredClaims.ID,
		},
	}

	if registeredClaims.Expiry != nil {
		validatedClaims.RegisteredClaims.Expiry = registeredClaims.Expiry.Time().Unix()
	}

	if registeredClaims.NotBefore != nil {
		validatedClaims.RegisteredClaims.NotBefore = registeredClaims.NotBefore.Time().Unix()
	}

	if registeredClaims.IssuedAt != nil {
		validatedClaims.RegisteredClaims.IssuedAt = registeredClaims.IssuedAt.Time().Unix()
	}

	if v.customClaims != nil {
		validatedClaims.CustomClaims = claimDest[1].(CustomClaims)
		if err = validatedClaims.CustomClaims.Validate(ctx); err != nil {
			return nil, fmt.Errorf("custom claims not validated: %w", err)
		}
	}

	return validatedClaims, nil
}
<commit_msg>Set expected time claim just before validating<commit_after>package validator

import (
	"context"
	"fmt"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/square/go-jose.v2"
	"gopkg.in/square/go-jose.v2/jwt"
)

// Validator to use with the jose v2 package.
type Validator struct {
	keyFunc            func(context.Context) (interface{}, error) // Required.
	signatureAlgorithm jose.SignatureAlgorithm                    // Required.
	expectedClaims     jwt.Expected                               // Optional.
	customClaims       CustomClaims                               // Optional.
	allowedClockSkew   time.Duration                              // Optional.
}

// New sets up a new Validator with the required keyFunc
// and signatureAlgorithm as well as custom options.
func New(
	keyFunc func(context.Context) (interface{}, error),
	signatureAlgorithm string,
	issuerURL string,
	audience []string,
	opts ...Option,
) (*Validator, error) {
	if keyFunc == nil {
		return nil, errors.New("keyFunc is required but was nil")
	}
	if signatureAlgorithm == "" {
		return nil, errors.New("signature algorithm is required but was empty")
	}
	if issuerURL == "" {
		return nil, errors.New("issuer url is required but was empty")
	}
	if audience == nil {
		return nil, errors.New("audience is required but was nil")
	}

	v := &Validator{
		keyFunc:            keyFunc,
		signatureAlgorithm: jose.SignatureAlgorithm(signatureAlgorithm),
		expectedClaims: jwt.Expected{
			Issuer:   issuerURL,
			Audience: audience,
		},
	}

	for _, opt := range opts {
		opt(v)
	}

	return v, nil
}

// ValidateToken validates the passed in JWT using the jose v2 package.
func (v *Validator) ValidateToken(ctx context.Context, tokenString string) (interface{}, error) {
	token, err := jwt.ParseSigned(tokenString)
	if err != nil {
		return nil, fmt.Errorf("could not parse the token: %w", err)
	}

	if string(v.signatureAlgorithm) != token.Headers[0].Algorithm {
		return nil, fmt.Errorf(
			"expected %q signing algorithm but token specified %q",
			v.signatureAlgorithm,
			token.Headers[0].Algorithm,
		)
	}

	key, err := v.keyFunc(ctx)
	if err != nil {
		return nil, fmt.Errorf("error getting the keys from the key func: %w", err)
	}

	claimDest := []interface{}{&jwt.Claims{}}
	if v.customClaims != nil {
		claimDest = append(claimDest, v.customClaims)
	}

	if err = token.Claims(key, claimDest...); err != nil {
		return nil, fmt.Errorf("could not get token claims: %w", err)
	}

	registeredClaims := *claimDest[0].(*jwt.Claims)
	v.expectedClaims.Time = time.Now()
	if err = registeredClaims.ValidateWithLeeway(v.expectedClaims, v.allowedClockSkew); err != nil {
		return nil, fmt.Errorf("expected claims not validated: %w", err)
	}

	validatedClaims := &ValidatedClaims{
		RegisteredClaims: RegisteredClaims{
			Issuer:   registeredClaims.Issuer,
			Subject:  registeredClaims.Subject,
			Audience: registeredClaims.Audience,
			ID:       registeredClaims.ID,
		},
	}

	if registeredClaims.Expiry != nil {
		validatedClaims.RegisteredClaims.Expiry = registeredClaims.Expiry.Time().Unix()
	}

	if registeredClaims.NotBefore != nil {
		validatedClaims.RegisteredClaims.NotBefore = registeredClaims.NotBefore.Time().Unix()
	}

	if registeredClaims.IssuedAt != nil {
		validatedClaims.RegisteredClaims.IssuedAt = registeredClaims.IssuedAt.Time().Unix()
	}

	if v.customClaims != nil {
		validatedClaims.CustomClaims = claimDest[1].(CustomClaims)
		if err = validatedClaims.CustomClaims.Validate(ctx); err != nil {
			return nil, fmt.Errorf("custom claims not validated: %w", err)
		}
	}

	return validatedClaims, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package linux

import (
	"fmt"
	"strings"

	"gvisor.dev/gvisor/pkg/abi"
	"gvisor.dev/gvisor/pkg/binary"
)

// Constants for open(2).
const (
	O_ACCMODE   = 000000003
	O_RDONLY    = 000000000
	O_WRONLY    = 000000001
	O_RDWR      = 000000002
	O_CREAT     = 000000100
	O_EXCL      = 000000200
	O_NOCTTY    = 000000400
	O_TRUNC     = 000001000
	O_APPEND    = 000002000
	O_NONBLOCK  = 000004000
	O_DSYNC     = 000010000
	O_ASYNC     = 000020000
	O_DIRECT    = 000040000
	O_LARGEFILE = 000100000
	O_DIRECTORY = 000200000
	O_NOFOLLOW  = 000400000
	O_NOATIME   = 001000000
	O_CLOEXEC   = 002000000
	O_SYNC      = 004000000 // __O_SYNC in Linux
	O_PATH      = 010000000
	O_TMPFILE   = 020000000 // __O_TMPFILE in Linux
)

// Constants for fstatat(2).
const (
	AT_SYMLINK_NOFOLLOW = 0x100
)

// Constants for mount(2).
const (
	MS_RDONLY      = 0x1
	MS_NOSUID      = 0x2
	MS_NODEV       = 0x4
	MS_NOEXEC      = 0x8
	MS_SYNCHRONOUS = 0x10
	MS_REMOUNT     = 0x20
	MS_MANDLOCK    = 0x40
	MS_DIRSYNC     = 0x80
	MS_NOATIME     = 0x400
	MS_NODIRATIME  = 0x800
	MS_BIND        = 0x1000
	MS_MOVE        = 0x2000
	MS_REC         = 0x4000

	MS_POSIXACL    = 0x10000
	MS_UNBINDABLE  = 0x20000
	MS_PRIVATE     = 0x40000
	MS_SLAVE       = 0x80000
	MS_SHARED      = 0x100000
	MS_RELATIME    = 0x200000
	MS_KERNMOUNT   = 0x400000
	MS_I_VERSION   = 0x800000
	MS_STRICTATIME = 0x1000000

	MS_MGC_VAL = 0xC0ED0000
	MS_MGC_MSK = 0xffff0000
)

// Constants for umount2(2).
const (
	MNT_FORCE       = 0x1
	MNT_DETACH      = 0x2
	MNT_EXPIRE      = 0x4
	UMOUNT_NOFOLLOW = 0x8
)

// Constants for unlinkat(2).
const (
	AT_REMOVEDIR = 0x200
)

// Constants for linkat(2) and fchownat(2).
const (
	AT_SYMLINK_FOLLOW = 0x400
	AT_EMPTY_PATH     = 0x1000
)

// Constants for all file-related ...at(2) syscalls.
const (
	AT_FDCWD = -100
)

// Special values for the ns field in utimensat(2).
const (
	UTIME_NOW  = ((1 << 30) - 1)
	UTIME_OMIT = ((1 << 30) - 2)
)

// MaxSymlinkTraversals is the maximum number of links that will be followed by
// the kernel to resolve a symlink.
const MaxSymlinkTraversals = 40

// Constants for flock(2).
const (
	LOCK_SH = 1 // shared lock
	LOCK_EX = 2 // exclusive lock
	LOCK_NB = 4 // or'd with one of the above to prevent blocking
	LOCK_UN = 8 // remove lock
)

// Values for mode_t.
const (
	S_IFMT   = 0170000
	S_IFSOCK = 0140000
	S_IFLNK  = 0120000
	S_IFREG  = 0100000
	S_IFBLK  = 060000
	S_IFDIR  = 040000
	S_IFCHR  = 020000
	S_IFIFO  = 010000

	FileTypeMask        = S_IFMT
	ModeSocket          = S_IFSOCK
	ModeSymlink         = S_IFLNK
	ModeRegular         = S_IFREG
	ModeBlockDevice     = S_IFBLK
	ModeDirectory       = S_IFDIR
	ModeCharacterDevice = S_IFCHR
	ModeNamedPipe       = S_IFIFO

	ModeSetUID = 04000
	ModeSetGID = 02000
	ModeSticky = 01000

	ModeUserAll     = 0700
	ModeUserRead    = 0400
	ModeUserWrite   = 0200
	ModeUserExec    = 0100
	ModeGroupAll    = 0070
	ModeGroupRead   = 0040
	ModeGroupWrite  = 0020
	ModeGroupExec   = 0010
	ModeOtherAll    = 0007
	ModeOtherRead   = 0004
	ModeOtherWrite  = 0002
	ModeOtherExec   = 0001
	PermissionsMask = 0777
)

// Values for linux_dirent64.d_type.
const (
	DT_UNKNOWN = 0
	DT_FIFO    = 1
	DT_CHR     = 2
	DT_DIR     = 4
	DT_BLK     = 6
	DT_REG     = 8
	DT_LNK     = 10
	DT_SOCK    = 12
	DT_WHT     = 14
)

// Values for preadv2/pwritev2.
const (
	RWF_HIPRI = 0x00000001
	RWF_DSYNC = 0x00000002
	RWF_SYNC  = 0x00000004
	RWF_VALID = RWF_HIPRI | RWF_DSYNC | RWF_SYNC
)

// Stat represents struct stat.
type Stat struct {
	Dev     uint64
	Ino     uint64
	Nlink   uint64
	Mode    uint32
	UID     uint32
	GID     uint32
	_       int32
	Rdev    uint64
	Size    int64
	Blksize int64
	Blocks  int64
	ATime   Timespec
	MTime   Timespec
	CTime   Timespec
	_       [3]int64
}

// SizeOfStat is the size of a Stat struct.
var SizeOfStat = binary.Size(Stat{})

// Flags for statx.
const (
	AT_STATX_SYNC_TYPE    = 0x6000
	AT_STATX_SYNC_AS_STAT = 0x0000
	AT_STATX_FORCE_SYNC   = 0x2000
	AT_STATX_DONT_SYNC    = 0x4000
)

// Mask values for statx.
const (
	STATX_TYPE        = 0x00000001
	STATX_MODE        = 0x00000002
	STATX_NLINK       = 0x00000004
	STATX_UID         = 0x00000008
	STATX_GID         = 0x00000010
	STATX_ATIME       = 0x00000020
	STATX_MTIME       = 0x00000040
	STATX_CTIME       = 0x00000080
	STATX_INO         = 0x00000100
	STATX_SIZE        = 0x00000200
	STATX_BLOCKS      = 0x00000400
	STATX_BASIC_STATS = 0x000007ff
	STATX_BTIME       = 0x00000800
	STATX_ALL         = 0x00000fff
	STATX__RESERVED   = 0x80000000
)

// Bitmasks for Statx.Attributes and Statx.AttributesMask, from
// include/uapi/linux/stat.h.
const (
	STATX_ATTR_COMPRESSED = 0x00000004
	STATX_ATTR_IMMUTABLE  = 0x00000010
	STATX_ATTR_APPEND     = 0x00000020
	STATX_ATTR_NODUMP     = 0x00000040
	STATX_ATTR_ENCRYPTED  = 0x00000800
	STATX_ATTR_AUTOMOUNT  = 0x00001000
)

// Statx represents struct statx.
type Statx struct {
	Mask           uint32
	Blksize        uint32
	Attributes     uint64
	Nlink          uint32
	UID            uint32
	GID            uint32
	Mode           uint16
	Ino            uint64
	Size           uint64
	Blocks         uint64
	AttributesMask uint64
	Atime          StatxTimestamp
	Btime          StatxTimestamp
	Ctime          StatxTimestamp
	Mtime          StatxTimestamp
	RdevMajor      uint32
	RdevMinor      uint32
	DevMajor       uint32
	DevMinor       uint32
}

// FileMode represents a mode_t.
type FileMode uint

// Permissions returns just the permission bits.
func (m FileMode) Permissions() FileMode {
	return m & PermissionsMask
}

// FileType returns just the file type bits.
func (m FileMode) FileType() FileMode {
	return m & FileTypeMask
}

// ExtraBits returns everything but the file type and permission bits.
func (m FileMode) ExtraBits() FileMode {
	return m &^ (PermissionsMask | FileTypeMask)
}

// String returns a string representation of m.
func (m FileMode) String() string {
	var s []string
	if ft := m.FileType(); ft != 0 {
		s = append(s, fileType.Parse(uint64(ft)))
	}
	if eb := m.ExtraBits(); eb != 0 {
		s = append(s, modeExtraBits.Parse(uint64(eb)))
	}
	s = append(s, fmt.Sprintf("0o%o", m.Permissions()))
	return strings.Join(s, "|")
}

var modeExtraBits = abi.FlagSet{
	{
		Flag: ModeSetUID,
		Name: "S_ISUID",
	},
	{
		Flag: ModeSetGID,
		Name: "S_ISGID",
	},
	{
		Flag: ModeSticky,
		Name: "S_ISVTX",
	},
}

var fileType = abi.ValueSet{
	ModeSocket:          "S_IFSOCK",
	ModeSymlink:         "S_IFLINK",
	ModeRegular:         "S_IFREG",
	ModeBlockDevice:     "S_IFBLK",
	ModeDirectory:       "S_IFDIR",
	ModeCharacterDevice: "S_IFCHR",
	ModeNamedPipe:       "S_IFIFO",
}

// Constants for memfd_create(2). Source: include/uapi/linux/memfd.h
const (
	MFD_CLOEXEC       = 0x0001
	MFD_ALLOW_SEALING = 0x0002
)

// Constants related to file seals. Source: include/uapi/{asm-generic,linux}/fcntl.h
const (
	F_LINUX_SPECIFIC_BASE = 1024
	F_ADD_SEALS           = F_LINUX_SPECIFIC_BASE + 9
	F_GET_SEALS           = F_LINUX_SPECIFIC_BASE + 10

	F_SEAL_SEAL   = 0x0001 // Prevent further seals from being set.
	F_SEAL_SHRINK = 0x0002 // Prevent file from shrinking.
	F_SEAL_GROW   = 0x0004 // Prevent file from growing.
	F_SEAL_WRITE  = 0x0008 // Prevent writes.
)

// Constants related to fallocate(2). Source: include/uapi/linux/falloc.h
const (
	FALLOC_FL_KEEP_SIZE      = 0x01
	FALLOC_FL_PUNCH_HOLE     = 0x02
	FALLOC_FL_NO_HIDE_STALE  = 0x04
	FALLOC_FL_COLLAPSE_RANGE = 0x08
	FALLOC_FL_ZERO_RANGE     = 0x10
	FALLOC_FL_INSERT_RANGE   = 0x20
	FALLOC_FL_UNSHARE_RANGE  = 0x40
)
<commit_msg>Fix struct statx field alignment.<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package linux

import (
	"fmt"
	"strings"

	"gvisor.dev/gvisor/pkg/abi"
	"gvisor.dev/gvisor/pkg/binary"
)

// Constants for open(2).
const (
	O_ACCMODE   = 000000003
	O_RDONLY    = 000000000
	O_WRONLY    = 000000001
	O_RDWR      = 000000002
	O_CREAT     = 000000100
	O_EXCL      = 000000200
	O_NOCTTY    = 000000400
	O_TRUNC     = 000001000
	O_APPEND    = 000002000
	O_NONBLOCK  = 000004000
	O_DSYNC     = 000010000
	O_ASYNC     = 000020000
	O_DIRECT    = 000040000
	O_LARGEFILE = 000100000
	O_DIRECTORY = 000200000
	O_NOFOLLOW  = 000400000
	O_NOATIME   = 001000000
	O_CLOEXEC   = 002000000
	O_SYNC      = 004000000 // __O_SYNC in Linux
	O_PATH      = 010000000
	O_TMPFILE   = 020000000 // __O_TMPFILE in Linux
)

// Constants for fstatat(2).
const (
	AT_SYMLINK_NOFOLLOW = 0x100
)

// Constants for mount(2).
const (
	MS_RDONLY      = 0x1
	MS_NOSUID      = 0x2
	MS_NODEV       = 0x4
	MS_NOEXEC      = 0x8
	MS_SYNCHRONOUS = 0x10
	MS_REMOUNT     = 0x20
	MS_MANDLOCK    = 0x40
	MS_DIRSYNC     = 0x80
	MS_NOATIME     = 0x400
	MS_NODIRATIME  = 0x800
	MS_BIND        = 0x1000
	MS_MOVE        = 0x2000
	MS_REC         = 0x4000

	MS_POSIXACL    = 0x10000
	MS_UNBINDABLE  = 0x20000
	MS_PRIVATE     = 0x40000
	MS_SLAVE       = 0x80000
	MS_SHARED      = 0x100000
	MS_RELATIME    = 0x200000
	MS_KERNMOUNT   = 0x400000
	MS_I_VERSION   = 0x800000
	MS_STRICTATIME = 0x1000000

	MS_MGC_VAL = 0xC0ED0000
	MS_MGC_MSK = 0xffff0000
)

// Constants for umount2(2).
const (
	MNT_FORCE       = 0x1
	MNT_DETACH      = 0x2
	MNT_EXPIRE      = 0x4
	UMOUNT_NOFOLLOW = 0x8
)

// Constants for unlinkat(2).
const (
	AT_REMOVEDIR = 0x200
)

// Constants for linkat(2) and fchownat(2).
const (
	AT_SYMLINK_FOLLOW = 0x400
	AT_EMPTY_PATH     = 0x1000
)

// Constants for all file-related ...at(2) syscalls.
const (
	AT_FDCWD = -100
)

// Special values for the ns field in utimensat(2).
const (
	UTIME_NOW  = ((1 << 30) - 1)
	UTIME_OMIT = ((1 << 30) - 2)
)

// MaxSymlinkTraversals is the maximum number of links that will be followed by
// the kernel to resolve a symlink.
const MaxSymlinkTraversals = 40

// Constants for flock(2).
const (
	LOCK_SH = 1 // shared lock
	LOCK_EX = 2 // exclusive lock
	LOCK_NB = 4 // or'd with one of the above to prevent blocking
	LOCK_UN = 8 // remove lock
)

// Values for mode_t.
const (
	S_IFMT   = 0170000
	S_IFSOCK = 0140000
	S_IFLNK  = 0120000
	S_IFREG  = 0100000
	S_IFBLK  = 060000
	S_IFDIR  = 040000
	S_IFCHR  = 020000
	S_IFIFO  = 010000

	FileTypeMask        = S_IFMT
	ModeSocket          = S_IFSOCK
	ModeSymlink         = S_IFLNK
	ModeRegular         = S_IFREG
	ModeBlockDevice     = S_IFBLK
	ModeDirectory       = S_IFDIR
	ModeCharacterDevice = S_IFCHR
	ModeNamedPipe       = S_IFIFO

	ModeSetUID = 04000
	ModeSetGID = 02000
	ModeSticky = 01000

	ModeUserAll     = 0700
	ModeUserRead    = 0400
	ModeUserWrite   = 0200
	ModeUserExec    = 0100
	ModeGroupAll    = 0070
	ModeGroupRead   = 0040
	ModeGroupWrite  = 0020
	ModeGroupExec   = 0010
	ModeOtherAll    = 0007
	ModeOtherRead   = 0004
	ModeOtherWrite  = 0002
	ModeOtherExec   = 0001
	PermissionsMask = 0777
)

// Values for linux_dirent64.d_type.
const (
	DT_UNKNOWN = 0
	DT_FIFO    = 1
	DT_CHR     = 2
	DT_DIR     = 4
	DT_BLK     = 6
	DT_REG     = 8
	DT_LNK     = 10
	DT_SOCK    = 12
	DT_WHT     = 14
)

// Values for preadv2/pwritev2.
const (
	RWF_HIPRI = 0x00000001
	RWF_DSYNC = 0x00000002
	RWF_SYNC  = 0x00000004
	RWF_VALID = RWF_HIPRI | RWF_DSYNC | RWF_SYNC
)

// Stat represents struct stat.
type Stat struct {
	Dev     uint64
	Ino     uint64
	Nlink   uint64
	Mode    uint32
	UID     uint32
	GID     uint32
	_       int32
	Rdev    uint64
	Size    int64
	Blksize int64
	Blocks  int64
	ATime   Timespec
	MTime   Timespec
	CTime   Timespec
	_       [3]int64
}

// SizeOfStat is the size of a Stat struct.
var SizeOfStat = binary.Size(Stat{})

// Flags for statx.
const (
	AT_STATX_SYNC_TYPE    = 0x6000
	AT_STATX_SYNC_AS_STAT = 0x0000
	AT_STATX_FORCE_SYNC   = 0x2000
	AT_STATX_DONT_SYNC    = 0x4000
)

// Mask values for statx.
const (
	STATX_TYPE        = 0x00000001
	STATX_MODE        = 0x00000002
	STATX_NLINK       = 0x00000004
	STATX_UID         = 0x00000008
	STATX_GID         = 0x00000010
	STATX_ATIME       = 0x00000020
	STATX_MTIME       = 0x00000040
	STATX_CTIME       = 0x00000080
	STATX_INO         = 0x00000100
	STATX_SIZE        = 0x00000200
	STATX_BLOCKS      = 0x00000400
	STATX_BASIC_STATS = 0x000007ff
	STATX_BTIME       = 0x00000800
	STATX_ALL         = 0x00000fff
	STATX__RESERVED   = 0x80000000
)

// Bitmasks for Statx.Attributes and Statx.AttributesMask, from
// include/uapi/linux/stat.h.
const (
	STATX_ATTR_COMPRESSED = 0x00000004
	STATX_ATTR_IMMUTABLE  = 0x00000010
	STATX_ATTR_APPEND     = 0x00000020
	STATX_ATTR_NODUMP     = 0x00000040
	STATX_ATTR_ENCRYPTED  = 0x00000800
	STATX_ATTR_AUTOMOUNT  = 0x00001000
)

// Statx represents struct statx.
type Statx struct {
	Mask           uint32
	Blksize        uint32
	Attributes     uint64
	Nlink          uint32
	UID            uint32
	GID            uint32
	Mode           uint16
	_              uint16
	Ino            uint64
	Size           uint64
	Blocks         uint64
	AttributesMask uint64
	Atime          StatxTimestamp
	Btime          StatxTimestamp
	Ctime          StatxTimestamp
	Mtime          StatxTimestamp
	RdevMajor      uint32
	RdevMinor      uint32
	DevMajor       uint32
	DevMinor       uint32
}

// FileMode represents a mode_t.
type FileMode uint

// Permissions returns just the permission bits.
func (m FileMode) Permissions() FileMode {
	return m & PermissionsMask
}

// FileType returns just the file type bits.
func (m FileMode) FileType() FileMode {
	return m & FileTypeMask
}

// ExtraBits returns everything but the file type and permission bits.
func (m FileMode) ExtraBits() FileMode {
	return m &^ (PermissionsMask | FileTypeMask)
}

// String returns a string representation of m.
func (m FileMode) String() string {
	var s []string
	if ft := m.FileType(); ft != 0 {
		s = append(s, fileType.Parse(uint64(ft)))
	}
	if eb := m.ExtraBits(); eb != 0 {
		s = append(s, modeExtraBits.Parse(uint64(eb)))
	}
	s = append(s, fmt.Sprintf("0o%o", m.Permissions()))
	return strings.Join(s, "|")
}

var modeExtraBits = abi.FlagSet{
	{
		Flag: ModeSetUID,
		Name: "S_ISUID",
	},
	{
		Flag: ModeSetGID,
		Name: "S_ISGID",
	},
	{
		Flag: ModeSticky,
		Name: "S_ISVTX",
	},
}

var fileType = abi.ValueSet{
	ModeSocket:          "S_IFSOCK",
	ModeSymlink:         "S_IFLINK",
	ModeRegular:         "S_IFREG",
	ModeBlockDevice:     "S_IFBLK",
	ModeDirectory:       "S_IFDIR",
	ModeCharacterDevice: "S_IFCHR",
	ModeNamedPipe:       "S_IFIFO",
}

// Constants for memfd_create(2). Source: include/uapi/linux/memfd.h
const (
	MFD_CLOEXEC       = 0x0001
	MFD_ALLOW_SEALING = 0x0002
)

// Constants related to file seals. Source: include/uapi/{asm-generic,linux}/fcntl.h
const (
	F_LINUX_SPECIFIC_BASE = 1024
	F_ADD_SEALS           = F_LINUX_SPECIFIC_BASE + 9
	F_GET_SEALS           = F_LINUX_SPECIFIC_BASE + 10

	F_SEAL_SEAL   = 0x0001 // Prevent further seals from being set.
	F_SEAL_SHRINK = 0x0002 // Prevent file from shrinking.
	F_SEAL_GROW   = 0x0004 // Prevent file from growing.
	F_SEAL_WRITE  = 0x0008 // Prevent writes.
)

// Constants related to fallocate(2). Source: include/uapi/linux/falloc.h
const (
	FALLOC_FL_KEEP_SIZE      = 0x01
	FALLOC_FL_PUNCH_HOLE     = 0x02
	FALLOC_FL_NO_HIDE_STALE  = 0x04
	FALLOC_FL_COLLAPSE_RANGE = 0x08
	FALLOC_FL_ZERO_RANGE     = 0x10
	FALLOC_FL_INSERT_RANGE   = 0x20
	FALLOC_FL_UNSHARE_RANGE  = 0x40
)
<|endoftext|>


<commit_before>package penname

import ()

type PenName struct {
	Closed      bool
	Written     []byte
	returnError error
}

func New() *PenName {
	return &PenName{}
}

func (p *PenName) Close() error {
	if p.returnError != nil {
		return p.returnError
	}

	p.Closed = true
	return nil
}

// Convencinece method for reseting state.
func (p *PenName) Reset() {
	p.Closed = false
	p.Written = []byte{}
}

// Sets the error that will be returned when actions are attempted.
func (p *PenName) ReturnError(err error) {
	p.returnError = err
}

// Implements the Writer interface, returning an error if returnError is set.
// The contents of what is written is stored in Written for inspection later.
func (p *PenName) Write(b []byte) (n int, err error) {
	if p.returnError != nil {
		return 0, p.returnError
	}

	p.Written = b
	return len(p.Written), nil
}
<commit_msg>adding methods to implement responsewriter interface<commit_after>package penname

import (
	"fmt"
	"net/http"
)

type PenName struct {
	Closed      bool
	Written     []byte
	returnError error
}

func New() *PenName {
	return &PenName{}
}

func (p *PenName) Close() error {
	if p.returnError != nil {
		return p.returnError
	}

	p.Closed = true
	return nil
}

// Implements the ResponseWriter interface, returning an empty set of headers
// to meet the interface requirements
func (p *PenName) Header() http.Header {
	return http.Header{}
}

// Convencinece method for reseting state.
func (p *PenName) Reset() {
	p.Closed = false
	p.Written = []byte{}
}

// Sets the error that will be returned when actions are attempted.
func (p *PenName) ReturnError(err error) {
	p.returnError = err
}

// Implements the Writer interface, returning an error if returnError is set.
// The contents of what is written is stored in Written for inspection later.
func (p *PenName) Write(b []byte) (n int, err error) {
	if p.returnError != nil {
		return 0, p.returnError
	}

	p.Written = b
	return len(p.Written), nil
}

// Implements the ResponseWriter interface, capturing headers to the same written buffer
func (p *PenName) WriteHeader(i int) {
	p.Write([]byte(fmt.Sprintf("Header: %v", i)))
}
<|endoftext|>


<commit_before>package fileutils

import (
	"bytes"
	"io"
	"os"
	"path/filepath"
)

func Open(path string) (file *os.File, err error) {
	err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModeTemporary|os.ModePerm)
	if err != nil {
		return
	}

	return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
}

func Create(path string) (file *os.File, err error) {
	err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModeTemporary|os.ModePerm)
	if err != nil {
		return
	}

	return os.Create(path)
}

func Read(file *os.File) string {
	buf := &bytes.Buffer{}
	_, err := io.Copy(buf, file)

	if err != nil {
		return ""
	}

	return string(buf.Bytes())
}

func CopyPathToPath(fromPath, toPath string) (err error) {
	dst, err := Create(toPath)
	if err != nil {
		return
	}
	defer dst.Close()

	return CopyPathToWriter(fromPath, dst)
}

func IsDirEmpty(dir string) (isEmpty bool, err error) {
	dirFile, err := os.Open(dir)
	if err != nil {
		return
	}

	_, readErr := dirFile.Readdirnames(1)
	if readErr != nil {
		isEmpty = true
	} else {
		isEmpty = false
	}
	return
}

func CopyPathToWriter(originalFilePath string, targetWriter io.Writer) (err error) {
	originalFile, err := os.Open(originalFilePath)
	if err != nil {
		return
	}
	defer originalFile.Close()

	_, err = io.Copy(targetWriter, originalFile)
	if err != nil {
		return
	}

	return
}

func CopyReaderToPath(src io.Reader, targetPath string) (err error) {
	destFile, err := Create(targetPath)
	if err != nil {
		return
	}
	defer destFile.Close()

	_, err = io.Copy(destFile, src)
	return
}

func SetMode(dest string, fileMode os.FileMode) (err error) {
	err = os.Chmod(dest, fileMode)
	return
}

func SetModeFromPath(dest string, src string) (err error) {
	fileInfo, err := os.Stat(src)
	if err != nil {
		return
	}

	return SetMode(dest, fileInfo.Mode())
}
<commit_msg>Stop setting ModeTemporary on files<commit_after>package fileutils

import (
	"bytes"
	"io"
	"os"
	"path/filepath"
)

func Open(path string) (file *os.File, err error) {
	err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm)
	if err != nil {
		return
	}

	return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
}

func Create(path string) (file *os.File, err error) {
	err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm)
	if err != nil {
		return
	}

	return os.Create(path)
}

func Read(file *os.File) string {
	buf := &bytes.Buffer{}
	_, err := io.Copy(buf, file)

	if err != nil {
		return ""
	}

	return string(buf.Bytes())
}

func CopyPathToPath(fromPath, toPath string) (err error) {
	dst, err := Create(toPath)
	if err != nil {
		return
	}
	defer dst.Close()

	return CopyPathToWriter(fromPath, dst)
}

func IsDirEmpty(dir string) (isEmpty bool, err error) {
	dirFile, err := os.Open(dir)
	if err != nil {
		return
	}

	_, readErr := dirFile.Readdirnames(1)
	if readErr != nil {
		isEmpty = true
	} else {
		isEmpty = false
	}
	return
}

func CopyPathToWriter(originalFilePath string, targetWriter io.Writer) (err error) {
	originalFile, err := os.Open(originalFilePath)
	if err != nil {
		return
	}
	defer originalFile.Close()

	_, err = io.Copy(targetWriter, originalFile)
	if err != nil {
		return
	}

	return
}

func CopyReaderToPath(src io.Reader, targetPath string) (err error) {
	destFile, err := Create(targetPath)
	if err != nil {
		return
	}
	defer destFile.Close()

	_, err = io.Copy(destFile, src)
	return
}

func SetMode(dest string, fileMode os.FileMode) (err error) {
	err = os.Chmod(dest, fileMode)
	return
}

func SetModeFromPath(dest string, src string) (err error) {
	fileInfo, err := os.Stat(src)
	if err != nil {
		return
	}

	return SetMode(dest, fileInfo.Mode())
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package price

import (
	"fmt"
	"math"
	"time"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	"k8s.io/autoscaler/cluster-autoscaler/expander"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"

	"github.com/golang/glog"
)

// *************
// The detailed description of what is going on in this expander can be found
// here:
// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/pricing.md
// **********

type priceBased struct {
	pricingModel          cloudprovider.PricingModel
	preferredNodeProvider PreferredNodeProvider
	nodeUnfitness         NodeUnfitness
}

var (
	// defaultPreferredNode is the node that is preferred if PreferredNodeProvider fails.
	// 4 cpu, 16gb ram.
	defaultPreferredNode = buildNode(4*1000, 4*4*1024*1024*1024)

	// priceStabilizationPod is the pod cost to stabilize node_cost/pod_cost ratio a bit.
	// 0.5 cpu, 500 mb ram
	priceStabilizationPod = buildPod("stabilize", 500, 500*1024*1024)
)

// NewStrategy returns an expansion strategy that picks nodes based on price and preferred node type.
func NewStrategy(pricingModel cloudprovider.PricingModel,
	preferredNodeProvider PreferredNodeProvider,
	nodeUnfitness NodeUnfitness,
) expander.Strategy {
	return &priceBased{
		pricingModel:          pricingModel,
		preferredNodeProvider: preferredNodeProvider,
		nodeUnfitness:         nodeUnfitness,
	}
}

// BestOption selects option based on cost and preferred node type.
func (p *priceBased) BestOption(expansionOptions []expander.Option, nodeInfos map[string]*schedulercache.NodeInfo) *expander.Option {
	var bestOption *expander.Option
	bestOptionScore := 0.0
	now := time.Now()
	then := now.Add(time.Hour)

	preferredNode, err := p.preferredNodeProvider.Node()
	if err != nil {
		glog.Errorf("Failed to get preferred node, switching to default: %v", err)
		preferredNode = defaultPreferredNode
	}
	stabilizationPrice, err := p.pricingModel.PodPrice(priceStabilizationPod, now, then)
	if err != nil {
		glog.Errorf("Failed to get price for stabilization pod: %v", err)
		// continuing without stabilization.
	}

nextoption:
	for _, option := range expansionOptions {
		nodeInfo, found := nodeInfos[option.NodeGroup.Id()]
		if !found {
			glog.Warningf("No node info for %s", option.NodeGroup.Id())
			continue
		}
		nodePrice, err := p.pricingModel.NodePrice(nodeInfo.Node(), now, then)
		if err != nil {
			glog.Warningf("Failed to calculate node price for %s: %v", option.NodeGroup.Id(), err)
			continue
		}
		totalNodePrice := nodePrice * float64(option.NodeCount)
		totalPodPrice := 0.0
		for _, pod := range option.Pods {
			podPrice, err := p.pricingModel.PodPrice(pod, now, then)
			if err != nil {
				glog.Warningf("Failed to calculate pod price for %s/%s: %v", pod.Namespace, pod.Name, err)
				continue nextoption
			}
			totalPodPrice += podPrice
		}
		// Total pod price is 0 when the pods have no requests. The pods must have some other
		// requirements that prevent them from scheduling like AntiAffinity, HostPort or the
		// pods quota on all nodes has been already used. We use stabilizationPrice in the formula
		// below so this should not be a problem.

		// How well the money is spent.
		priceSubScore := (totalNodePrice + stabilizationPrice) / (totalPodPrice + stabilizationPrice)
		// How well the node matches generic cluster needs
		nodeUnfitness := p.nodeUnfitness(preferredNode, nodeInfo.Node())

		// TODO: normalize node count against preferred node.
		supressedUnfitness := (nodeUnfitness-1.0)*(1.0-math.Tanh(float64(option.NodeCount-1)/15.0)) + 1.0

		optionScore := supressedUnfitness * priceSubScore

		debug := fmt.Sprintf("all_nodes_price=%f pods_price=%f stabilized_ratio=%f unfitness=%f supressed=%f final_score=%f",
			totalNodePrice,
			totalPodPrice,
			priceSubScore,
			nodeUnfitness,
			supressedUnfitness,
			optionScore,
		)

		glog.V(5).Infof("Price expander for %s: %s", option.NodeGroup.Id(), debug)

		if bestOption == nil || bestOptionScore > optionScore {
			bestOption = &expander.Option{
				NodeGroup: option.NodeGroup,
				NodeCount: option.NodeCount,
				Debug:     fmt.Sprintf("%s | price-expander: %s", option.Debug, debug),
				Pods:      option.Pods,
			}
			bestOptionScore = optionScore
		}
	}
	return bestOption
}

// buildPod creates a pod with specified resources.
func buildPod(name string, millicpu int64, mem int64) *apiv1.Pod {
	return &apiv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "default",
			Name:      name,
			SelfLink:  fmt.Sprintf("/api/v1/namespaces/default/pods/%s", name),
		},
		Spec: apiv1.PodSpec{
			Containers: []apiv1.Container{
				{
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceCPU:    *resource.NewMilliQuantity(millicpu, resource.DecimalSI),
							apiv1.ResourceMemory: *resource.NewQuantity(mem, resource.DecimalSI),
						},
					},
				},
			},
		},
	}
}
<commit_msg>Penalty for non-existing node groups in price expander<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package price

import (
	"fmt"
	"math"
	"time"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
	"k8s.io/autoscaler/cluster-autoscaler/expander"
	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"

	"github.com/golang/glog"
)

// *************
// The detailed description of what is going on in this expander can be found
// here:
// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/pricing.md
// **********

type priceBased struct {
	pricingModel          cloudprovider.PricingModel
	preferredNodeProvider PreferredNodeProvider
	nodeUnfitness         NodeUnfitness
}

var (
	// defaultPreferredNode is the node that is preferred if PreferredNodeProvider fails.
	// 4 cpu, 16gb ram.
	defaultPreferredNode = buildNode(4*1000, 4*4*1024*1024*1024)

	// priceStabilizationPod is the pod cost to stabilize node_cost/pod_cost ratio a bit.
	// 0.5 cpu, 500 mb ram
	priceStabilizationPod = buildPod("stabilize", 500, 500*1024*1024)

	// Penalty given to node groups that are yet to be created.
	// TODO: make it a flag
	// TODO: investigate what a proper value should be
	notExistCoeficient = 2.0
)

// NewStrategy returns an expansion strategy that picks nodes based on price and preferred node type.
func NewStrategy(pricingModel cloudprovider.PricingModel,
	preferredNodeProvider PreferredNodeProvider,
	nodeUnfitness NodeUnfitness,
) expander.Strategy {
	return &priceBased{
		pricingModel:          pricingModel,
		preferredNodeProvider: preferredNodeProvider,
		nodeUnfitness:         nodeUnfitness,
	}
}

// BestOption selects option based on cost and preferred node type.
func (p *priceBased) BestOption(expansionOptions []expander.Option, nodeInfos map[string]*schedulercache.NodeInfo) *expander.Option {
	var bestOption *expander.Option
	bestOptionScore := 0.0
	now := time.Now()
	then := now.Add(time.Hour)

	preferredNode, err := p.preferredNodeProvider.Node()
	if err != nil {
		glog.Errorf("Failed to get preferred node, switching to default: %v", err)
		preferredNode = defaultPreferredNode
	}
	stabilizationPrice, err := p.pricingModel.PodPrice(priceStabilizationPod, now, then)
	if err != nil {
		glog.Errorf("Failed to get price for stabilization pod: %v", err)
		// continuing without stabilization.
	}

nextoption:
	for _, option := range expansionOptions {
		nodeInfo, found := nodeInfos[option.NodeGroup.Id()]
		if !found {
			glog.Warningf("No node info for %s", option.NodeGroup.Id())
			continue
		}
		nodePrice, err := p.pricingModel.NodePrice(nodeInfo.Node(), now, then)
		if err != nil {
			glog.Warningf("Failed to calculate node price for %s: %v", option.NodeGroup.Id(), err)
			continue
		}
		totalNodePrice := nodePrice * float64(option.NodeCount)
		totalPodPrice := 0.0
		for _, pod := range option.Pods {
			podPrice, err := p.pricingModel.PodPrice(pod, now, then)
			if err != nil {
				glog.Warningf("Failed to calculate pod price for %s/%s: %v", pod.Namespace, pod.Name, err)
				continue nextoption
			}
			totalPodPrice += podPrice
		}
		// Total pod price is 0 when the pods have no requests. The pods must have some other
		// requirements that prevent them from scheduling like AntiAffinity, HostPort or the
		// pods quota on all nodes has been already used. We use stabilizationPrice in the formula
		// below so this should not be a problem.

		// How well the money is spent.
		priceSubScore := (totalNodePrice + stabilizationPrice) / (totalPodPrice + stabilizationPrice)
		// How well the node matches generic cluster needs
		nodeUnfitness := p.nodeUnfitness(preferredNode, nodeInfo.Node())

		// TODO: normalize node count against preferred node.
		supressedUnfitness := (nodeUnfitness-1.0)*(1.0-math.Tanh(float64(option.NodeCount-1)/15.0)) + 1.0

		optionScore := supressedUnfitness * priceSubScore

		if exist, err := option.NodeGroup.Exist(); err != nil && !exist {
			optionScore *= notExistCoeficient
		}

		debug := fmt.Sprintf("all_nodes_price=%f pods_price=%f stabilized_ratio=%f unfitness=%f supressed=%f final_score=%f",
			totalNodePrice,
			totalPodPrice,
			priceSubScore,
			nodeUnfitness,
			supressedUnfitness,
			optionScore,
		)

		glog.V(5).Infof("Price expander for %s: %s", option.NodeGroup.Id(), debug)

		if bestOption == nil || bestOptionScore > optionScore {
			bestOption = &expander.Option{
				NodeGroup: option.NodeGroup,
				NodeCount: option.NodeCount,
				Debug:     fmt.Sprintf("%s | price-expander: %s", option.Debug, debug),
				Pods:      option.Pods,
			}
			bestOptionScore = optionScore
		}
	}
	return bestOption
}

// buildPod creates a pod with specified resources.
func buildPod(name string, millicpu int64, mem int64) *apiv1.Pod {
	return &apiv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: "default",
			Name:      name,
			SelfLink:  fmt.Sprintf("/api/v1/namespaces/default/pods/%s", name),
		},
		Spec: apiv1.PodSpec{
			Containers: []apiv1.Container{
				{
					Resources: apiv1.ResourceRequirements{
						Requests: apiv1.ResourceList{
							apiv1.ResourceCPU:    *resource.NewMilliQuantity(millicpu, resource.DecimalSI),
							apiv1.ResourceMemory: *resource.NewQuantity(mem, resource.DecimalSI),
						},
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>// Package utils contains various utility functions and whatnot.
package utils

import (
	"bufio"
	"os"
	"path"
	"path/filepath"
	"strings"

	"gopkg.in/op/go-logging.v1"

	"core"
)

var log = logging.MustGetLogger("utils")

// FindAllSubpackages finds all packages under a particular path.
// Used to implement rules with ... where we need to know all possible packages
// under that location.
func FindAllSubpackages(config *core.Configuration, rootPath string, prefix string) <-chan string {
	ch := make(chan string)
	go func() {
		if rootPath == "" {
			rootPath = "."
		}
		if err := filepath.Walk(rootPath, func(name string, info os.FileInfo, err error) error {
			if err != nil {
				return err // stop on any error
			} else if name == core.OutDir || (info.IsDir() && strings.HasPrefix(info.Name(), ".") && name != ".") {
				return filepath.SkipDir // Don't walk output or hidden directories
			} else if info.IsDir() && !strings.HasPrefix(name, prefix) && !strings.HasPrefix(prefix, name) {
				return filepath.SkipDir // Skip any directory without the prefix we're after (but not any directory beneath that)
			} else if isABuildFile(info.Name(), config) && !info.IsDir() {
				dir, _ := path.Split(name)
				ch <- strings.TrimRight(dir, "/")
			} else if name == config.Parse.ExperimentalDir {
				return filepath.SkipDir // Skip the experimental directory if it's set
			}
			// Check against blacklist
			for _, dir := range config.Parse.BlacklistDirs {
				if dir == info.Name() {
					return filepath.SkipDir
				}
			}
			return nil
		}); err != nil {
			log.Fatalf("Failed to walk tree under %s; %s\n", rootPath, err)
		}
		close(ch)
	}()
	return ch
}

var seenStdin = false // Used to track that we don't try to read stdin twice

// isABuildFile returns true if given filename is a build file name.
func isABuildFile(name string, config *core.Configuration) bool {
	for _, buildFileName := range config.Parse.BuildFileName {
		if name == buildFileName {
			return true
		}
	}
	return false
}

// ReadStdin reads a sequence of space-delimited words from standard input.
// Words are pushed onto the returned channel asynchronously.
func ReadStdin() <-chan string {
	c := make(chan string)
	if seenStdin {
		log.Fatalf("Repeated - on command line; can't reread stdin.")
	}
	seenStdin = true
	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Split(bufio.ScanWords)
		for scanner.Scan() {
			s := strings.TrimSpace(scanner.Text())
			if s != "" {
				c <- s
			}
		}
		if err := scanner.Err(); err != nil {
			log.Fatalf("Error reading stdin: %s", err)
		}
		close(c)
	}()
	return c
}

// ReadAllStdin reads standard input in its entirety to a slice.
// Since this reads it completely before returning it won't handle a slow input
// very nicely. ReadStdin is therefore preferable when possible.
func ReadAllStdin() []string {
	var ret []string
	for s := range ReadStdin() {
		ret = append(ret, s)
	}
	return ret
}
<commit_msg>Allow blacklisting directories in complete paths from the top of the repo<commit_after>// Package utils contains various utility functions and whatnot.
package utils

import (
	"bufio"
	"os"
	"path"
	"path/filepath"
	"strings"

	"gopkg.in/op/go-logging.v1"

	"core"
)

var log = logging.MustGetLogger("utils")

// FindAllSubpackages finds all packages under a particular path.
// Used to implement rules with ... where we need to know all possible packages
// under that location.
func FindAllSubpackages(config *core.Configuration, rootPath string, prefix string) <-chan string {
	ch := make(chan string)
	go func() {
		if rootPath == "" {
			rootPath = "."
		}
		if err := filepath.Walk(rootPath, func(name string, info os.FileInfo, err error) error {
			if err != nil {
				return err // stop on any error
			} else if name == core.OutDir || (info.IsDir() && strings.HasPrefix(info.Name(), ".") && name != ".") {
				return filepath.SkipDir // Don't walk output or hidden directories
			} else if info.IsDir() && !strings.HasPrefix(name, prefix) && !strings.HasPrefix(prefix, name) {
				return filepath.SkipDir // Skip any directory without the prefix we're after (but not any directory beneath that)
			} else if isABuildFile(info.Name(), config) && !info.IsDir() {
				dir, _ := path.Split(name)
				ch <- strings.TrimRight(dir, "/")
			} else if name == config.Parse.ExperimentalDir {
				return filepath.SkipDir // Skip the experimental directory if it's set
			}
			// Check against blacklist
			for _, dir := range config.Parse.BlacklistDirs {
				if dir == info.Name() || strings.HasPrefix(name, dir) {
					return filepath.SkipDir
				}
			}
			return nil
		}); err != nil {
			log.Fatalf("Failed to walk tree under %s; %s\n", rootPath, err)
		}
		close(ch)
	}()
	return ch
}

var seenStdin = false // Used to track that we don't try to read stdin twice

// isABuildFile returns true if given filename is a build file name.
func isABuildFile(name string, config *core.Configuration) bool {
	for _, buildFileName := range config.Parse.BuildFileName {
		if name == buildFileName {
			return true
		}
	}
	return false
}

// ReadStdin reads a sequence of space-delimited words from standard input.
// Words are pushed onto the returned channel asynchronously.
func ReadStdin() <-chan string {
	c := make(chan string)
	if seenStdin {
		log.Fatalf("Repeated - on command line; can't reread stdin.")
	}
	seenStdin = true
	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		scanner.Split(bufio.ScanWords)
		for scanner.Scan() {
			s := strings.TrimSpace(scanner.Text())
			if s != "" {
				c <- s
			}
		}
		if err := scanner.Err(); err != nil {
			log.Fatalf("Error reading stdin: %s", err)
		}
		close(c)
	}()
	return c
}

// ReadAllStdin reads standard input in its entirety to a slice.
// Since this reads it completely before returning it won't handle a slow input
// very nicely. ReadStdin is therefore preferable when possible.
func ReadAllStdin() []string {
	var ret []string
	for s := range ReadStdin() {
		ret = append(ret, s)
	}
	return ret
}
<|endoftext|>