<commit_before>// Copyright 2015 Eleme Inc. All rights reserved.

package detector

import (
	"bufio"
	"fmt"
	"net"
	"path/filepath"
	"time"

	"github.com/eleme/banshee/config"
	"github.com/eleme/banshee/filter"
	"github.com/eleme/banshee/models"
	"github.com/eleme/banshee/storage"
	"github.com/eleme/banshee/storage/indexdb"
	"github.com/eleme/banshee/util/log"
)

// Timeout in milliseconds.
const timeout = 100

// Detector is to detect anomalies.
type Detector struct {
	cfg  *config.Config
	db   *storage.DB
	flt  *filter.Filter
	outs []chan *models.Metric
}

// New creates a detector.
func New(cfg *config.Config, db *storage.DB, flt *filter.Filter) *Detector {
	return &Detector{cfg, db, flt, make([]chan *models.Metric, 0)}
}

// Out adds a channel to receive detection results.
func (d *Detector) Out(ch chan *models.Metric) {
	d.outs = append(d.outs, ch)
}

// Output detected metrics to channels in outs, will skip if the target channel
// is full.
func (d *Detector) output(m *models.Metric) {
	for _, ch := range d.outs {
		select {
		case ch <- m:
		default:
			log.Error("output channel is full, skipping..")
			continue
		}
	}
}

// Start the tcp server.
func (d *Detector) Start() {
	// Listen
	addr := fmt.Sprintf("0.0.0.0:%d", d.cfg.Detector.Port)
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal("listen: %v", err)
	}
	log.Info("detector is listening on %s..", addr)
	// Accept
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Error("cannot accept conn: %v, skipping..", err)
			continue
		}
		go d.handle(conn)
	}
}

// Handle a new connection, it will:
//
//	1. Read input from the connection line by line.
//	2. Parse the lines into metrics.
//	3. Validate the metrics.
//
func (d *Detector) handle(conn net.Conn) {
	// New conn established.
	addr := conn.RemoteAddr()
	log.Info("conn %s established", addr)
	// Read
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		// Read line by line.
		if err := scanner.Err(); err != nil {
			// Close conn on read error.
			log.Error("read error: %v, closing conn..", err)
			break
		}
		line := scanner.Text()
		// Parse metric.
		m, err := parseMetric(line)
		if err != nil {
			// Skip invalid input.
			log.Error("parse error: %v, skipping..", err)
			continue
		}
		// Validate metric.
		if err := validateMetric(m); err != nil {
			log.Error("invalid metric: %v, skipping..", err)
			continue
		}
		// Process
		d.process(m)
	}
	// Close conn.
	conn.Close()
	log.Info("conn %s disconnected", addr)
}

// Process the input metric.
//
//	1. Match metric with rules.
//	2. Test with its matched rules and output it.
//
func (d *Detector) process(m *models.Metric) {
	// Time it.
	startAt := time.Now()
	// Match
	ok, rules := d.match(m)
	if !ok {
		// Not matched.
		return
	}
	// Detect
	err := d.detect(m)
	if err != nil {
		log.Error("detect: %v, skipping..", err)
		return
	}
	// Time end.
	elapsed := time.Since(startAt)
	// Test with rules.
	for _, rule := range rules {
		if rule.Test(m, d.cfg) {
			// Add tested ok rules.
			m.TestedRules = append(m.TestedRules, rule)
			log.Info("%dμs %s test ok", elapsed.Nanoseconds()/1000, m.Name)
		}
	}
	// Output result.
	if len(m.TestedRules) > 0 {
		// Out only if tested ok.
		d.output(m)
	}
}

// Match a metric with rules, and return matched rules.
//
//	If no rules matched, return false.
//	If any black patterns matched, return false.
//	Else, return true and matched rules.
//
func (d *Detector) match(m *models.Metric) (bool, []*models.Rule) {
	// Check rules.
	rules := d.flt.MatchedRules(m)
	if len(rules) == 0 {
		// Hit no rules.
		return false, rules
	}
	// Check blacklist.
	for _, p := range d.cfg.Detector.BlackList {
		ok, err := filepath.Match(p, m.Name)
		if err != nil {
			// Invalid black pattern.
			log.Error("invalid black pattern: %s, %v", p, err)
			continue
		}
		if ok {
			// Hit black pattern.
			log.Debug("%s hit black pattern %s", m.Name, p)
			return false, rules
		}
	}
	// Ok
	return true, rules
}

// Detect input metric with 3-sigma rule.
//
//	1. Get history values for this metric.
//	2. Get current index for this metric.
//	3. Calculate score via 3-sigma.
//	4. Get score trending via ewma.
//	5. Save the metric and index to db.
//
func (d *Detector) detect(m *models.Metric) error {
	// Get index.
	idx, err := d.db.Index.Get(m.Name)
	if err != nil {
		if err == indexdb.ErrNotFound {
			idx = nil
		} else {
			return err // unexcepted
		}
	}
	// Fill zero?
	fz := idx != nil && d.shouldFz(m)
	// History values.
	vals, err := d.values(m, fz)
	if err != nil {
		return err // unexcepted
	}
	// Apply 3-sigma.
	d.div3Sigma(m, vals)
	// Get new index.
	nIdx := d.nextIdx(idx, m)
	// Save
	if err := d.db.Index.Put(nIdx); err != nil {
		return err
	}
	if err := d.db.Metric.Put(m); err != nil {
		return err
	}
	return nil
}

// Test whether a metric need to fill blank with zeros to its history
// values.
func (d *Detector) shouldFz(m *models.Metric) bool {
	for _, p := range d.cfg.Detector.FillBlankZeros {
		ok, err := filepath.Match(p, m.Name)
		if err != nil {
			// Invalid pattern.
			log.Error("invalid fillBlankZeros pattern: %s, %v", p, err)
			continue
		}
		if ok {
			// Ok.
			return true
		}
	}
	// No need.
	return false
}

// Fill blank with zeros into history values, mainly for dispersed
// metrics such as counters. The start and stop is for periodicity
// reasons.
func (d *Detector) fill0(ms []*models.Metric, start, stop uint32) []float64 {
	i := 0 // record real-metric.
	step := d.cfg.Interval
	var vals []float64
	for start < stop {
		if i < len(ms) {
			m := ms[i]
			// start is smaller than current stamp.
			for ; start < m.Stamp; start += step {
				vals = append(vals, 0)
			}
			// Push real-metric.
			vals = append(vals, m.Value)
			i++
		} else {
			// No more real-metric.
			vals = append(vals, 0)
		}
		start += step
	}
	return vals
}

// Get history values for the input metric, will only fetch the history
// values with the same phase around this timestamp, within an filter
// offset.
func (d *Detector) values(m *models.Metric, fz bool) ([]float64, error) {
	offset := uint32(d.cfg.Detector.FilterOffset * float64(d.cfg.Period))
	expiration := d.cfg.Expiration
	period := d.cfg.Period
	var vals []float64
	// Get values with the same phase.
	for stamp := m.Stamp; stamp+expiration > m.Stamp; stamp -= period {
		start := stamp - offset
		stop := stamp + offset
		ms, err := d.db.Metric.Get(m.Name, start, stop)
		if err != nil {
			// Unexcepted db error.
			return vals, err
		}
		if !fz {
			for i := 0; i < len(ms); i++ {
				vals = append(vals, ms[i].Value)
			}
		} else {
			// Fill blank with zeros.
			vals = append(vals, d.fill0(ms, start, stop)...)
		}
	}
	// Append m
	vals = append(vals, m.Value)
	return vals, nil
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func (d *Detector) div3Sigma(m *models.Metric, vals []float64) {
	if len(vals) == 0 {
		// Values empty.
		m.Score = 0
		m.Average = m.Value
		return
	}
	// Values average and standard deviation.
	avg := average(vals)
	std := stdDev(vals, avg)
	// Set metric average
	m.Average = avg
	// Set metric score
	if len(vals) <= int(d.cfg.Detector.LeastCount) {
		// Values not enough.
		m.Score = 0
		return
	}
	last := vals[len(vals)-1]
	if std == 0 {
		switch {
		case last == avg:
			m.Score = 0
		case last > avg:
			m.Score = 1
		case last < avg:
			m.Score = -1
		}
		return
	}
	// 3-sigma
	m.Score = (last - avg) / (3 * std)
}

// Calculate next score for index via ewma, called the weighted exponential
// moving average.
//
//	t[0] = x[1], f: 0~1
//	t[n] = t[n-1] * (1 - f) + f * x[n]
//
// The index score will follow the metric's score, and additionally the index
// average is the latest metric average.
//
// Index score is the trending description of metric score.
//
func (d *Detector) nextIdx(idx *models.Index, m *models.Metric) *models.Index {
	n := &models.Index{Name: m.Name, Stamp: m.Stamp}
	if idx == nil {
		// As first
		n.Score = m.Score
		n.Average = m.Value
		return n
	}
	// Move next
	f := d.cfg.Detector.TrendingFactor
	n.Score = idx.Score*(1-f) + f*m.Score
	n.Average = m.Average
	return n
}
<commit_msg>Log warn on slow detection<commit_after>// Copyright 2015 Eleme Inc. All rights reserved.

package detector

import (
	"bufio"
	"fmt"
	"net"
	"path/filepath"
	"time"

	"github.com/eleme/banshee/config"
	"github.com/eleme/banshee/filter"
	"github.com/eleme/banshee/models"
	"github.com/eleme/banshee/storage"
	"github.com/eleme/banshee/storage/indexdb"
	"github.com/eleme/banshee/util/log"
)

// Timeout in milliseconds.
const timeout = 100

// Detector is to detect anomalies.
type Detector struct {
	cfg  *config.Config
	db   *storage.DB
	flt  *filter.Filter
	outs []chan *models.Metric
}

// New creates a detector.
func New(cfg *config.Config, db *storage.DB, flt *filter.Filter) *Detector {
	return &Detector{cfg, db, flt, make([]chan *models.Metric, 0)}
}

// Out adds a channel to receive detection results.
func (d *Detector) Out(ch chan *models.Metric) {
	d.outs = append(d.outs, ch)
}

// Output detected metrics to channels in outs, will skip if the target channel
// is full.
func (d *Detector) output(m *models.Metric) {
	for _, ch := range d.outs {
		select {
		case ch <- m:
		default:
			log.Error("output channel is full, skipping..")
			continue
		}
	}
}

// Start the tcp server.
func (d *Detector) Start() {
	// Listen
	addr := fmt.Sprintf("0.0.0.0:%d", d.cfg.Detector.Port)
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal("listen: %v", err)
	}
	log.Info("detector is listening on %s..", addr)
	// Accept
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Error("cannot accept conn: %v, skipping..", err)
			continue
		}
		go d.handle(conn)
	}
}

// Handle a new connection, it will:
//
//	1. Read input from the connection line by line.
//	2. Parse the lines into metrics.
//	3. Validate the metrics.
//
func (d *Detector) handle(conn net.Conn) {
	// New conn established.
	addr := conn.RemoteAddr()
	log.Info("conn %s established", addr)
	// Read
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		// Read line by line.
		if err := scanner.Err(); err != nil {
			// Close conn on read error.
			log.Error("read error: %v, closing conn..", err)
			break
		}
		line := scanner.Text()
		// Parse metric.
		m, err := parseMetric(line)
		if err != nil {
			// Skip invalid input.
			log.Error("parse error: %v, skipping..", err)
			continue
		}
		// Validate metric.
		if err := validateMetric(m); err != nil {
			log.Error("invalid metric: %v, skipping..", err)
			continue
		}
		// Process
		d.process(m)
	}
	// Close conn.
	conn.Close()
	log.Info("conn %s disconnected", addr)
}

// Process the input metric.
//
//	1. Match metric with rules.
//	2. Test with its matched rules and output it.
//
func (d *Detector) process(m *models.Metric) {
	// Time it.
	startAt := time.Now()
	// Match
	ok, rules := d.match(m)
	if !ok {
		// Not matched.
		return
	}
	// Detect
	err := d.detect(m)
	if err != nil {
		log.Error("detect: %v, skipping..", err)
		return
	}
	// Time end.
	elapsed := time.Since(startAt).Nanoseconds() / (1000 * 1000)
	if elapsed > timeout {
		log.Warn("detection is slow: %dms", elapsed)
	}
	// Test with rules.
	for _, rule := range rules {
		if rule.Test(m, d.cfg) {
			// Add tested ok rules.
			m.TestedRules = append(m.TestedRules, rule)
			log.Info("%dms %s test ok", elapsed, m.Name)
		}
	}
	// Output result.
	if len(m.TestedRules) > 0 {
		// Out only if tested ok.
		d.output(m)
	}
}

// Match a metric with rules, and return matched rules.
//
//	If no rules matched, return false.
//	If any black patterns matched, return false.
//	Else, return true and matched rules.
//
func (d *Detector) match(m *models.Metric) (bool, []*models.Rule) {
	// Check rules.
	rules := d.flt.MatchedRules(m)
	if len(rules) == 0 {
		// Hit no rules.
		return false, rules
	}
	// Check blacklist.
	for _, p := range d.cfg.Detector.BlackList {
		ok, err := filepath.Match(p, m.Name)
		if err != nil {
			// Invalid black pattern.
			log.Error("invalid black pattern: %s, %v", p, err)
			continue
		}
		if ok {
			// Hit black pattern.
			log.Debug("%s hit black pattern %s", m.Name, p)
			return false, rules
		}
	}
	// Ok
	return true, rules
}

// Detect input metric with 3-sigma rule.
//
//	1. Get history values for this metric.
//	2. Get current index for this metric.
//	3. Calculate score via 3-sigma.
//	4. Get score trending via ewma.
//	5. Save the metric and index to db.
//
func (d *Detector) detect(m *models.Metric) error {
	// Get index.
	idx, err := d.db.Index.Get(m.Name)
	if err != nil {
		if err == indexdb.ErrNotFound {
			idx = nil
		} else {
			return err // unexcepted
		}
	}
	// Fill zero?
	fz := idx != nil && d.shouldFz(m)
	// History values.
	vals, err := d.values(m, fz)
	if err != nil {
		return err // unexcepted
	}
	// Apply 3-sigma.
	d.div3Sigma(m, vals)
	// Get new index.
	nIdx := d.nextIdx(idx, m)
	// Save
	if err := d.db.Index.Put(nIdx); err != nil {
		return err
	}
	if err := d.db.Metric.Put(m); err != nil {
		return err
	}
	return nil
}

// Test whether a metric need to fill blank with zeros to its history
// values.
func (d *Detector) shouldFz(m *models.Metric) bool {
	for _, p := range d.cfg.Detector.FillBlankZeros {
		ok, err := filepath.Match(p, m.Name)
		if err != nil {
			// Invalid pattern.
			log.Error("invalid fillBlankZeros pattern: %s, %v", p, err)
			continue
		}
		if ok {
			// Ok.
			return true
		}
	}
	// No need.
	return false
}

// Fill blank with zeros into history values, mainly for dispersed
// metrics such as counters. The start and stop is for periodicity
// reasons.
func (d *Detector) fill0(ms []*models.Metric, start, stop uint32) []float64 {
	i := 0 // record real-metric.
	step := d.cfg.Interval
	var vals []float64
	for start < stop {
		if i < len(ms) {
			m := ms[i]
			// start is smaller than current stamp.
			for ; start < m.Stamp; start += step {
				vals = append(vals, 0)
			}
			// Push real-metric.
			vals = append(vals, m.Value)
			i++
		} else {
			// No more real-metric.
			vals = append(vals, 0)
		}
		start += step
	}
	return vals
}

// Get history values for the input metric, will only fetch the history
// values with the same phase around this timestamp, within an filter
// offset.
func (d *Detector) values(m *models.Metric, fz bool) ([]float64, error) {
	offset := uint32(d.cfg.Detector.FilterOffset * float64(d.cfg.Period))
	expiration := d.cfg.Expiration
	period := d.cfg.Period
	var vals []float64
	// Get values with the same phase.
	for stamp := m.Stamp; stamp+expiration > m.Stamp; stamp -= period {
		start := stamp - offset
		stop := stamp + offset
		ms, err := d.db.Metric.Get(m.Name, start, stop)
		if err != nil {
			// Unexcepted db error.
			return vals, err
		}
		if !fz {
			for i := 0; i < len(ms); i++ {
				vals = append(vals, ms[i].Value)
			}
		} else {
			// Fill blank with zeros.
			vals = append(vals, d.fill0(ms, start, stop)...)
		}
	}
	// Append m
	vals = append(vals, m.Value)
	return vals, nil
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func (d *Detector) div3Sigma(m *models.Metric, vals []float64) {
	if len(vals) == 0 {
		// Values empty.
		m.Score = 0
		m.Average = m.Value
		return
	}
	// Values average and standard deviation.
	avg := average(vals)
	std := stdDev(vals, avg)
	// Set metric average
	m.Average = avg
	// Set metric score
	if len(vals) <= int(d.cfg.Detector.LeastCount) {
		// Values not enough.
		m.Score = 0
		return
	}
	last := vals[len(vals)-1]
	if std == 0 {
		switch {
		case last == avg:
			m.Score = 0
		case last > avg:
			m.Score = 1
		case last < avg:
			m.Score = -1
		}
		return
	}
	// 3-sigma
	m.Score = (last - avg) / (3 * std)
}

// Calculate next score for index via ewma, called the weighted exponential
// moving average.
//
//	t[0] = x[1], f: 0~1
//	t[n] = t[n-1] * (1 - f) + f * x[n]
//
// The index score will follow the metric's score, and additionally the index
// average is the latest metric average.
//
// Index score is the trending description of metric score.
//
func (d *Detector) nextIdx(idx *models.Index, m *models.Metric) *models.Index {
	n := &models.Index{Name: m.Name, Stamp: m.Stamp}
	if idx == nil {
		// As first
		n.Score = m.Score
		n.Average = m.Value
		return n
	}
	// Move next
	f := d.cfg.Detector.TrendingFactor
	n.Score = idx.Score*(1-f) + f*m.Score
	n.Average = m.Average
	return n
}
<|endoftext|>


<commit_before>package arn

import (
	"io/ioutil"
	"os"
	"path"

	"github.com/animenotifier/anilist"
	"github.com/animenotifier/osu"
	jsoniter "github.com/json-iterator/go"
)

// Root is the full path to the root directory of notify.moe repository.
var Root = path.Join(os.Getenv("GOPATH"), "src/github.com/animenotifier/notify.moe")

// APIKeys are global API keys for several services
var APIKeys APIKeysData

// APIKeysData ...
type APIKeysData struct {
	Google struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"google"`

	Facebook struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"facebook"`

	Twitter struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"twitter"`

	Discord struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
		Token  string `json:"token"`
	} `json:"discord"`

	SoundCloud struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"soundcloud"`

	GoogleAPI struct {
		Key string `json:"key"`
	} `json:"googleAPI"`

	IPInfoDB struct {
		ID string `json:"id"`
	} `json:"ipInfoDB"`

	AniList struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"anilist"`

	Osu struct {
		Secret string `json:"secret"`
	} `json:"osu"`

	PayPal struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"paypal"`

	VAPID struct {
		Subject    string `json:"subject"`
		PublicKey  string `json:"publicKey"`
		PrivateKey string `json:"privateKey"`
	} `json:"vapid"`

	SMTP struct {
		Server   string `json:"server"`
		Address  string `json:"address"`
		Password string `json:"password"`
	} `json:"smtp"`

	S3 struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"s3"`
}

func init() {
	// Path for API keys
	apiKeysPath := path.Join(Root, "security/api-keys.json")

	// If the API keys file is not available, create a symlink to the default API keys
	if _, err := os.Stat(apiKeysPath); os.IsNotExist(err) {
		defaultAPIKeysPath := path.Join(Root, "security/default/api-keys.json")
		err := os.Link(defaultAPIKeysPath, apiKeysPath)

		if err != nil {
			panic(err)
		}
	}

	// Load API keys
	data, err := ioutil.ReadFile(apiKeysPath)

	if err != nil {
		panic(err)
	}

	// Parse JSON
	err = jsoniter.Unmarshal(data, &APIKeys)

	if err != nil {
		panic(err)
	}

	// Set Osu API key
	osu.APIKey = APIKeys.Osu.Secret

	// Set Anilist API keys
	anilist.APIKeyID = APIKeys.AniList.ID
	anilist.APIKeySecret = APIKeys.AniList.Secret
}
<commit_msg>Use ARN_ROOT environment variable<commit_after>package arn

import (
	"io/ioutil"
	"os"
	"path"

	"github.com/animenotifier/anilist"
	"github.com/animenotifier/osu"
	jsoniter "github.com/json-iterator/go"
)

// Root is the full path to the root directory of notify.moe repository.
var Root = os.Getenv("ARN_ROOT")

// APIKeys are global API keys for several services
var APIKeys APIKeysData

// APIKeysData ...
type APIKeysData struct {
	Google struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"google"`

	Facebook struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"facebook"`

	Twitter struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"twitter"`

	Discord struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
		Token  string `json:"token"`
	} `json:"discord"`

	SoundCloud struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"soundcloud"`

	GoogleAPI struct {
		Key string `json:"key"`
	} `json:"googleAPI"`

	IPInfoDB struct {
		ID string `json:"id"`
	} `json:"ipInfoDB"`

	AniList struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"anilist"`

	Osu struct {
		Secret string `json:"secret"`
	} `json:"osu"`

	PayPal struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"paypal"`

	VAPID struct {
		Subject    string `json:"subject"`
		PublicKey  string `json:"publicKey"`
		PrivateKey string `json:"privateKey"`
	} `json:"vapid"`

	SMTP struct {
		Server   string `json:"server"`
		Address  string `json:"address"`
		Password string `json:"password"`
	} `json:"smtp"`

	S3 struct {
		ID     string `json:"id"`
		Secret string `json:"secret"`
	} `json:"s3"`
}

func init() {
	// Path for API keys
	apiKeysPath := path.Join(Root, "security/api-keys.json")

	// If the API keys file is not available, create a symlink to the default API keys
	if _, err := os.Stat(apiKeysPath); os.IsNotExist(err) {
		defaultAPIKeysPath := path.Join(Root, "security/default/api-keys.json")
		err := os.Link(defaultAPIKeysPath, apiKeysPath)

		if err != nil {
			panic(err)
		}
	}

	// Load API keys
	data, err := ioutil.ReadFile(apiKeysPath)

	if err != nil {
		panic(err)
	}

	// Parse JSON
	err = jsoniter.Unmarshal(data, &APIKeys)

	if err != nil {
		panic(err)
	}

	// Set Osu API key
	osu.APIKey = APIKeys.Osu.Secret

	// Set Anilist API keys
	anilist.APIKeyID = APIKeys.AniList.ID
	anilist.APIKeySecret = APIKeys.AniList.Secret
}
<|endoftext|>


<commit_before>package b5

// Permutation5Inc generates all permutations of s,
// and apply f to each of them.
func Permutation5Inc(s []uint, f func([]uint)) {
	n := len(s)
	a := make([]int, n)
	for d := 0; d < n; d++ {
		a[d] = d
	}
	d := 0
	// This condition is the same as:
	//     !(d == 0 && a[d] == n)
	for d != 0 || a[d] != n {
		if a[d] < n {
			if d < n-1 {
				s[a[d]], s[d] = s[d], s[a[d]]
				d++
				continue
			}
			// The following sections refers to d == n
			f(s)
			a[d]++
			continue
		}
		// The following section refers to a[d] == n:
		a[d] = d
		d--
		s[a[d]], s[d] = s[d], s[a[d]]
		a[d]++
	}
}
<commit_msg>Fixed the bug<commit_after>package b5

// Permutation5Inc generates all permutations of s,
// and apply f to each of them.
func Permutation5Inc(s []uint, f func([]uint)) {
	n := len(s)
	if n == 0 {
		f(s)
		return
	}
	a := make([]int, n)
	for d := 0; d < n; d++ {
		a[d] = d
	}
	d := 0
	// This condition is the same as:
	//     !(d == 0 && a[d] == n)
	for d != 0 || a[d] != n {
		if a[d] < n {
			if d < n-1 {
				s[a[d]], s[d] = s[d], s[a[d]]
				d++
				continue
			}
			// The following sections refers to d == n
			f(s)
			a[d]++
			continue
		}
		// The following section refers to a[d] == n:
		a[d] = d
		d--
		s[a[d]], s[d] = s[d], s[a[d]]
		a[d]++
	}
}
<|endoftext|>


<commit_before>package wikipediapageview

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	hd "github.com/nownabe/cloud-demos/api/pkg/handler"
)

type postQueriesRequest struct {
	Wiki       string    `json:"wiki"`
	TitleLike  string    `json:"title_like"`
	StartDate  time.Time `json:"start_date"`
	EndDate    time.Time `json:"end_date"`
	OrderBy    string    `json:"order_by"`
	GroupBy    string    `json:"group_by"`
	QueryCache bool      `json:"query_cache"`
}

type postQueriesResponse struct {
}

func (h *handler) postQueries(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req := &postQueriesRequest{}
	if err := hd.DecodeJSONBody(r, req); err != nil {
		hd.RespondErrorJSON(w, r, err)
		return
	}

	if req.OrderBy != "asc" && req.OrderBy != "desc" {
		hd.RespondErrorMessage(w, r,
			http.StatusBadRequest,
			"order_by can be 'asc' or 'desc'")
		return
	}

	q := h.buildQuery(req)
	q.DisableQueryCache = !req.QueryCache

	job, err := q.Run(ctx)
	if err != nil {
		hd.RespondErrorJSON(w, r, hd.Errorf(ctx,
			http.StatusInternalServerError,
			http.StatusText(http.StatusInternalServerError),
			"failed to run query: %w", err))
	}

	hd.RespondJSON(w, r, http.StatusOK, job.ID())
}

func (h *handler) buildQuery(req *postQueriesRequest) *bigquery.Query {
	var q *bigquery.Query

	cond := []string{"datehour >= @startDate", "datehour < @endDate"}
	if req.Wiki != "" {
		cond = append(cond, "wiki = @wiki")
	}
	if req.TitleLike != "" {
		cond = append(cond, "title LIKE @titleLike")
	}

	if req.GroupBy == "title" {
		q = h.Bigquery.Query(fmt.Sprintf(`SELECT
	wiki,
	title,
	SUM(views) AS views
FROM bigquery-public-data.wikipedia.pageviews_2020
WHERE %s
GROUP BY wiki, title
ORDER BY views %s
LIMIT 100`, strings.Join(cond, " AND "), req.OrderBy))
	} else {
		q = h.Bigquery.Query(fmt.Sprintf(`SELECT
	DATE(datehour) AS date,
	SUM(views) AS views
FROM bigquery-public-data.wikipedia.pageviews_2020
WHERE %s
GROUP BY date
ORDER BY views %s
LIMIT 366`, strings.Join(cond, " AND "), req.OrderBy))
	}

	q.Parameters = []bigquery.QueryParameter{
		{Name: "wiki", Value: req.Wiki},
		{Name: "titleLike", Value: "%" + req.TitleLike + "%"},
		{Name: "startDate", Value: req.StartDate},
		{Name: "endDate", Value: req.EndDate.AddDate(0, 0, 1)},
	}

	return q
}
<commit_msg>:sparkles: [api] Return JSON<commit_after>package wikipediapageview

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"

	hd "github.com/nownabe/cloud-demos/api/pkg/handler"
)

type postQueriesRequest struct {
	Wiki       string    `json:"wiki"`
	TitleLike  string    `json:"title_like"`
	StartDate  time.Time `json:"start_date"`
	EndDate    time.Time `json:"end_date"`
	OrderBy    string    `json:"order_by"`
	GroupBy    string    `json:"group_by"`
	QueryCache bool      `json:"query_cache"`
}

type postQueriesResponse struct {
	JobID string `json:"job_id"`
}

func (h *handler) postQueries(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req := &postQueriesRequest{}
	if err := hd.DecodeJSONBody(r, req); err != nil {
		hd.RespondErrorJSON(w, r, err)
		return
	}

	if req.OrderBy != "asc" && req.OrderBy != "desc" {
		hd.RespondErrorMessage(w, r,
			http.StatusBadRequest,
			"order_by can be 'asc' or 'desc'")
		return
	}

	q := h.buildQuery(req)
	q.DisableQueryCache = !req.QueryCache

	job, err := q.Run(ctx)
	if err != nil {
		hd.RespondErrorJSON(w, r, hd.Errorf(ctx,
			http.StatusInternalServerError,
			http.StatusText(http.StatusInternalServerError),
			"failed to run query: %w", err))
	}

	hd.RespondJSON(w, r, http.StatusOK, &postQueriesResponse{JobID: job.ID()})
}

func (h *handler) buildQuery(req *postQueriesRequest) *bigquery.Query {
	var q *bigquery.Query

	cond := []string{"datehour >= @startDate", "datehour < @endDate"}
	if req.Wiki != "" {
		cond = append(cond, "wiki = @wiki")
	}
	if req.TitleLike != "" {
		cond = append(cond, "title LIKE @titleLike")
	}

	if req.GroupBy == "title" {
		q = h.Bigquery.Query(fmt.Sprintf(`SELECT
	wiki,
	title,
	SUM(views) AS views
FROM bigquery-public-data.wikipedia.pageviews_2020
WHERE %s
GROUP BY wiki, title
ORDER BY views %s
LIMIT 100`, strings.Join(cond, " AND "), req.OrderBy))
	} else {
		q = h.Bigquery.Query(fmt.Sprintf(`SELECT
	DATE(datehour) AS date,
	SUM(views) AS views
FROM bigquery-public-data.wikipedia.pageviews_2020
WHERE %s
GROUP BY date
ORDER BY views %s
LIMIT 366`, strings.Join(cond, " AND "), req.OrderBy))
	}

	q.Parameters = []bigquery.QueryParameter{
		{Name: "wiki", Value: req.Wiki},
		{Name: "titleLike", Value: "%" + req.TitleLike + "%"},
		{Name: "startDate", Value: req.StartDate},
		{Name: "endDate", Value: req.EndDate.AddDate(0, 0, 1)},
	}

	return q
}
<|endoftext|>


<commit_before>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha2

import (
	"fmt"

	"kubedb.dev/apimachinery/apis"
	"kubedb.dev/apimachinery/apis/kubedb"
	"kubedb.dev/apimachinery/crds"

	"gomodules.xyz/pointer"
	core "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	appslister "k8s.io/client-go/listers/apps/v1"
	kmapi "kmodules.xyz/client-go/api/v1"
	"kmodules.xyz/client-go/apiextensions"
	meta_util "kmodules.xyz/client-go/meta"
	appcat "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
	mona "kmodules.xyz/monitoring-agent-api/api/v1"
)

func (_ MySQL) CustomResourceDefinition() *apiextensions.CustomResourceDefinition {
	return crds.MustCustomResourceDefinition(SchemeGroupVersion.WithResource(ResourcePluralMySQL))
}

var _ apis.ResourceInfo = &MySQL{}

func (m MySQL) OffshootName() string {
	return m.Name
}

func (m MySQL) OffshootSelectors() map[string]string {
	return map[string]string{
		meta_util.NameLabelKey:      m.ResourceFQN(),
		meta_util.InstanceLabelKey:  m.Name,
		meta_util.ManagedByLabelKey: kubedb.GroupName,
	}
}

func (m MySQL) OffshootLabels() map[string]string {
	out := m.OffshootSelectors()
	out[meta_util.ComponentLabelKey] = ComponentDatabase
	return meta_util.FilterKeys(kubedb.GroupName, out, m.Labels)
}

func (m MySQL) ResourceFQN() string {
	return fmt.Sprintf("%s.%s", ResourcePluralMySQL, kubedb.GroupName)
}

func (m MySQL) ResourceShortCode() string {
	return ResourceCodeMySQL
}

func (m MySQL) ResourceKind() string {
	return ResourceKindMySQL
}

func (m MySQL) ResourceSingular() string {
	return ResourceSingularMySQL
}

func (m MySQL) ResourcePlural() string {
	return ResourcePluralMySQL
}

func (m MySQL) ServiceName() string {
	return m.OffshootName()
}

func (m MySQL) StandbyServiceName() string {
	return meta_util.NameWithPrefix(m.ServiceName(), "standby")
}

func (m MySQL) GoverningServiceName() string {
	return meta_util.NameWithSuffix(m.ServiceName(), "pods")
}

func (m MySQL) PrimaryServiceDNS() string {
	return fmt.Sprintf("%s.%s.svc", m.ServiceName(), m.Namespace)
}

func (m MySQL) PeerName(idx int) string {
	return fmt.Sprintf("%s-%d.%s.%s", m.OffshootName(), idx, m.GoverningServiceName(), m.Namespace)
}

func (m MySQL) GetAuthSecretName() string {
	return m.Spec.AuthSecret.Name
}

type mysqlApp struct {
	*MySQL
}

func (r mysqlApp) Name() string {
	return r.MySQL.Name
}

func (r mysqlApp) Type() appcat.AppType {
	return appcat.AppType(fmt.Sprintf("%s/%s", kubedb.GroupName, ResourceSingularMySQL))
}

func (m MySQL) AppBindingMeta() appcat.AppBindingMeta {
	return &mysqlApp{&m}
}

type mysqlStatsService struct {
	*MySQL
}

func (m mysqlStatsService) GetNamespace() string {
	return m.MySQL.GetNamespace()
}

func (m mysqlStatsService) ServiceName() string {
	return m.OffshootName() + "-stats"
}

func (m mysqlStatsService) ServiceMonitorName() string {
	return m.ServiceName()
}

func (m mysqlStatsService) ServiceMonitorAdditionalLabels() map[string]string {
	return m.OffshootLabels()
}

func (m mysqlStatsService) Path() string {
	return DefaultStatsPath
}

func (m mysqlStatsService) Scheme() string {
	return ""
}

func (m MySQL) StatsService() mona.StatsAccessor {
	return &mysqlStatsService{&m}
}

func (m MySQL) StatsServiceLabels() map[string]string {
	lbl := meta_util.FilterKeys(kubedb.GroupName, m.OffshootSelectors(), m.Labels)
	lbl[LabelRole] = RoleStats
	return lbl
}

func (m *MySQL) UsesGroupReplication() bool {
	return m.Spec.Topology != nil && m.Spec.Topology.Mode != nil && *m.Spec.Topology.Mode == MySQLClusterModeGroup
}

func (m *MySQL) SetDefaults() {
	if m == nil {
		return
	}
	if m.Spec.StorageType == "" {
		m.Spec.StorageType = StorageTypeDurable
	}
	if m.Spec.TerminationPolicy == "" {
		m.Spec.TerminationPolicy = TerminationPolicyDelete
	}

	if m.UsesGroupReplication() {
		if m.Spec.Replicas == nil {
			m.Spec.Replicas = pointer.Int32P(MySQLDefaultGroupSize)
		}
		m.Spec.setDefaultProbes()
	} else {
		if m.Spec.Replicas == nil {
			m.Spec.Replicas = pointer.Int32P(1)
		}
	}

	if m.Spec.PodTemplate.Spec.ServiceAccountName == "" {
		m.Spec.PodTemplate.Spec.ServiceAccountName = m.OffshootName()
	}

	m.Spec.Monitor.SetDefaults()

	m.SetTLSDefaults()
	setDefaultResourceLimits(&m.Spec.PodTemplate.Spec.Resources, defaultResourceLimits, defaultResourceLimits)
}

func (m *MySQL) SetTLSDefaults() {
	if m.Spec.TLS == nil || m.Spec.TLS.IssuerRef == nil {
		return
	}
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLServerCert), m.CertificateName(MySQLServerCert))
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLClientCert), m.CertificateName(MySQLClientCert))
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLMetricsExporterCert), m.CertificateName(MySQLMetricsExporterCert))
}

// setDefaultProbes sets defaults only when probe fields are nil.
// In operator, check if the value of probe fields is "{}".
// For "{}", ignore readinessprobe or livenessprobe in statefulset.
// Ref: https://github.com/mattlord/Docker-InnoDB-Cluster/blob/master/healthcheck.sh#L10
func (m *MySQLSpec) setDefaultProbes() {
	probe := &core.Probe{
		Handler: core.Handler{
			Exec: &core.ExecAction{
				Command: []string{
					"bash",
					"-c",
					`
export MYSQL_PWD=${MYSQL_ROOT_PASSWORD}
mysql -h localhost -nsLNE -e "select member_state from performance_schema.replication_group_members where member_id=@@server_uuid;" 2>/dev/null | grep "ONLINE"
`,
				},
			},
		},
		InitialDelaySeconds: 30,
		PeriodSeconds:       5,
	}

	if m.PodTemplate.Spec.LivenessProbe == nil {
		m.PodTemplate.Spec.LivenessProbe = probe
	}
	if m.PodTemplate.Spec.ReadinessProbe == nil {
		m.PodTemplate.Spec.ReadinessProbe = probe
	}
}

func (m *MySQLSpec) GetPersistentSecrets() []string {
	if m == nil {
		return nil
	}

	var secrets []string
	if m.AuthSecret != nil {
		secrets = append(secrets, m.AuthSecret.Name)
	}
	return secrets
}

// CertificateName returns the default certificate name and/or certificate secret name for a certificate alias
func (m *MySQL) CertificateName(alias MySQLCertificateAlias) string {
	return meta_util.NameWithSuffix(m.Name, fmt.Sprintf("%s-cert", string(alias)))
}

// MustCertSecretName returns the secret name for a certificate alias
func (m *MySQL) MustCertSecretName(alias MySQLCertificateAlias) string {
	if m == nil {
		panic("missing MySQL database")
	} else if m.Spec.TLS == nil {
		panic(fmt.Errorf("MySQL %s/%s is missing tls spec", m.Namespace, m.Name))
	}
	name, ok := kmapi.GetCertificateSecretName(m.Spec.TLS.Certificates, string(alias))
	if !ok {
		panic(fmt.Errorf("MySQL %s/%s is missing secret name for %s certificate", m.Namespace, m.Name, alias))
	}
	return name
}

func (m *MySQL) ReplicasAreReady(lister appslister.StatefulSetLister) (bool, string, error) {
	// Desire number of statefulSets
	expectedItems := 1
	return checkReplicas(lister.StatefulSets(m.Namespace), labels.SelectorFromSet(m.OffshootLabels()), expectedItems)
}

func MySQLRequireSSLArg() string {
	return "--require-secure-transport=ON"
}

func MySQLExporterTLSArg() string {
	return "--config.my-cnf=/etc/mysql/certs/exporter.cnf"
}

func (m *MySQL) MySQLTLSArgs() []string {
	tlsArgs := []string{
		"--ssl-capath=/etc/mysql/certs",
		"--ssl-ca=/etc/mysql/certs/ca.crt",
		"--ssl-cert=/etc/mysql/certs/server.crt",
		"--ssl-key=/etc/mysql/certs/server.key",
	}
	if m.Spec.RequireSSL {
		tlsArgs = append(tlsArgs, MySQLRequireSSLArg())
	}
	return tlsArgs
}
<commit_msg>Add Hosts helper for MySQL (#679)<commit_after>/*
Copyright AppsCode Inc. and Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha2

import (
	"fmt"

	"kubedb.dev/apimachinery/apis"
	"kubedb.dev/apimachinery/apis/kubedb"
	"kubedb.dev/apimachinery/crds"

	"gomodules.xyz/pointer"
	core "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/labels"
	appslister "k8s.io/client-go/listers/apps/v1"
	kmapi "kmodules.xyz/client-go/api/v1"
	"kmodules.xyz/client-go/apiextensions"
	meta_util "kmodules.xyz/client-go/meta"
	appcat "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
	mona "kmodules.xyz/monitoring-agent-api/api/v1"
)

func (_ MySQL) CustomResourceDefinition() *apiextensions.CustomResourceDefinition {
	return crds.MustCustomResourceDefinition(SchemeGroupVersion.WithResource(ResourcePluralMySQL))
}

var _ apis.ResourceInfo = &MySQL{}

func (m MySQL) OffshootName() string {
	return m.Name
}

func (m MySQL) OffshootSelectors() map[string]string {
	return map[string]string{
		meta_util.NameLabelKey:      m.ResourceFQN(),
		meta_util.InstanceLabelKey:  m.Name,
		meta_util.ManagedByLabelKey: kubedb.GroupName,
	}
}

func (m MySQL) OffshootLabels() map[string]string {
	out := m.OffshootSelectors()
	out[meta_util.ComponentLabelKey] = ComponentDatabase
	return meta_util.FilterKeys(kubedb.GroupName, out, m.Labels)
}

func (m MySQL) ResourceFQN() string {
	return fmt.Sprintf("%s.%s", ResourcePluralMySQL, kubedb.GroupName)
}

func (m MySQL) ResourceShortCode() string {
	return ResourceCodeMySQL
}

func (m MySQL) ResourceKind() string {
	return ResourceKindMySQL
}

func (m MySQL) ResourceSingular() string {
	return ResourceSingularMySQL
}

func (m MySQL) ResourcePlural() string {
	return ResourcePluralMySQL
}

func (m MySQL) ServiceName() string {
	return m.OffshootName()
}

func (m MySQL) StandbyServiceName() string {
	return meta_util.NameWithPrefix(m.ServiceName(), "standby")
}

func (m MySQL) GoverningServiceName() string {
	return meta_util.NameWithSuffix(m.ServiceName(), "pods")
}

func (m MySQL) PrimaryServiceDNS() string {
	return fmt.Sprintf("%s.%s.svc", m.ServiceName(), m.Namespace)
}

func (m MySQL) Hosts() []string {
	replicas := 1
	if m.Spec.Replicas != nil {
		replicas = int(*m.Spec.Replicas)
	}
	hosts := make([]string, replicas)
	for i := 0; i < replicas; i++ {
		hosts[i] = fmt.Sprintf("%v-%d.%v.%v.svc", m.Name, i, m.GoverningServiceName(), m.Namespace)
	}
	return hosts
}

func (m MySQL) PeerName(idx int) string {
	return fmt.Sprintf("%s-%d.%s.%s", m.OffshootName(), idx, m.GoverningServiceName(), m.Namespace)
}

func (m MySQL) GetAuthSecretName() string {
	return m.Spec.AuthSecret.Name
}

type mysqlApp struct {
	*MySQL
}

func (r mysqlApp) Name() string {
	return r.MySQL.Name
}

func (r mysqlApp) Type() appcat.AppType {
	return appcat.AppType(fmt.Sprintf("%s/%s", kubedb.GroupName, ResourceSingularMySQL))
}

func (m MySQL) AppBindingMeta() appcat.AppBindingMeta {
	return &mysqlApp{&m}
}

type mysqlStatsService struct {
	*MySQL
}

func (m mysqlStatsService) GetNamespace() string {
	return m.MySQL.GetNamespace()
}

func (m mysqlStatsService) ServiceName() string {
	return m.OffshootName() + "-stats"
}

func (m mysqlStatsService) ServiceMonitorName() string {
	return m.ServiceName()
}

func (m mysqlStatsService) ServiceMonitorAdditionalLabels() map[string]string {
	return m.OffshootLabels()
}

func (m mysqlStatsService) Path() string {
	return DefaultStatsPath
}

func (m mysqlStatsService) Scheme() string {
	return ""
}

func (m MySQL) StatsService() mona.StatsAccessor {
	return &mysqlStatsService{&m}
}

func (m MySQL) StatsServiceLabels() map[string]string {
	lbl := meta_util.FilterKeys(kubedb.GroupName, m.OffshootSelectors(), m.Labels)
	lbl[LabelRole] = RoleStats
	return lbl
}

func (m *MySQL) UsesGroupReplication() bool {
	return m.Spec.Topology != nil && m.Spec.Topology.Mode != nil && *m.Spec.Topology.Mode == MySQLClusterModeGroup
}

func (m *MySQL) SetDefaults() {
	if m == nil {
		return
	}
	if m.Spec.StorageType == "" {
		m.Spec.StorageType = StorageTypeDurable
	}
	if m.Spec.TerminationPolicy == "" {
		m.Spec.TerminationPolicy = TerminationPolicyDelete
	}

	if m.UsesGroupReplication() {
		if m.Spec.Replicas == nil {
			m.Spec.Replicas = pointer.Int32P(MySQLDefaultGroupSize)
		}
		m.Spec.setDefaultProbes()
	} else {
		if m.Spec.Replicas == nil {
			m.Spec.Replicas = pointer.Int32P(1)
		}
	}

	if m.Spec.PodTemplate.Spec.ServiceAccountName == "" {
		m.Spec.PodTemplate.Spec.ServiceAccountName = m.OffshootName()
	}

	m.Spec.Monitor.SetDefaults()

	m.SetTLSDefaults()
	setDefaultResourceLimits(&m.Spec.PodTemplate.Spec.Resources, defaultResourceLimits, defaultResourceLimits)
}

func (m *MySQL) SetTLSDefaults() {
	if m.Spec.TLS == nil || m.Spec.TLS.IssuerRef == nil {
		return
	}
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLServerCert), m.CertificateName(MySQLServerCert))
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLClientCert), m.CertificateName(MySQLClientCert))
	m.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(m.Spec.TLS.Certificates, string(MySQLMetricsExporterCert), m.CertificateName(MySQLMetricsExporterCert))
}

// setDefaultProbes sets defaults only when probe fields are nil.
// In operator, check if the value of probe fields is "{}".
// For "{}", ignore readinessprobe or livenessprobe in statefulset.
// Ref: https://github.com/mattlord/Docker-InnoDB-Cluster/blob/master/healthcheck.sh#L10
func (m *MySQLSpec) setDefaultProbes() {
	probe := &core.Probe{
		Handler: core.Handler{
			Exec: &core.ExecAction{
				Command: []string{
					"bash",
					"-c",
					`
export MYSQL_PWD=${MYSQL_ROOT_PASSWORD}
mysql -h localhost -nsLNE -e "select member_state from performance_schema.replication_group_members where member_id=@@server_uuid;" 2>/dev/null | grep "ONLINE"
`,
				},
			},
		},
		InitialDelaySeconds: 30,
		PeriodSeconds:       5,
	}

	if m.PodTemplate.Spec.LivenessProbe == nil {
		m.PodTemplate.Spec.LivenessProbe = probe
	}
	if m.PodTemplate.Spec.ReadinessProbe == nil {
		m.PodTemplate.Spec.ReadinessProbe = probe
	}
}

func (m *MySQLSpec) GetPersistentSecrets() []string {
	if m == nil {
		return nil
	}

	var secrets []string
	if m.AuthSecret != nil {
		secrets = append(secrets, m.AuthSecret.Name)
	}
	return secrets
}

// CertificateName returns the default certificate name and/or certificate secret name for a certificate alias
func (m *MySQL) CertificateName(alias MySQLCertificateAlias) string {
	return meta_util.NameWithSuffix(m.Name, fmt.Sprintf("%s-cert", string(alias)))
}

// MustCertSecretName returns the secret name for a certificate alias
func (m *MySQL) MustCertSecretName(alias MySQLCertificateAlias) string {
	if m == nil {
		panic("missing MySQL database")
	} else if m.Spec.TLS == nil {
		panic(fmt.Errorf("MySQL %s/%s is missing tls spec", m.Namespace, m.Name))
	}
	name, ok := kmapi.GetCertificateSecretName(m.Spec.TLS.Certificates, string(alias))
	if !ok {
		panic(fmt.Errorf("MySQL %s/%s is missing secret name for %s certificate", m.Namespace, m.Name, alias))
	}
	return name
}

func (m *MySQL) ReplicasAreReady(lister appslister.StatefulSetLister) (bool, string, error) {
	// Desire number of statefulSets
	expectedItems := 1
	return checkReplicas(lister.StatefulSets(m.Namespace), labels.SelectorFromSet(m.OffshootLabels()), expectedItems)
}

func MySQLRequireSSLArg() string {
	return "--require-secure-transport=ON"
}

func MySQLExporterTLSArg() string {
	return "--config.my-cnf=/etc/mysql/certs/exporter.cnf"
}

func (m *MySQL) MySQLTLSArgs() []string {
	tlsArgs := []string{
		"--ssl-capath=/etc/mysql/certs",
		"--ssl-ca=/etc/mysql/certs/ca.crt",
		"--ssl-cert=/etc/mysql/certs/server.crt",
		"--ssl-key=/etc/mysql/certs/server.key",
	}
	if m.Spec.RequireSSL {
		tlsArgs = append(tlsArgs, MySQLRequireSSLArg())
	}
	return tlsArgs
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"os"
	"time"

	"github.com/mdlayher/wavepipe/config"
)

// App is the application's name
const App = "wavepipe"

// Version is the application's version
const Version = "git-master"

// ConfigPath is the application's configuration path
var ConfigPath string

// StartTime is the application's starting UNIX timestamp
var StartTime = time.Now().Unix()

// Manager is responsible for coordinating the application
func Manager(killChan chan struct{}, exitChan chan int) {
	log.Printf("manager: initializing %s %s...", App, Version)

	// Gather information about the operating system
	stat, err := OSInfo()
	if err != nil {
		log.Println("manager: could not get operating system info:", err)
	} else {
		log.Printf("manager: %s - %s_%s (%d CPU) [pid: %d]", stat.Hostname, stat.Platform, stat.Architecture, stat.NumCPU, stat.PID)
	}

	// Set configuration (if default path used, config will be created)
	config.C = new(config.JSONFileConfig)
	if err := config.C.Use(ConfigPath); err != nil {
		log.Fatalf("manager: could not use config: %s, %s", ConfigPath, err.Error())
	}

	// Load configuration from specified source
	conf, err := config.C.Load()
	if err != nil {
		log.Fatalf("manager: could not load config: %s, %s", ConfigPath, err.Error())
	}

	// Check valid media folder
	folder := conf.Media()
	if folder == "" {
		log.Fatalf("manager: no media folder set in config: %s", ConfigPath)
	} else if _, err := os.Stat(folder); err != nil {
		log.Fatalf("manager: invalid media folder set in config: %s", err.Error())
	}

	// Launch database manager to handle database/ORM connections
	dbKillChan := make(chan struct{})
	go dbManager(*conf, dbKillChan)

	// Launch cron manager to handle timed events
	cronKillChan := make(chan struct{})
	go cronManager(cronKillChan)

	// Launch filesystem manager to handle file scanning
	fsKillChan := make(chan struct{})
	go fsManager(folder, fsKillChan)

	// Launch HTTP API server
	apiKillChan := make(chan struct{})
	go apiRouter(apiKillChan)

	// Wait for termination signal
	for {
		select {
		// Trigger a graceful shutdown
		case <-killChan:
			log.Println("manager: triggering graceful shutdown, press Ctrl+C again to force halt")

			// Stop API, wait for confirmation
			apiKillChan <- struct{}{}
			<-apiKillChan
			close(apiKillChan)

			// Stop filesystem, wait for confirmation
			fsKillChan <- struct{}{}
			<-fsKillChan
			close(fsKillChan)

			// Stop database, wait for confirmation
			dbKillChan <- struct{}{}
			<-dbKillChan
			close(dbKillChan)

			// Stop cron, wait for confirmation
			cronKillChan <- struct{}{}
			<-cronKillChan
			close(cronKillChan)

			// Exit gracefully
			log.Println("manager: stopped!")
			exitChan <- 0
		}
	}
}
<commit_msg>Don't check for valid folder on test mode<commit_after>package core

import (
	"log"
	"os"
	"time"

	"github.com/mdlayher/wavepipe/config"
)

// App is the application's name
const App = "wavepipe"

// Version is the application's version
const Version = "git-master"

// ConfigPath is the application's configuration path
var ConfigPath string

// StartTime is the application's starting UNIX timestamp
var StartTime = time.Now().Unix()

// Manager is responsible for coordinating the application
func Manager(killChan chan struct{}, exitChan chan int) {
	log.Printf("manager: initializing %s %s...", App, Version)

	// Gather information about the operating system
	stat, err := OSInfo()
	if err != nil {
		log.Println("manager: could not get operating system info:", err)
	} else {
		log.Printf("manager: %s - %s_%s (%d CPU) [pid: %d]", stat.Hostname, stat.Platform, stat.Architecture, stat.NumCPU, stat.PID)
	}

	// Set configuration (if default path used, config will be created)
	config.C = new(config.JSONFileConfig)
	if err := config.C.Use(ConfigPath); err != nil {
		log.Fatalf("manager: could not use config: %s, %s", ConfigPath, err.Error())
	}

	// Load configuration from specified source
	conf, err := config.C.Load()
	if err != nil {
		log.Fatalf("manager: could not load config: %s, %s", ConfigPath, err.Error())
	}

	// Check valid media folder, unless in test mode
	folder := conf.Media()
	if os.Getenv("WAVEPIPE_TEST") != "1" {
		// Check empty folder
		if folder == "" {
			log.Fatalf("manager: no media folder set in config: %s", ConfigPath)
		} else if _, err := os.Stat(folder); err != nil {
			// Check file existence
			log.Fatalf("manager: invalid media folder set in config: %s", err.Error())
		}
	}

	// Launch database manager to handle database/ORM connections
	dbKillChan := make(chan struct{})
	go dbManager(*conf, dbKillChan)

	// Launch cron manager to handle timed events
	cronKillChan := make(chan struct{})
	go cronManager(cronKillChan)

	// Launch filesystem manager to handle file scanning
	fsKillChan := make(chan struct{})
	go fsManager(folder, fsKillChan)

	// Launch HTTP API server
	apiKillChan := make(chan struct{})
	go apiRouter(apiKillChan)

	// Wait for termination signal
	for {
		select {
		// Trigger a graceful shutdown
		case <-killChan:
			log.Println("manager: triggering graceful shutdown, press Ctrl+C again to force halt")

			// Stop API, wait for confirmation
			apiKillChan <- struct{}{}
			<-apiKillChan
			close(apiKillChan)

			// Stop filesystem, wait for confirmation
			fsKillChan <- struct{}{}
			<-fsKillChan
			close(fsKillChan)

			// Stop database, wait for confirmation
			dbKillChan <- struct{}{}
			<-dbKillChan
			close(dbKillChan)

			// Stop cron, wait for confirmation
			cronKillChan <- struct{}{}
			<-cronKillChan
			close(cronKillChan)

			// Exit gracefully
			log.Println("manager: stopped!")
			exitChan <- 0
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"testing"

	. "gopkg.in/check.v1"

	"github.com/minio-io/mc/pkg/client"
)

const (
	globalMaxKeys = 1000 // Maximum number of keys to fetch per request
)

func Test(t *testing.T) { TestingT(t) }

type MySuite struct{}

var _ = Suite(&MySuite{})

func setupNodeDiskMap(c *C) map[string][]string {
	var disks []string
	for i := 0; i < 16; i++ {
		root, err := ioutil.TempDir(os.TempDir(), "donut-")
		c.Assert(err, IsNil)
		disks = append(disks, root)
	}
	nodeDiskMap := make(map[string][]string)
	nodeDiskMap["localhost"] = disks
	return nodeDiskMap
}

func removeDisks(c *C, disks []string) {
	for _, disk := range disks {
		err := os.RemoveAll(disk)
		c.Assert(err, IsNil)
	}
}

func (s *MySuite) TestEmptyBucket(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// check buckets are empty
	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)
	c.Assert(len(buckets), Equals, 0)
}

func (s *MySuite) TestBucketWithoutNameFails(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// fail to create new bucket without a name
	err = donut.PutBucket("")
	c.Assert(err, Not(IsNil))

	err = donut.PutBucket(" ")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestCreateBucketAndList(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// make bucket
	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	// check bucket exists
	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)
	c.Assert(len(buckets), Equals, 1)
	c.Assert(buckets[0].Name, Equals, "foo")
}

func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// make bucket
	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	// make bucket fail
	err = donut.PutBucket("foo")
	c.Assert(err, Not(IsNil))
}

func getBucketNames(buckets []*client.Bucket) []string {
	var bucketNames []string
	for _, bucket := range buckets {
		bucketNames = append(bucketNames, bucket.Name)
	}
	return bucketNames
}

func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	err = donut.PutBucket("bar")
	c.Assert(err, IsNil)

	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)

	bucketNamesReceived := getBucketNames(buckets)
	bucketNamesProvided := []string{"bar", "foo"}
	c.Assert(bucketNamesReceived, DeepEquals, bucketNamesProvided)

	err = donut.PutBucket("foobar")
	c.Assert(err, IsNil)
	bucketNamesProvided = append(bucketNamesProvided, "foobar")

	buckets, err = donut.ListBuckets()
	c.Assert(err, IsNil)
	bucketNamesReceived = getBucketNames(buckets)
	c.Assert(bucketNamesReceived, DeepEquals, bucketNamesProvided)
}

func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	_, _, err = donut.Get("foo", "obj")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	_, _, err = donut.Get("foo", "")
	c.Assert(err, Not(IsNil))

	_, _, err = donut.Get("foo", " ")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	data := "Hello World"
	err = donut.Put("foo", "obj", int64(len(data)), bytes.NewBuffer([]byte(data)))
	c.Assert(err, IsNil)

	reader, size, err := donut.Get("foo", "obj")
	c.Assert(err, IsNil)

	var actualData bytes.Buffer
	_, err = io.Copy(&actualData, reader)
	c.Assert(err, IsNil)
	c.Assert(actualData.Bytes(), DeepEquals, []byte(data))
	c.Assert(int64(actualData.Len()), Equals, size)
}

func getObjectNames(objects []*client.Item) []string {
	var objectNames []string
	for _, object := range objects {
		objectNames = append(objectNames, object.Key)
	}
	return objectNames
}

func (s *MySuite) TestMultipleNewObjects(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	c.Assert(donut.PutBucket("foo"), IsNil)
	err = donut.Put("foo", "obj1", int64(len([]byte("one"))), bytes.NewBuffer([]byte("one")))
	c.Assert(err, IsNil)

	err = donut.Put("foo", "obj2", int64(len([]byte("two"))), bytes.NewBuffer([]byte("two")))
	c.Assert(err, IsNil)

	reader, _, err := donut.Get("foo", "obj1")
	c.Assert(err, IsNil)

	var readerBuffer1 bytes.Buffer
	_, err = io.Copy(&readerBuffer1, reader)
	c.Assert(err, IsNil)
	c.Assert(readerBuffer1.Bytes(), DeepEquals, []byte("one"))

	reader, _, err = donut.Get("foo", "obj2")
	c.Assert(err, IsNil)

	var readerBuffer2 bytes.Buffer
	_, err = io.Copy(&readerBuffer2, reader)
	c.Assert(err, IsNil)
	c.Assert(readerBuffer2.Bytes(), DeepEquals, []byte("two"))

	// test list objects
	listObjects, _, err := donut.ListObjects("foo", "", "", "", globalMaxKeys)
	c.Assert(err, IsNil)

	receivedObjectNames := getObjectNames(listObjects)
	c.Assert(receivedObjectNames, DeepEquals, []string{"obj1", "obj2"})
}
<commit_msg>Update donut tests<commit_after>/*
 * Minimalist Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"testing"

	. "gopkg.in/check.v1"

	"github.com/minio-io/mc/pkg/client"
)

const (
	globalMaxKeys = 1000 // Maximum number of keys to fetch per request
)

func Test(t *testing.T) { TestingT(t) }

type MySuite struct{}

var _ = Suite(&MySuite{})

func setupNodeDiskMap(c *C) map[string][]string {
	var disks []string
	for i := 0; i < 16; i++ {
		root, err := ioutil.TempDir(os.TempDir(), "donut-")
		c.Assert(err, IsNil)
		disks = append(disks, root)
	}
	nodeDiskMap := make(map[string][]string)
	nodeDiskMap["localhost"] = disks
	return nodeDiskMap
}

func removeDisks(c *C, disks []string) {
	for _, disk := range disks {
		err := os.RemoveAll(disk)
		c.Assert(err, IsNil)
	}
}

func (s *MySuite) TestEmptyBucket(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// check buckets are empty
	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)
	c.Assert(len(buckets), Equals, 0)
}

func (s *MySuite) TestBucketWithoutNameFails(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// fail to create new bucket without a name
	err = donut.PutBucket("")
	c.Assert(err, Not(IsNil))

	err = donut.PutBucket(" ")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestCreateBucketAndList(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// make bucket
	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	// check bucket exists
	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)
	c.Assert(len(buckets), Equals, 1)
	c.Assert(buckets[0].Name, Equals, "foo")
}

func (s *MySuite) TestCreateBucketWithSameNameFails(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	// make bucket
	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	// make bucket fail
	err = donut.PutBucket("foo")
	c.Assert(err, Not(IsNil))
}

func getBucketNames(buckets []*client.Bucket) []string {
	var bucketNames []string
	for _, bucket := range buckets {
		bucketNames = append(bucketNames, bucket.Name)
	}
	return bucketNames
}

func (s *MySuite) TestCreateMultipleBucketsAndList(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	err = donut.PutBucket("bar")
	c.Assert(err, IsNil)

	buckets, err := donut.ListBuckets()
	c.Assert(err, IsNil)

	bucketNamesReceived := getBucketNames(buckets)
	bucketNamesProvided := []string{"bar", "foo"}
	c.Assert(bucketNamesReceived, DeepEquals, bucketNamesProvided)

	err = donut.PutBucket("foobar")
	c.Assert(err, IsNil)
	bucketNamesProvided = append(bucketNamesProvided, "foobar")

	buckets, err = donut.ListBuckets()
	c.Assert(err, IsNil)
	bucketNamesReceived = getBucketNames(buckets)
	c.Assert(bucketNamesReceived, DeepEquals, bucketNamesProvided)
}

func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	_, _, _, err = donut.Get("foo", "obj")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	_, _, _, err = donut.Get("foo", "")
	c.Assert(err, Not(IsNil))

	_, _, _, err = donut.Get("foo", " ")
	c.Assert(err, Not(IsNil))
}

func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	err = donut.PutBucket("foo")
	c.Assert(err, IsNil)

	m := md5.New()
	data := "Hello World"
	io.WriteString(m, data)
	err = donut.Put("foo", "obj", hex.EncodeToString(m.Sum(nil)), int64(len(data)), bytes.NewBuffer([]byte(data)))
	c.Assert(err, IsNil)

	reader, size, _, err := donut.Get("foo", "obj")
	c.Assert(err, IsNil)

	var actualData bytes.Buffer
	_, err = io.Copy(&actualData, reader)
	c.Assert(err, IsNil)
	c.Assert(actualData.Bytes(), DeepEquals, []byte(data))
	c.Assert(int64(actualData.Len()), Equals, size)
}

func getObjectNames(objects []*client.Item) []string {
	var objectNames []string
	for _, object := range objects {
		objectNames = append(objectNames, object.Key)
	}
	return objectNames
}

func (s *MySuite) TestMultipleNewObjects(c *C) {
	nodeDiskMap := setupNodeDiskMap(c)
	donut, err := GetNewClient("testemptydonut", nodeDiskMap)
	defer removeDisks(c, nodeDiskMap["localhost"])
	c.Assert(err, IsNil)

	c.Assert(donut.PutBucket("foo"), IsNil)

	m1 := md5.New()
	m2 := md5.New()
	io.WriteString(m1, "one")
	io.WriteString(m2, "two")

	err = donut.Put("foo", "obj1",
		hex.EncodeToString(m1.Sum(nil)), int64(len([]byte("one"))), bytes.NewBuffer([]byte("one")))
	c.Assert(err, IsNil)

	err = donut.Put("foo", "obj2",
		hex.EncodeToString(m2.Sum(nil)), int64(len([]byte("two"))), bytes.NewBuffer([]byte("two")))
	c.Assert(err, IsNil)

	reader, _, _, err := donut.Get("foo", "obj1")
	c.Assert(err, IsNil)

	var readerBuffer1 bytes.Buffer
	_, err = io.Copy(&readerBuffer1, reader)
	c.Assert(err, IsNil)
	c.Assert(readerBuffer1.Bytes(), DeepEquals, []byte("one"))

	reader, _, _, err = donut.Get("foo", "obj2")
	c.Assert(err, IsNil)

	var readerBuffer2 bytes.Buffer
	_, err = io.Copy(&readerBuffer2, reader)
	c.Assert(err, IsNil)
	c.Assert(readerBuffer2.Bytes(), DeepEquals, []byte("two"))

	// test list objects
	listObjects, _, err := donut.ListObjects("foo", "", "", "", globalMaxKeys)
	c.Assert(err, IsNil)

	receivedObjectNames := getObjectNames(listObjects)
	c.Assert(receivedObjectNames, DeepEquals, []string{"obj1", "obj2"})
}
<|endoftext|>


<commit_before>package compression

import (
	"compress/flate"
	"io"

	"github.com/pkg/errors"
)

const (
	// defaultCompressionLevel is the default compression level to use for
	// writers.
	defaultCompressionLevel = 6
)

// NewDecompressingReader wraps an io.Reader in a decompressor.
func NewDecompressingReader(source io.Reader) io.Reader {
	// HACK: Technically this function returns an io.ReadCloser, and it
	// documents that it is the caller's function to call Close on the reader.
	// However, it turns out that the underlying implementation of Close just
	// checks for stream errors and that it isn't really necessary to call it.
	// As a result, we ignore this portion of the interface, but we're sort of
	// relying on an implementaiton detail in doing so.
	return flate.NewReader(source)
}

// automaticallyFlushingFlateWriter is a type that wraps a flate.Writer and
// automatically flushes data on every write.
type automaticallyFlushingFlateWriter struct {
	// compressor is the underlying flate compressor.
	compressor *flate.Writer
}

// Write writes data to the compressor and automatically flushes it to the
// underlying writer.
func (w *automaticallyFlushingFlateWriter) Write(buffer []byte) (int, error) {
	count, err := w.compressor.Write(buffer)
	if err != nil {
		return count, err
	} else if err = w.compressor.Flush(); err != nil {
		return 0, errors.Wrap(err, "unable to flush compressor")
	}
	return count, nil
}

// NewCompressingWriter wraps an io.Writer in a compressor.
func NewCompressingWriter(destination io.Writer) io.Writer {
	// Create the compressor. If a sane compression level is provided, the flate
	// API guarantees that creation of the compressor will succeed.
	compressor, _ := flate.NewWriter(destination, defaultCompressionLevel)

	// Wrap the compressor.
	return &automaticallyFlushingFlateWriter{compressor}
}
<commit_msg>Fixed typo in compression package.<commit_after>package compression

import (
	"compress/flate"
	"io"

	"github.com/pkg/errors"
)

const (
	// defaultCompressionLevel is the default compression level to use for
	// writers.
	defaultCompressionLevel = 6
)

// NewDecompressingReader wraps an io.Reader in a decompressor.
func NewDecompressingReader(source io.Reader) io.Reader {
	// HACK: Technically this function returns an io.ReadCloser, and it
	// documents that it is the caller's function to call Close on the reader.
	// However, it turns out that the underlying implementation of Close just
	// checks for stream errors and that it isn't really necessary to call it.
	// As a result, we ignore this portion of the interface, but we're sort of
	// relying on an implementation detail in doing so.
	return flate.NewReader(source)
}

// automaticallyFlushingFlateWriter is a type that wraps a flate.Writer and
// automatically flushes data on every write.
type automaticallyFlushingFlateWriter struct {
	// compressor is the underlying flate compressor.
	compressor *flate.Writer
}

// Write writes data to the compressor and automatically flushes it to the
// underlying writer.
func (w *automaticallyFlushingFlateWriter) Write(buffer []byte) (int, error) {
	count, err := w.compressor.Write(buffer)
	if err != nil {
		return count, err
	} else if err = w.compressor.Flush(); err != nil {
		return 0, errors.Wrap(err, "unable to flush compressor")
	}
	return count, nil
}

// NewCompressingWriter wraps an io.Writer in a compressor.
func NewCompressingWriter(destination io.Writer) io.Writer {
	// Create the compressor. If a sane compression level is provided, the flate
	// API guarantees that creation of the compressor will succeed.
	compressor, _ := flate.NewWriter(destination, defaultCompressionLevel)

	// Wrap the compressor.
	return &automaticallyFlushingFlateWriter{compressor}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package loader

import (
	"context"
	"fmt"
	"net"

	"github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"
)

type baseDeviceMode string

const (
	ipvlanMode = baseDeviceMode("ipvlan")
	directMode = baseDeviceMode("direct")
	tunnelMode = baseDeviceMode("tunnel")

	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint or XDP program.
func replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string, xdp bool, xdpMode string) error {
	var (
		loaderProg string
		args       []string
	)

	if !xdp {
		if err := replaceQdisc(ifName); err != nil {
			return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
		}
	}

	// Temporarily rename bpffs pins of maps whose definitions have changed in
	// a new version of a datapath ELF.
	if err := bpf.StartBPFFSMigration(bpf.GetMapRoot(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	// Schedule finalizing the migration. If the iproute2 call below is successful,
	// any 'pending' pins will be removed. If not, any pending maps will be re-pinned
	// back to their initial paths.
	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.GetMapRoot(), objPath, revert); err != nil {
			log.WithError(err).WithField("device", ifName).WithField("objPath", objPath).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	if xdp {
		loaderProg = "ip"
		args = []string{"-force", "link", "set", "dev", ifName, xdpMode,
			"obj", objPath, "sec", progSec}
	} else {
		loaderProg = "tc"
		args = []string{"filter", "replace", "dev", ifName, progDirection,
			"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
			"sec", progSec,
		}
	}

	cmd := exec.CommandContext(ctx, loaderProg, args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		// Program/object replacement unsuccessful, revert bpffs migration.
		revert = true
		return fmt.Errorf("Failed to load prog with %s: %w", loaderProg, err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	if err := bpf.StartBPFFSMigration(bpf.GetMapRoot(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.GetMapRoot(), objPath, revert); err != nil {
			log.WithError(err).WithFields(logrus.Fields{logfields.BPFMapPath: mapPath, "objPath": objPath}).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd := exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		revert = true
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// RemoveTCFilters removes all tc filters from the given interface.
// Direction is passed as netlink.HANDLE_MIN_{INGRESS,EGRESS} via tcDir.
func RemoveTCFilters(ifName string, tcDir uint32) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}

	filters, err := netlink.FilterList(link, tcDir)
	if err != nil {
		return err
	}

	for _, f := range filters {
		if err := netlink.FilterDel(f); err != nil {
			return err
		}
	}

	return nil
}

func setupDev(link netlink.Link) error {
	ifName := link.Attrs().Name

	if err := netlink.LinkSetUp(link); err != nil {
		log.WithError(err).WithField("device", ifName).Warn("Could not set up the link")
		return err
	}

	sysSettings := make([]sysctl.Setting, 0, 5)
	if option.Config.EnableIPv6 {
		sysSettings = append(sysSettings, sysctl.Setting{
			Name: fmt.Sprintf("net.ipv6.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false})
	}
	if option.Config.EnableIPv4 {
		sysSettings = append(sysSettings, []sysctl.Setting{
			{Name: fmt.Sprintf("net.ipv4.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.rp_filter", ifName), Val: "0", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.accept_local", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.send_redirects", ifName), Val: "0", IgnoreErr: false},
		}...)
	}
	if err := sysctl.ApplySettings(sysSettings); err != nil {
		return err
	}

	return nil
}

func setupVethPair(name, peerName string) error {
	// Create the veth pair if it doesn't exist.
	if _, err := netlink.LinkByName(name); err != nil {
		hostMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}
		peerMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}

		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:         name,
				HardwareAddr: net.HardwareAddr(hostMac),
				TxQLen:       1000,
			},
			PeerName:         peerName,
			PeerHardwareAddr: net.HardwareAddr(peerMac),
		}
		if err := netlink.LinkAdd(veth); err != nil {
			return err
		}
	}

	veth, err := netlink.LinkByName(name)
	if err != nil {
		return err
	}
	if err := setupDev(veth); err != nil {
		return err
	}
	peer, err := netlink.LinkByName(peerName)
	if err != nil {
		return err
	}
	if err := setupDev(peer); err != nil {
		return err
	}

	return nil
}

func setupIpvlan(name string, nativeLink netlink.Link) (*netlink.IPVlan, error) {
	hostLink, err := netlink.LinkByName(name)
	if err == nil {
		// Ignore the error.
		netlink.LinkDel(hostLink)
	}

	ipvlan := &netlink.IPVlan{
		LinkAttrs: netlink.LinkAttrs{
			Name:        name,
			ParentIndex: nativeLink.Attrs().Index,
			TxQLen:      1000,
		},
		Mode: netlink.IPVLAN_MODE_L3,
	}
	if err := netlink.LinkAdd(ipvlan); err != nil {
		return nil, err
	}

	if err := setupDev(ipvlan); err != nil {
		return nil, err
	}

	return ipvlan, nil
}

// setupBaseDevice decides which and what kind of interfaces should be set up as
// the first step of datapath initialization, then performs the setup (and
// creation, if needed) of those interfaces. It returns two links and an error.
// By default, it sets up the veth pair - cilium_host and cilium_net.
// In ipvlan mode, it creates the cilium_host ipvlan with the native device as a
// parent.
func setupBaseDevice(nativeDevs []netlink.Link, mode baseDeviceMode, mtu int) (netlink.Link, netlink.Link, error) {
	switch mode {
	case ipvlanMode:
		ipvlan, err := setupIpvlan(defaults.HostDevice, nativeDevs[0])
		if err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(ipvlan, mtu); err != nil {
			return nil, nil, err
		}

		return ipvlan, ipvlan, nil
	default:
		if err := setupVethPair(defaults.HostDevice, defaults.SecondHostDevice); err != nil {
			return nil, nil, err
		}

		linkHost, err := netlink.LinkByName(defaults.HostDevice)
		if err != nil {
			return nil, nil, err
		}
		linkNet, err := netlink.LinkByName(defaults.SecondHostDevice)
		if err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetARPOff(linkHost); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetARPOff(linkNet); err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetMTU(linkHost, mtu); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(linkNet, mtu); err != nil {
			return nil, nil, err
		}

		return linkHost, linkNet, nil
	}
}
<commit_msg>bpf,datapath: map-migrate - specify full tc/globals/ bpffs path<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2017-2018 Authors of Cilium

package loader

import (
	"context"
	"fmt"
	"net"

	"github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/mac"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/sysctl"
)

type baseDeviceMode string

const (
	ipvlanMode = baseDeviceMode("ipvlan")
	directMode = baseDeviceMode("direct")
	tunnelMode = baseDeviceMode("tunnel")

	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint or XDP program.
func replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string, xdp bool, xdpMode string) error {
	var (
		loaderProg string
		args       []string
	)

	if !xdp {
		if err := replaceQdisc(ifName); err != nil {
			return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
		}
	}

	// Temporarily rename bpffs pins of maps whose definitions have changed in
	// a new version of a datapath ELF.
	if err := bpf.StartBPFFSMigration(bpf.MapPrefixPath(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	// Schedule finalizing the migration. If the iproute2 call below is successful,
	// any 'pending' pins will be removed. If not, any pending maps will be re-pinned
	// back to their initial paths.
	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.MapPrefixPath(), objPath, revert); err != nil {
			log.WithError(err).WithField("device", ifName).WithField("objPath", objPath).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	if xdp {
		loaderProg = "ip"
		args = []string{"-force", "link", "set", "dev", ifName, xdpMode,
			"obj", objPath, "sec", progSec}
	} else {
		loaderProg = "tc"
		args = []string{"filter", "replace", "dev", ifName, progDirection,
			"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
			"sec", progSec,
		}
	}

	cmd := exec.CommandContext(ctx, loaderProg, args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		// Program/object replacement unsuccessful, revert bpffs migration.
		revert = true
		return fmt.Errorf("Failed to load prog with %s: %w", loaderProg, err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	if err := bpf.StartBPFFSMigration(bpf.MapPrefixPath(), objPath); err != nil {
		return fmt.Errorf("Failed to start bpffs map migration: %w", err)
	}

	var revert bool
	defer func() {
		if err := bpf.FinalizeBPFFSMigration(bpf.MapPrefixPath(), objPath, revert); err != nil {
			log.WithError(err).WithFields(logrus.Fields{logfields.BPFMapPath: mapPath, "objPath": objPath}).
				Error("Could not finalize bpffs map migration")
		}
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd := exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	if _, err := cmd.CombinedOutput(log, true); err != nil {
		revert = true
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// RemoveTCFilters removes all tc filters from the given interface.
// Direction is passed as netlink.HANDLE_MIN_{INGRESS,EGRESS} via tcDir.
func RemoveTCFilters(ifName string, tcDir uint32) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}

	filters, err := netlink.FilterList(link, tcDir)
	if err != nil {
		return err
	}

	for _, f := range filters {
		if err := netlink.FilterDel(f); err != nil {
			return err
		}
	}

	return nil
}

func setupDev(link netlink.Link) error {
	ifName := link.Attrs().Name

	if err := netlink.LinkSetUp(link); err != nil {
		log.WithError(err).WithField("device", ifName).Warn("Could not set up the link")
		return err
	}

	sysSettings := make([]sysctl.Setting, 0, 5)
	if option.Config.EnableIPv6 {
		sysSettings = append(sysSettings, sysctl.Setting{
			Name: fmt.Sprintf("net.ipv6.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false})
	}
	if option.Config.EnableIPv4 {
		sysSettings = append(sysSettings, []sysctl.Setting{
			{Name: fmt.Sprintf("net.ipv4.conf.%s.forwarding", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.rp_filter", ifName), Val: "0", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.accept_local", ifName), Val: "1", IgnoreErr: false},
			{Name: fmt.Sprintf("net.ipv4.conf.%s.send_redirects", ifName), Val: "0", IgnoreErr: false},
		}...)
	}
	if err := sysctl.ApplySettings(sysSettings); err != nil {
		return err
	}

	return nil
}

func setupVethPair(name, peerName string) error {
	// Create the veth pair if it doesn't exist.
	if _, err := netlink.LinkByName(name); err != nil {
		hostMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}
		peerMac, err := mac.GenerateRandMAC()
		if err != nil {
			return err
		}

		veth := &netlink.Veth{
			LinkAttrs: netlink.LinkAttrs{
				Name:         name,
				HardwareAddr: net.HardwareAddr(hostMac),
				TxQLen:       1000,
			},
			PeerName:         peerName,
			PeerHardwareAddr: net.HardwareAddr(peerMac),
		}
		if err := netlink.LinkAdd(veth); err != nil {
			return err
		}
	}

	veth, err := netlink.LinkByName(name)
	if err != nil {
		return err
	}
	if err := setupDev(veth); err != nil {
		return err
	}
	peer, err := netlink.LinkByName(peerName)
	if err != nil {
		return err
	}
	if err := setupDev(peer); err != nil {
		return err
	}

	return nil
}

func setupIpvlan(name string, nativeLink netlink.Link) (*netlink.IPVlan, error) {
	hostLink, err := netlink.LinkByName(name)
	if err == nil {
		// Ignore the error.
		netlink.LinkDel(hostLink)
	}

	ipvlan := &netlink.IPVlan{
		LinkAttrs: netlink.LinkAttrs{
			Name:        name,
			ParentIndex: nativeLink.Attrs().Index,
			TxQLen:      1000,
		},
		Mode: netlink.IPVLAN_MODE_L3,
	}
	if err := netlink.LinkAdd(ipvlan); err != nil {
		return nil, err
	}

	if err := setupDev(ipvlan); err != nil {
		return nil, err
	}

	return ipvlan, nil
}

// setupBaseDevice decides which and what kind of interfaces should be set up as
// the first step of datapath initialization, then performs the setup (and
// creation, if needed) of those interfaces. It returns two links and an error.
// By default, it sets up the veth pair - cilium_host and cilium_net.
// In ipvlan mode, it creates the cilium_host ipvlan with the native device as a
// parent.
func setupBaseDevice(nativeDevs []netlink.Link, mode baseDeviceMode, mtu int) (netlink.Link, netlink.Link, error) {
	switch mode {
	case ipvlanMode:
		ipvlan, err := setupIpvlan(defaults.HostDevice, nativeDevs[0])
		if err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(ipvlan, mtu); err != nil {
			return nil, nil, err
		}

		return ipvlan, ipvlan, nil
	default:
		if err := setupVethPair(defaults.HostDevice, defaults.SecondHostDevice); err != nil {
			return nil, nil, err
		}

		linkHost, err := netlink.LinkByName(defaults.HostDevice)
		if err != nil {
			return nil, nil, err
		}
		linkNet, err := netlink.LinkByName(defaults.SecondHostDevice)
		if err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetARPOff(linkHost); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetARPOff(linkNet); err != nil {
			return nil, nil, err
		}

		if err := netlink.LinkSetMTU(linkHost, mtu); err != nil {
			return nil, nil, err
		}
		if err := netlink.LinkSetMTU(linkNet, mtu); err != nil {
			return nil, nil, err
		}

		return linkHost, linkNet, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Kubernetes Authors.
// Modifications copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package export

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/googleapi"
	"k8s.io/client-go/util/flowcontrol"
)

const (
	// Max QPS to allow through to the token URL.
	tokenURLQPS = .05 // back off to once every 20 seconds when failing
	// Maximum burst of requests to token URL before limiting.
	tokenURLBurst = 3
)

// AltTokenSource is the structure holding the data for the functionality needed to generates tokens
type AltTokenSource struct {
	oauthClient *http.Client
	tokenURL    string
	tokenBody   string
	throttle    flowcontrol.RateLimiter
}

// Token returns a token which may be used for authentication
func (a *AltTokenSource) Token() (*oauth2.Token, error) {
	a.throttle.Accept()
	return a.token()
}
func (a *AltTokenSource) token() (*oauth2.Token, error) {
	req, err := http.NewRequest("POST", a.tokenURL, strings.NewReader(a.tokenBody))
	if err != nil {
		return nil, err
	}
	res, err := a.oauthClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if err := googleapi.CheckResponse(res); err != nil {
		return nil, err
	}
	var tok struct {
		AccessToken string    `json:"accessToken"`
		ExpireTime  time.Time `json:"expireTime"`
	}
	if err := json.NewDecoder(res.Body).Decode(&tok); err != nil {
		return nil, err
	}
	return &oauth2.Token{
		AccessToken: tok.AccessToken,
		Expiry:      tok.ExpireTime,
	}, nil
}

// NewAltTokenSource constructs a new alternate token source for generating tokens.
func NewAltTokenSource(tokenURL, tokenBody string) oauth2.TokenSource {
	client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
	a := &AltTokenSource{
		oauthClient: client,
		tokenURL:    tokenURL,
		tokenBody:   tokenBody,
		throttle:    flowcontrol.NewTokenBucketRateLimiter(tokenURLQPS, tokenURLBurst),
	}
	return oauth2.ReuseTokenSource( /* token */ nil, a)
}
<commit_msg>Remove the dependency on k8s.io/client-go/util/flowcontrol<commit_after>// Copyright 2015 The Kubernetes Authors.
// Modifications copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package export

import (
	"context"
	"encoding/json"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"golang.org/x/time/rate"
	"google.golang.org/api/googleapi"
)

const (
	// Max QPS to allow through to the token URL.
	tokenURLQPS = .05 // back off to once every 20 seconds when failing
	// Maximum burst of requests to token URL before limiting.
	tokenURLBurst = 3
)

// AltTokenSource is the structure holding the data for the functionality needed to generates tokens
type AltTokenSource struct {
	oauthClient *http.Client
	tokenURL    string
	tokenBody   string
	throttle    *rate.Limiter
}

// Token returns a token which may be used for authentication
func (a *AltTokenSource) Token() (*oauth2.Token, error) {
	r := a.throttle.Reserve()
	if !r.OK() {
		return nil, errors.Errorf("Rate limiter (rate: %f, burst: %d) cannot provide the requested token.",
			a.throttle.Limit(), a.throttle.Burst())
	}
	time.Sleep(r.Delay())
	return a.token()
}
func (a *AltTokenSource) token() (*oauth2.Token, error) {
	req, err := http.NewRequest("POST", a.tokenURL, strings.NewReader(a.tokenBody))
	if err != nil {
		return nil, err
	}
	res, err := a.oauthClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if err := googleapi.CheckResponse(res); err != nil {
		return nil, err
	}
	var tok struct {
		AccessToken string    `json:"accessToken"`
		ExpireTime  time.Time `json:"expireTime"`
	}
	if err := json.NewDecoder(res.Body).Decode(&tok); err != nil {
		return nil, err
	}
	return &oauth2.Token{
		AccessToken: tok.AccessToken,
		Expiry:      tok.ExpireTime,
	}, nil
}

// NewAltTokenSource constructs a new alternate token source for generating tokens.
func NewAltTokenSource(tokenURL, tokenBody string) oauth2.TokenSource {
	client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
	a := &AltTokenSource{
		oauthClient: client,
		tokenURL:    tokenURL,
		tokenBody:   tokenBody,
		throttle:    rate.NewLimiter(tokenURLQPS, tokenURLBurst),
	}
	return oauth2.ReuseTokenSource( /* token */ nil, a)
}
<|endoftext|>