<commit_before>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type BChannel struct {
	funcValueType int
	funcs         []func(float64, int) float64
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]func(float64, int) float64, 0),
	}
	return bc
}

func (bc *BChannel) push(fn func(float64, int) float64) {
	bc.funcs = append(bc.funcs, fn)
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
}

func New( /*opts map[string]string*/ fn func(float64, int) float64) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	//TODO
	/*
		if val, ok := opts["size"]; ok {
			b.size = val
		}
		if val, ok := opts["rate"]; ok {
			b.rate = val
		}
	*/

	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

func (b *B) end() {
	b.ended = true
}

func (b *B) destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) pause() {
	b.paused = true
}

func (b *B) resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn func(float64, int) float64) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) PushTo(index int, fn func(float64, int) float64) {
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) Push(fn func(float64, int) float64) {
	b.PushTo(len(b.channels), fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func mergeArgs(opts, args map[string]string) []string {
	for k, _ := range opts {
		args[k] = opts[k]
	}
	var resultsLast []string
	var results []string
	for k, _ := range args {
		switch k {
		case "-":
			resultsLast = append(resultsLast, k)
		case "-o":
			resultsLast = append(resultsLast, k, args[k])
		default:
			var dash string
			if len(k) == 1 {
				dash = "-"
			} else {
				dash = "--"
			}
			results = append(results, dash+k, args[k])
		}
	}
	results = append(results, resultsLast...)
	fmt.Printf("results = %v\n", results)
	return results
}

func (b *B) runCommand(command string, mergedArgs []string) {
	cmd := exec.Command(command, mergedArgs...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	defer func() {
		fmt.Println("runCommand: before stdin.Close()")
		stdin.Close()
	}()
	//var out bytes.Buffer
	//cmd.Stdout = &out
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	defer func() {
		if p := cmd.Process; p != nil {
			fmt.Println("runCommand: before p.Kill()")
			p.Kill()
		}
	}()

	readBuf := make([]byte, b.size*len(b.channels))
	for {
		//fmt.Println("play loop header")
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		if _, err = stdin.Write(readBuf); err != nil {
			// TODO: more better error handling
			if err.Error() == "write |1: broken pipe" {
				fmt.Printf("ERR: stdin.Write(readBuf): err = %v\n", err)
				runtime.Gosched()
				break
			}
			panic(err)
		}
	}
}

func (b *B) Play(opts map[string]string) {
	go b.runCommand("play", mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts map[string]string) {
	go b.runCommand("sox", mergeArgs(opts, map[string]string{
		"c":  strconv.Itoa(len(b.channels)),
		"r":  strconv.Itoa(b.rate),
		"t":  "s16",
		"-":  "DUMMY",
		"-o": file,
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}
<commit_msg>teporary comment out sox stdout.<commit_after>package baudio

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
	//"os"
	"os/exec"
	"runtime"
	"strconv"
	//"time"
)

const (
	FuncValueTypeFloat    = 0
	FuncValueTypeNotFloat = 1
)

type BChannel struct {
	funcValueType int
	funcs         []func(float64, int) float64
}

func newBChannel(fvt int) *BChannel {
	bc := &BChannel{
		funcValueType: fvt,
		funcs:         make([]func(float64, int) float64, 0),
	}
	return bc
}

func (bc *BChannel) push(fn func(float64, int) float64) {
	bc.funcs = append(bc.funcs, fn)
}

type B struct {
	readable   bool
	size       int
	rate       int
	t          float64
	i          int
	paused     bool
	ended      bool
	destroyed  bool
	channels   []*BChannel
	chEnd      chan bool
	chEndSox   chan bool
	chResume   chan func()
	chNextTick chan bool
	pipeReader *io.PipeReader
	pipeWriter *io.PipeWriter
}

func New( /*opts map[string]string*/ fn func(float64, int) float64) *B {
	b := &B{
		readable:   true,
		size:       2048,
		rate:       44000,
		t:          0,
		i:          0,
		paused:     false,
		ended:      false,
		destroyed:  false,
		chEnd:      make(chan bool),
		chEndSox:   make(chan bool),
		chResume:   make(chan func()),
		chNextTick: make(chan bool),
	}
	b.pipeReader, b.pipeWriter = io.Pipe()
	//TODO
	/*
		if val, ok := opts["size"]; ok {
			b.size = val
		}
		if val, ok := opts["rate"]; ok {
			b.rate = val
		}
	*/

	if fn != nil {
		b.Push(fn)
	}
	go func() {
		if b.paused {
			b.chResume <- func() {
				go b.loop()
				b.main()
			}
		} else {
			go b.loop()
			b.main()
		}
	}()
	//go b.loop()
	return b
}

func (b *B) main() {
	for {
		// 2013-02-28 koyachi ここで何かしないとループまわらないのなぜ
		// => fmt.PrinfすることでnodeのnextTick的なものがつまれててそのうちPlay()のread待ちまで進めるのでは。
		//L1:
		//fmt.Println("main loop header")
		//fmt.Printf(".")
		//time.Sleep(1 * time.Millisecond)
		runtime.Gosched()
		select {
		case <-b.chEnd:
			fmt.Println("main chEnd")
			b.terminateMain()
			break
		case fn := <-b.chResume:
			//fmt.Println("main chResume")
			fn()
		case <-b.chNextTick:
			//fmt.Println("main chNextTick")
			go b.loop()
			//b.loop()
		default:
			//fmt.Println("main default")
			//go b.loop()
			//goto L1
		}
	}
}

func (b *B) terminateMain() {
	b.pipeWriter.Close()
	b.ended = true
	b.chEndSox <- true
}

func (b *B) end() {
	b.ended = true
}

func (b *B) destroy() {
	b.destroyed = true
	b.chEnd <- true
}

func (b *B) pause() {
	b.paused = true
}

func (b *B) resume() {
	if !b.paused {
		return
	}
	b.paused = false
	b.chResume <- func() {}
}

func (b *B) AddChannel(funcValueType int, fn func(float64, int) float64) {
	bc := newBChannel(funcValueType)
	bc.push(fn)
	b.channels = append(b.channels, bc)
}

func (b *B) PushTo(index int, fn func(float64, int) float64) {
	if len(b.channels) <= index {
		bc := newBChannel(FuncValueTypeFloat)
		b.channels = append(b.channels, bc)
	}
	b.channels[index].funcs = append(b.channels[index].funcs, fn)
}

func (b *B) Push(fn func(float64, int) float64) {
	b.PushTo(len(b.channels), fn)
}

func (b *B) loop() {
	buf := b.tick()
	if b.destroyed {
		// no more events
		//fmt.Println("loop destroyed")
	} else if b.paused {
		//fmt.Println("loop paused")
		b.chResume <- func() {
			b.pipeWriter.Write(buf.Bytes())
			b.chNextTick <- true
		}
	} else {
		//fmt.Println("loop !(destroyed || paused)")
		b.pipeWriter.Write(buf.Bytes())
		if b.ended {
			//fmt.Println("loop ended")
			b.chEnd <- true
		} else {
			//fmt.Println("loop !ended")
			b.chNextTick <- true
		}
	}
}

func (b *B) tick() *bytes.Buffer {
	bufSize := b.size * len(b.channels)
	byteBuffer := make([]byte, 0)
	buf := bytes.NewBuffer(byteBuffer)
	for i := 0; i < bufSize; i += 2 {
		lrIndex := int(i / 2)
		lenCh := len(b.channels)
		ch := b.channels[lrIndex%lenCh]
		t := float64(b.t) + math.Floor(float64(lrIndex))/float64(b.rate)/float64(lenCh)
		counter := b.i + int(math.Floor(float64(lrIndex)/float64(lenCh)))

		value := float64(0)
		n := float64(0)
		for j := 0; j < len(ch.funcs); j++ {
			x := ch.funcs[j](float64(t), counter)
			n += x
		}
		n /= float64(len(ch.funcs))

		if ch.funcValueType == FuncValueTypeFloat {
			value = signed(n)
		} else {
			b_ := math.Pow(2, float64(ch.funcValueType))
			x := math.Mod(math.Floor(n), b_) / b_ * math.Pow(2, 15)
			value = x
		}
		if err := binary.Write(buf, binary.LittleEndian, int16(clamp(value))); err != nil {
			panic(err)
		}
	}
	b.i += b.size / 2
	b.t += float64(b.size) / float64(2) / float64(b.rate)
	return buf
}

func clamp(x float64) float64 {
	return math.Max(math.Min(x, math.Pow(2, 15)-1), -math.Pow(2, 15))
}

func signed(n float64) float64 {
	b := math.Pow(2, 15)
	if n > 0 {
		return math.Min(b-1, math.Floor(b*n-1))
	}
	return math.Max(-b, math.Ceil(b*n-1))
}

func mergeArgs(opts, args map[string]string) []string {
	for k, _ := range opts {
		args[k] = opts[k]
	}
	var resultsLast []string
	var results []string
	for k, _ := range args {
		switch k {
		case "-":
			resultsLast = append(resultsLast, k)
		case "-o":
			resultsLast = append(resultsLast, k, args[k])
		default:
			var dash string
			if len(k) == 1 {
				dash = "-"
			} else {
				dash = "--"
			}
			results = append(results, dash+k, args[k])
		}
	}
	results = append(results, resultsLast...)
	fmt.Printf("results = %v\n", results)
	return results
}

func (b *B) runCommand(command string, mergedArgs []string) {
	cmd := exec.Command(command, mergedArgs...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	defer func() {
		fmt.Println("runCommand: before stdin.Close()")
		stdin.Close()
	}()
	var out bytes.Buffer
	cmd.Stdout = &out
	// TODO: option
	//cmd.Stdout = os.Stdout
	//cmd.Stderr = os.Stderr
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	defer func() {
		if p := cmd.Process; p != nil {
			fmt.Println("runCommand: before p.Kill()")
			p.Kill()
		}
	}()

	readBuf := make([]byte, b.size*len(b.channels))
	for {
		//fmt.Println("play loop header")
		if _, err := b.pipeReader.Read(readBuf); err != nil {
			panic(err)
		}
		if _, err = stdin.Write(readBuf); err != nil {
			// TODO: more better error handling
			if err.Error() == "write |1: broken pipe" {
				fmt.Printf("ERR: stdin.Write(readBuf): err = %v\n", err)
				runtime.Gosched()
				break
			}
			panic(err)
		}
	}
}

func (b *B) Play(opts map[string]string) {
	go b.runCommand("play", mergeArgs(opts, map[string]string{
		"c": strconv.Itoa(len(b.channels)),
		"r": strconv.Itoa(b.rate),
		"t": "s16",
		"-": "DUMMY",
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}

func (b *B) Record(file string, opts map[string]string) {
	go b.runCommand("sox", mergeArgs(opts, map[string]string{
		"c":  strconv.Itoa(len(b.channels)),
		"r":  strconv.Itoa(b.rate),
		"t":  "s16",
		"-":  "DUMMY",
		"-o": file,
	}))
	<-b.chEndSox
	b.pipeReader.Close()
}
<|endoftext|>


<commit_before>package oauth2

import (
	"net/http"
	"strings"
)

const Bearer = "bearer"

func NewBearerTokenResponse(token string, expiresIn int) *TokenResponse {
	return NewTokenResponse(Bearer, token, expiresIn)
}

func ParseBearerToken(r *http.Request) (string, error) {
	// read header
	h := r.Header.Get("Authorization")

	// split header
	s := strings.SplitN(h, " ", 2)
	if len(s) != 2 || !strings.EqualFold(s[0], Bearer) {
		return "", InvalidRequest(NoState, "Malformed or missing authorization header")
	}

	return s[1], nil
}
<commit_msg>added note<commit_after>package oauth2

import (
	"net/http"
	"strings"
)

const Bearer = "bearer"

func NewBearerTokenResponse(token string, expiresIn int) *TokenResponse {
	return NewTokenResponse(Bearer, token, expiresIn)
}

// Note: The spec also allows obtaining the bearer token from query parameters
// and the request body (form data). This implementations only supports obtaining
// the token from the "Authorization" header as this is the most common
// implementation and considered most secure.
func ParseBearerToken(r *http.Request) (string, error) {
	// read header
	h := r.Header.Get("Authorization")

	// split header
	s := strings.SplitN(h, " ", 2)
	if len(s) != 2 || !strings.EqualFold(s[0], Bearer) {
		return "", InvalidRequest(NoState, "Malformed or missing authorization header")
	}

	return s[1], nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package appsync

import (
	"context"
	"fmt"
	"net/http"
	"net/url"

	"github.com/google/exposure-notifications-server/pkg/logging"

	"github.com/google/exposure-notifications-verification-server/internal/clients"
	"github.com/google/exposure-notifications-verification-server/internal/project"
	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/database"

	"github.com/hashicorp/go-multierror"
)

const playStoreHost = `play.google.com/store/apps/details`

// HandleSync performs the logic to sync mobile apps.
func (c *Controller) HandleSync() http.Handler {
	type AppSyncResult struct {
		OK     bool     `json:"ok"`
		Errors []string `json:"errors,omitempty"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		apps, err := c.appSyncClient.AppSync(ctx)
		if err != nil {
			controller.InternalError(w, r, c.h, err)
			return
		}

		// If there are any errors, return them
		if merr := c.syncApps(ctx, apps); merr != nil {
			if errs := merr.WrappedErrors(); len(errs) > 0 {
				// Convert error messages to strings to they appear correctly in JSON.
				// See here for more information:
				//
				//   https://stackoverflow.com/questions/44989924/golang-error-types-are-empty-when-encoded-to-json
				//
				errMsgs := make([]string, len(errs))
				for i, err := range errs {
					errMsgs[i] = err.Error()
				}

				c.h.RenderJSON(w, http.StatusInternalServerError, &AppSyncResult{
					OK:     false,
					Errors: errMsgs,
				})
				return
			}
		}
		c.h.RenderJSON(w, http.StatusOK, &AppSyncResult{OK: true})
	})
}

// syncApps looks up the realm and associated list of MobileApps for each entry of AppsResponse. Then it
// checks to see if there exists an app with the AppResponse SHA hash, if not it creates a new MobileApp.
func (c *Controller) syncApps(ctx context.Context, apps *clients.AppsResponse) *multierror.Error {
	logger := logging.FromContext(ctx).Named("appsync.syncApps")
	var merr *multierror.Error

	realms := map[string]*database.Realm{}
	appsByRealm := map[uint][]*database.MobileApp{}

	for _, app := range apps.Apps {
		realm, err := c.findRealmForApp(app, realms)
		if err != nil {
			if database.IsNotFound(err) {
				logger.Warnw("no app corresponds to region, skipping",
					"app", app.AndroidTarget.AppName,
					"region", app.Region)
			} else {
				merr = multierror.Append(merr, fmt.Errorf("unable to lookup realm for region %q: %w", app.Region, err))
			}
			continue
		}

		realmApps, err := c.findAppsForRealm(realm.ID, appsByRealm)
		if err != nil {
			merr = multierror.Append(merr, fmt.Errorf("unable to list apps for realm %d: %w", realm.ID, err))
			continue
		}

		// Find out if this realm's applist already has an app with this fingerprint.
		hasSHA, hasGeneratedName := false, false
		for _, a := range realmApps {
			if a.SHA == app.SHA256CertFingerprints {
				hasSHA = true
			}
			if a.Name == generateAppName(app) {
				hasGeneratedName = true
			}
		}

		// Didn't find an app. make one.
		if !hasSHA {
			logger.Infow("app not found during sync, adding", "app", app)

			name := generateAppName(app)
			if hasGeneratedName { // add a random string to names on collision
				s, err := project.RandomBase64String(8)
				if err != nil {
					merr = multierror.Append(merr, fmt.Errorf("error generating app name: %w", err))
					continue
				}
				name += " " + s
			}

			var playStoreURL = &url.URL{
				Scheme:   "https",
				Host:     playStoreHost,
				RawQuery: "id=" + app.PackageName,
			}

			newApp := &database.MobileApp{
				Name:    name,
				RealmID: realm.ID,
				URL:     playStoreURL.String(),
				OS:      database.OSTypeAndroid,
				SHA:     app.SHA256CertFingerprints,
				AppID:   app.PackageName,
			}
			if err := c.db.SaveMobileApp(newApp, database.System); err != nil {
				merr = multierror.Append(merr, fmt.Errorf("failed saving mobile app: %w", err))
				continue
			}
		}
	}
	return merr
}

func (c *Controller) findRealmForApp(
	app clients.App, realms map[string]*database.Realm) (*database.Realm, error) {
	var err error
	realm, has := realms[app.Region]
	if !has { // Find this apps region and cache it in our realms map
		realm, err = c.db.FindRealmByRegion(app.Region)
		if err != nil {
			return nil, err
		}
		realms[app.Region] = realm
	}
	return realm, nil
}

func (c *Controller) findAppsForRealm(
	realmID uint, appsByRealm map[uint][]*database.MobileApp) ([]*database.MobileApp, error) {
	var err error
	realmApps, has := appsByRealm[realmID]
	if !has { // Find all of the apps for this realm and cache that list in our appByRealmMap
		realmApps, err = c.db.ListActiveApps(realmID, database.WithAppOS(database.OSTypeAndroid))
		if err != nil {
			return nil, err
		}
		appsByRealm[realmID] = realmApps
	}
	return realmApps, nil
}

func generateAppName(app clients.App) string {
	if app.AppName != "" {
		return app.AppName
	}
	return app.Region + " Android App"
}
<commit_msg>Move appsync warning to debug level (#1563)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package appsync

import (
	"context"
	"fmt"
	"net/http"
	"net/url"

	"github.com/google/exposure-notifications-server/pkg/logging"

	"github.com/google/exposure-notifications-verification-server/internal/clients"
	"github.com/google/exposure-notifications-verification-server/internal/project"
	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/database"

	"github.com/hashicorp/go-multierror"
)

const playStoreHost = `play.google.com/store/apps/details`

// HandleSync performs the logic to sync mobile apps.
func (c *Controller) HandleSync() http.Handler {
	type AppSyncResult struct {
		OK     bool     `json:"ok"`
		Errors []string `json:"errors,omitempty"`
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		apps, err := c.appSyncClient.AppSync(ctx)
		if err != nil {
			controller.InternalError(w, r, c.h, err)
			return
		}

		// If there are any errors, return them
		if merr := c.syncApps(ctx, apps); merr != nil {
			if errs := merr.WrappedErrors(); len(errs) > 0 {
				// Convert error messages to strings to they appear correctly in JSON.
				// See here for more information:
				//
				//   https://stackoverflow.com/questions/44989924/golang-error-types-are-empty-when-encoded-to-json
				//
				errMsgs := make([]string, len(errs))
				for i, err := range errs {
					errMsgs[i] = err.Error()
				}

				c.h.RenderJSON(w, http.StatusInternalServerError, &AppSyncResult{
					OK:     false,
					Errors: errMsgs,
				})
				return
			}
		}
		c.h.RenderJSON(w, http.StatusOK, &AppSyncResult{OK: true})
	})
}

// syncApps looks up the realm and associated list of MobileApps for each entry
// of AppsResponse. Then it checks to see if there exists an app with the
// AppResponse SHA hash, if not it creates a new MobileApp.
func (c *Controller) syncApps(ctx context.Context, apps *clients.AppsResponse) *multierror.Error {
	logger := logging.FromContext(ctx).Named("appsync.syncApps")
	var merr *multierror.Error

	realms := map[string]*database.Realm{}
	appsByRealm := map[uint][]*database.MobileApp{}

	for _, app := range apps.Apps {
		realm, err := c.findRealmForApp(app, realms)
		if err != nil {
			if database.IsNotFound(err) {
				logger.Debugw("no app corresponds to region, skipping",
					"app", app.AndroidTarget.AppName,
					"region", app.Region)
			} else {
				merr = multierror.Append(merr, fmt.Errorf("unable to lookup realm for region %q: %w", app.Region, err))
			}
			continue
		}

		realmApps, err := c.findAppsForRealm(realm.ID, appsByRealm)
		if err != nil {
			merr = multierror.Append(merr, fmt.Errorf("unable to list apps for realm %d: %w", realm.ID, err))
			continue
		}

		// Find out if this realm's applist already has an app with this fingerprint.
		hasSHA, hasGeneratedName := false, false
		for _, a := range realmApps {
			if a.SHA == app.SHA256CertFingerprints {
				hasSHA = true
			}
			if a.Name == generateAppName(app) {
				hasGeneratedName = true
			}
		}

		// Didn't find an app. make one.
		if !hasSHA {
			logger.Infow("app not found during sync, adding", "app", app)

			name := generateAppName(app)
			if hasGeneratedName { // add a random string to names on collision
				s, err := project.RandomBase64String(8)
				if err != nil {
					merr = multierror.Append(merr, fmt.Errorf("error generating app name: %w", err))
					continue
				}
				name += " " + s
			}

			var playStoreURL = &url.URL{
				Scheme:   "https",
				Host:     playStoreHost,
				RawQuery: "id=" + app.PackageName,
			}

			newApp := &database.MobileApp{
				Name:    name,
				RealmID: realm.ID,
				URL:     playStoreURL.String(),
				OS:      database.OSTypeAndroid,
				SHA:     app.SHA256CertFingerprints,
				AppID:   app.PackageName,
			}
			if err := c.db.SaveMobileApp(newApp, database.System); err != nil {
				merr = multierror.Append(merr, fmt.Errorf("failed saving mobile app: %w", err))
				continue
			}
		}
	}
	return merr
}

func (c *Controller) findRealmForApp(
	app clients.App, realms map[string]*database.Realm) (*database.Realm, error) {
	var err error
	realm, has := realms[app.Region]
	if !has { // Find this apps region and cache it in our realms map
		realm, err = c.db.FindRealmByRegion(app.Region)
		if err != nil {
			return nil, err
		}
		realms[app.Region] = realm
	}
	return realm, nil
}

func (c *Controller) findAppsForRealm(
	realmID uint, appsByRealm map[uint][]*database.MobileApp) ([]*database.MobileApp, error) {
	var err error
	realmApps, has := appsByRealm[realmID]
	if !has { // Find all of the apps for this realm and cache that list in our appByRealmMap
		realmApps, err = c.db.ListActiveApps(realmID, database.WithAppOS(database.OSTypeAndroid))
		if err != nil {
			return nil, err
		}
		appsByRealm[realmID] = realmApps
	}
	return realmApps, nil
}

func generateAppName(app clients.App) string {
	if app.AppName != "" {
		return app.AppName
	}
	return app.Region + " Android App"
}
<|endoftext|>


<commit_before>// +build unit_test

package controller_test

import (
	"fmt"

	. "github.com/kubevirt/containerized-data-importer/pkg/common"
	. "github.com/kubevirt/containerized-data-importer/pkg/controller"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pkg/errors"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/tools/cache"
	k8stesting "k8s.io/client-go/tools/cache/testing"
)

type operation int

const (
	opAdd operation = iota
	opUpdate
	opDelete
)

var verboseDebug = fmt.Sprintf("%d", Vdebug)

var _ = Describe("Controller", func() {
	var (
		controller *Controller
		fakeClient *fake.Clientset
		stop       chan struct{}
	)
	type testT struct {
		descr       string
		ns          string
		name        string // name of test pvc
		qops        operation
		annotations map[string]string
		expectError bool
	}

	setUpInformer := func(pvc *v1.PersistentVolumeClaim, op operation) {

		stop = make(chan struct{})
		fakeClient = fake.NewSimpleClientset(pvc)
		pvcSource := k8stesting.NewFakePVCControllerSource()
		podSource := k8stesting.NewFakeControllerSource()

		pvcInformer := cache.NewSharedIndexInformer(pvcSource, pvc, DEFAULT_RESYNC_PERIOD, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
		podInformer := cache.NewSharedIndexInformer(podSource, &v1.Pod{}, DEFAULT_RESYNC_PERIOD, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

		controller = NewController(fakeClient, pvcInformer, podInformer, IMPORTER_DEFAULT_IMAGE, IMPORTER_DEFAULT_PULL_POLICY, verboseDebug)
		if op == opAdd || op == opUpdate {
			pvcSource.Add(pvc)
		}
		go pvcInformer.Run(stop)
		go podInformer.Run(stop)
		Expect(cache.WaitForCacheSync(stop, pvcInformer.HasSynced)).To(BeTrue())
		Expect(cache.WaitForCacheSync(stop, podInformer.HasSynced)).To(BeTrue())
	}

	AfterEach(func() {
		close(stop)
	})

	tests := []testT{
		{
			descr:       "pvc, endpoint, blank ns: controller creates importer pod",
			ns:          "", // blank used originally in these unit tests
			name:        "test-pvc",
			qops:        opAdd,
			annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
			expectError: false,
		},
		{
			descr:       "pvc, endpoint, non-blank ns: controller creates importer pod",
			ns:          "ns-a",
			name:        "test-pvc",
			qops:        opAdd,
			annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
			expectError: false,
		},
		{
			descr:       "pvc, blank endpoint: controller does not create importer pod",
			ns:          "",
			name:        "test-pvc",
			qops:        opAdd,
			annotations: map[string]string{},
			expectError: true,
		},
		{
			descr:       "updated pvc should process",
			ns:          "ns-a",
			name:        "test-pvc",
			qops:        opUpdate,
			annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
			expectError: false,
		},
		{
			descr:       "updated pvc should not process based on annotation AnnImportPod indicating already been processed",
			ns:          "ns-a",
			name:        "test-pvc",
			qops:        opUpdate,
			annotations: map[string]string{AnnEndpoint: "http://www.google.com", AnnImportPod: "importer-test-pvc"},
			expectError: true,
		},
	}

	for _, test := range tests {
		ns := test.ns
		pvcName := test.name
		ops := test.qops
		annotations := test.annotations
		fullname := pvcName
		if len(ns) > 0 {
			fullname = fmt.Sprintf("%s/%s", ns, pvcName)
		}
		exptPod := fmt.Sprintf("importer-%s-", pvcName)
		exptErr := test.expectError

		It(test.descr, func() {
			By(fmt.Sprintf("creating in-mem pvc %q with endpt anno=%q", fullname, annotations))
			pvc := createInMemPVC(ns, pvcName, annotations)
			By("Creating the controller")
			setUpInformer(pvc, ops)
			controller.ProcessNextPvcItem()
			//controller.ProcessNextPodItem()
			By("checking if importer pod is present")
			pod, err := getImporterPod(fakeClient, ns, exptPod)
			if exptErr {
				Expect(err).ToNot(BeNil(), fmt.Sprintf("importer pod %s... should not exist\n", exptPod))
			} else {
				Expect(err).To(BeNil(), fmt.Sprintf("importer pod: %v\n", err))
				Expect(pod).ToNot(BeNil(), fmt.Sprintf("importer pod %q missing", exptPod))
				Expect(pod.GenerateName).To(HavePrefix(exptPod))
			}
		})
	}
})

// return an in-memory pvc using the passed-in namespace, name and the endpoint annotation.
func createInMemPVC(ns, name string, annotations map[string]string) *v1.PersistentVolumeClaim {
	return &v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:        name,
			Namespace:   ns,
			Annotations: annotations,
		},
	}
}

// getImporterPod gets the first pod with a generated name equal to the passed-in name.
// Nil is returned if no match is found.
func getImporterPod(fc *fake.Clientset, ns, podName string) (*v1.Pod, error) {
	podList, err := fc.CoreV1().Pods(ns).List(metav1.ListOptions{})
	if err != nil {
		return nil, errors.Wrapf(err, "could not list pods in namespace %q", ns)
	}
	if len(podList.Items) == 0 {
		return nil, errors.Errorf("Found 0 pods in namespace %q", ns)
	}
	for i, p := range podList.Items {
		if p.GenerateName == podName {
			return &podList.Items[i], nil
		}
	}
	return nil, errors.Errorf("no pods match %s/%s", ns, podName)
}
<commit_msg>Test controller writes pod phase to pvc annotation<commit_after>// +build unit_test

package controller_test

import (
	"fmt"

	. "github.com/kubevirt/containerized-data-importer/pkg/common"
	. "github.com/kubevirt/containerized-data-importer/pkg/controller"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pkg/errors"
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/tools/cache"
	k8stesting "k8s.io/client-go/tools/cache/testing"
)

type operation int

const (
	opAdd    operation = iota
	opUpdate
	opDelete
)

var verboseDebug = fmt.Sprintf("%d", Vdebug)

var _ = Describe("Controller", func() {
	var (
		controller *Controller
		fakeClient *fake.Clientset
		pvcSource  *k8stesting.FakePVCControllerSource
		podSource  *k8stesting.FakeControllerSource
		stop       chan struct{}
	)

	setUpInformers := func(pvc *v1.PersistentVolumeClaim, pod *v1.Pod, ns string, op operation, stop <-chan struct{}) {
		// Populate the informer caches with test objects
		var err error
		pvc, err = fakeClient.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
		Expect(err).NotTo(HaveOccurred())
		pvcSource.Add(pvc)

		if pod != nil {
			pod, err = fakeClient.CoreV1().Pods(ns).Create(pod)
			Expect(err).NotTo(HaveOccurred())
			podSource.Add(pod)
		}

		pvcInformer := cache.NewSharedIndexInformer(pvcSource, pvc, DEFAULT_RESYNC_PERIOD, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
		podInformer := cache.NewSharedIndexInformer(podSource, pod, DEFAULT_RESYNC_PERIOD, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})

		controller = NewController(fakeClient, pvcInformer, podInformer, IMPORTER_DEFAULT_IMAGE, IMPORTER_DEFAULT_PULL_POLICY, verboseDebug)

		go pvcInformer.Run(stop)
		go podInformer.Run(stop)
		Expect(cache.WaitForCacheSync(stop, pvcInformer.HasSynced)).To(BeTrue())
		Expect(cache.WaitForCacheSync(stop, podInformer.HasSynced)).To(BeTrue())
	}

	BeforeEach(func() {
		By("Setting up new fake kubernetes client")
		fakeClient = fake.NewSimpleClientset()
		stop = make(chan struct{})

		By("Setting up new fake controller sources")
		pvcSource = k8stesting.NewFakePVCControllerSource()
		podSource = k8stesting.NewFakeControllerSource()
	})

	AfterEach(func() {
		By("Stopping informer routines")
		close(stop)
	})

	Context("when parsing pvc annotations", func() {

		type testT struct {
			descr       string
			ns          string
			name        string // name of test pvc
			qops        operation
			annotations map[string]string
			expectError bool
		}

		tests := []testT{
			{
				descr:       "pvc, endpoint, blank ns: controller creates importer pod",
				ns:          "", // blank used originally in these unit tests
				name:        "test-pvc",
				qops:        opAdd,
				annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
				expectError: false,
			},
			{
				descr:       "pvc, endpoint, non-blank ns: controller creates importer pod",
				ns:          "ns-a",
				name:        "test-pvc",
				qops:        opAdd,
				annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
				expectError: false,
			},
			{
				descr:       "pvc, blank endpoint: controller does not create importer pod",
				ns:          "",
				name:        "test-pvc",
				qops:        opAdd,
				annotations: map[string]string{},
				expectError: true,
			},
			{
				descr:       "updated pvc should process",
				ns:          "ns-a",
				name:        "test-pvc",
				qops:        opUpdate,
				annotations: map[string]string{AnnEndpoint: "http://www.google.com"},
				expectError: false,
			},
			{
				descr:       "updated pvc should not process based on annotation AnnImportPod indicating already been processed",
				ns:          "ns-a",
				name:        "test-pvc",
				qops:        opUpdate,
				annotations: map[string]string{AnnEndpoint: "http://www.google.com", AnnImportPod: "importer-test-pvc"},
				expectError: true,
			},
		}
		for _, test := range tests {
			ns := test.ns
			pvcName := test.name
			ops := test.qops
			annotations := test.annotations
			fullname := pvcName
			if len(ns) > 0 {
				fullname = fmt.Sprintf("%s/%s", ns, pvcName)
			}
			exptPod := fmt.Sprintf("importer-%s-", pvcName)
			exptErr := test.expectError

			It(test.descr, func() {
				By(fmt.Sprintf("creating in-mem pvc %q with endpt anno=%q", fullname, annotations))
				pvc := createInMemPVC(ns, pvcName, annotations)
				By("Creating the controller")
				setUpInformers(pvc, nil, ns, ops, stop)
				controller.ProcessNextPvcItem()
				By("checking if importer pod is present")
				pod, err := getImporterPod(fakeClient, ns, exptPod)
				if exptErr {
					By("Expecting no pod created")
					Expect(err).ToNot(BeNil(), fmt.Sprintf("importer pod %s... should not exist\n", exptPod))
				} else {
					By("Expecting pod created")
					Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("importer pod: %v\n", err))
					Expect(pod).ToNot(BeNil(), fmt.Sprintf("importer pod %q missing", exptPod))
					Expect(pod.GenerateName).To(HavePrefix(exptPod))
				}
			})
		}
	})

	Context("when annotating pod status in pvc", func() {
		const (
			ep          = "https://www.google.com"
			pvcName     = "test-pvc"
			ns          = "testing-namespace"
			expectPhase = v1.PodPending
		)

		type test struct {
			desc                     string
			podAnn                   map[string]string
			hasVolume, shouldSucceed bool
		}
		tests := []test{
			{
				desc:          fmt.Sprintf("Should annotate the pod phase in the pvc when the pod has annotation \"%s: \"", AnnCreatedBy),
				podAnn:        map[string]string{AnnCreatedBy: ""},
				hasVolume:     true,
				shouldSucceed: true,
			},
			{
				desc:          fmt.Sprintf("Should do nothing when the pod is missing annotation \"%s: %s\"", AnnCreatedBy, ""),
				podAnn:        map[string]string{},
				hasVolume:     true,
				shouldSucceed: false,
			},
			{
				desc:          "Should do nothing if there is no volume attached to the pod",
				podAnn:        map[string]string{AnnCreatedBy: ""},
				hasVolume:     false,
				shouldSucceed: false,
			},
		}

		var pvcAnn = map[string]string{
			AnnEndpoint: ep,
		}

		for _, t := range tests {
			podAnn := t.podAnn
			ss := t.shouldSucceed
			hasVol := t.hasVolume
			It(t.desc, func() {
				By("Setting up API objects and starting informers")
				pvc := createInMemPVC(ns, pvcName, pvcAnn)
				pod := createInMemPod(ns, pvcName, expectPhase, hasVol, podAnn)
				pod.Status.Phase = expectPhase
				setUpInformers(pvc, pod, ns, opAdd, stop)

				By("Initiating pod phase write to pvc annotation")
				controller.ProcessNextPodItem()
				pvc, err := fakeClient.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
				Expect(err).NotTo(HaveOccurred())

				if ss {
					By("Expecting pod phase annotation in the pvc")
					Expect(pvc.Annotations[AnnPodPhase]).To(Equal(string(expectPhase)))
				} else {
					By("Expecting no pod phase annotation in the pvc")
					Expect(pvc.Annotations[AnnPodPhase]).To(BeEmpty())
				}
			})
		}

	})
})

// return an in-memory pvc using the passed-in namespace, name and the endpoint annotation.
func createInMemPVC(ns, name string, annotations map[string]string) *v1.PersistentVolumeClaim {
	return &v1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:        name,
			Namespace:   ns,
			Annotations: annotations,
		},
	}
}

// createInMemPod generates a pod with the passed-in values.
func createInMemPod(ns, pvcName string, phase v1.PodPhase, hasVol bool, annotations map[string]string) *v1.Pod {
	var volName string
	if hasVol {
		volName = DataVolName
	}
	genName := fmt.Sprintf("importer-%s-", pvcName)

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:    ns,
			GenerateName: genName,
			Name:         fmt.Sprintf("%s1234", genName),
			Annotations:  annotations,
		},
		Spec: v1.PodSpec{
			Volumes: []v1.Volume{
				{
					Name: volName,
					VolumeSource: v1.VolumeSource{
						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
							ClaimName: pvcName,
						},
					},
				},
			},
		},
		Status: v1.PodStatus{
			Phase: phase,
		},
	}
}

// getImporterPod gets the first pod with a generated name equal to the passed-in name.
// Nil is returned if no match is found.
func getImporterPod(fc *fake.Clientset, ns, podName string) (*v1.Pod, error) {
	podList, err := fc.CoreV1().Pods(ns).List(metav1.ListOptions{})
	if err != nil {
		return nil, errors.Wrapf(err, "could not list pods in namespace %q", ns)
	}
	if len(podList.Items) == 0 {
		return nil, errors.Errorf("Found 0 pods in namespace %q", ns)
	}
	for i, p := range podList.Items {
		if p.GenerateName == podName {
			return &podList.Items[i], nil
		}
	}
	return nil, errors.Errorf("no pods match %s/%s", ns, podName)
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package groups

import (
	"context"
	"fmt"
	"net"

	"github.com/cilium/cilium/pkg/checker"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
	"github.com/cilium/cilium/pkg/policy/api"

	. "gopkg.in/check.v1"
	"k8s.io/apimachinery/pkg/types"
)

func getSamplePolicy(name, ns string) *cilium_v2.CiliumNetworkPolicy {
	cnp := &cilium_v2.CiliumNetworkPolicy{}

	cnp.ObjectMeta.Name = name
	cnp.ObjectMeta.Namespace = ns
	cnp.ObjectMeta.UID = types.UID("123")
	cnp.Spec = &api.Rule{
		EndpointSelector: api.EndpointSelector{
			LabelSelector: &slim_metav1.LabelSelector{
				MatchLabels: map[string]string{
					"test": "true",
				},
			},
		},
	}
	return cnp
}

func (s *GroupsTestSuite) TestCorrectDerivativeName(c *C) {
	name := "test"
	cnp := getSamplePolicy(name, "testns")
	derivativeCNP, err := createDerivativeCNP(context.TODO(), cnp, false)
	c.Assert(err, IsNil)
	c.Assert(
		derivativeCNP.ObjectMeta.Name,
		Equals,
		fmt.Sprintf("%s-togroups-%s", name, cnp.ObjectMeta.UID))
}

func (s *GroupsTestSuite) TestDerivativePoliciesAreDeletedIfNoToGroups(c *C) {
	name := "test"
	cnp := getSamplePolicy(name, "testns")

	cnp.Spec.Egress = []api.EgressRule{
		{
			ToPorts: []api.PortRule{
				{
					Ports: []api.PortProtocol{
						{Port: "5555"},
					},
				},
			},
		},
	}

	derivativeCNP, err := createDerivativeCNP(context.TODO(), cnp, false)
	c.Assert(err, IsNil)
	c.Assert(derivativeCNP.Specs[0].Egress, checker.DeepEquals, cnp.Spec.Egress)
	c.Assert(len(derivativeCNP.Specs), Equals, 1)
}

func (s *GroupsTestSuite) TestDerivativePoliciesAreInheritCorrectly(c *C) {
	cb := func(ctx context.Context, group *api.ToGroups) ([]net.IP, error) {
		return []net.IP{net.ParseIP("192.168.1.1")}, nil
	}

	api.RegisterToGroupsProvider(api.AWSProvider, cb)

	name := "test"
	cnp := getSamplePolicy(name, "testns")

	cnp.Spec.Egress = []api.EgressRule{
		{
			ToPorts: []api.PortRule{
				{
					Ports: []api.PortProtocol{
						{Port: "5555"},
					},
				},
			},
			ToGroups: []api.ToGroups{
				{
					AWS: &api.AWSGroup{
						Labels: map[string]string{
							"test": "a",
						},
					},
				},
			},
		},
	}

	derivativeCNP, err := createDerivativeCNP(context.TODO(), cnp, false)
	c.Assert(err, IsNil)
	c.Assert(derivativeCNP.Spec, IsNil)
	c.Assert(len(derivativeCNP.Specs), Equals, 1)
	c.Assert(derivativeCNP.Specs[0].Egress[0].ToPorts, checker.DeepEquals, cnp.Spec.Egress[0].ToPorts)
	c.Assert(len(derivativeCNP.Specs[0].Egress[0].ToGroups), Equals, 0)
}
<commit_msg>policy/groups: add unit tests for CCNP toGroups derivative policies<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !privileged_tests

package groups

import (
	"context"
	"fmt"
	"net"

	"github.com/cilium/cilium/pkg/checker"
	cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
	slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
	"github.com/cilium/cilium/pkg/policy/api"

	. "gopkg.in/check.v1"
	"k8s.io/apimachinery/pkg/types"
)

func getSamplePolicy(name, ns string) *cilium_v2.CiliumNetworkPolicy {
	cnp := &cilium_v2.CiliumNetworkPolicy{}

	cnp.ObjectMeta.Name = name
	cnp.ObjectMeta.Namespace = ns
	cnp.ObjectMeta.UID = types.UID("123")
	cnp.Spec = &api.Rule{
		EndpointSelector: api.EndpointSelector{
			LabelSelector: &slim_metav1.LabelSelector{
				MatchLabels: map[string]string{
					"test": "true",
				},
			},
		},
	}
	return cnp
}

func (s *GroupsTestSuite) TestCorrectDerivativeName(c *C) {
	name := "test"
	cnp := getSamplePolicy(name, "testns")
	cnpDerivedPolicy, err := createDerivativeCNP(context.TODO(), cnp)
	c.Assert(err, IsNil)
	c.Assert(
		cnpDerivedPolicy.ObjectMeta.Name,
		Equals,
		fmt.Sprintf("%s-togroups-%s", name, cnp.ObjectMeta.UID))

	// Test clusterwide policy helper functions
	ccnpName := "ccnp-test"
	ccnp := getSamplePolicy(ccnpName, "")
	ccnpDerivedPolicy, err := createDerivativeCCNP(context.TODO(), ccnp)

	c.Assert(err, IsNil)
	c.Assert(
		ccnpDerivedPolicy.ObjectMeta.Name,
		Equals,
		fmt.Sprintf("%s-togroups-%s", ccnpName, ccnp.ObjectMeta.UID),
	)
}

func (s *GroupsTestSuite) TestDerivativePoliciesAreDeletedIfNoToGroups(c *C) {
	egressRule := []api.EgressRule{
		{
			ToPorts: []api.PortRule{
				{
					Ports: []api.PortProtocol{
						{Port: "5555"},
					},
				},
			},
		},
	}

	name := "test"
	cnp := getSamplePolicy(name, "testns")

	cnp.Spec.Egress = egressRule

	cnpDerivedPolicy, err := createDerivativeCNP(context.TODO(), cnp)
	c.Assert(err, IsNil)
	c.Assert(cnpDerivedPolicy.Specs[0].Egress, checker.DeepEquals, cnp.Spec.Egress)
	c.Assert(len(cnpDerivedPolicy.Specs), Equals, 1)

	// Clusterwide policies
	ccnpName := "ccnp-test"
	ccnp := getSamplePolicy(ccnpName, "")
	ccnp.Spec.Egress = egressRule

	ccnpDerivedPolicy, err := createDerivativeCCNP(context.TODO(), ccnp)
	c.Assert(err, IsNil)
	c.Assert(ccnpDerivedPolicy.Specs[0].Egress, checker.DeepEquals, ccnp.Spec.Egress)
	c.Assert(len(ccnpDerivedPolicy.Specs), Equals, 1)
}

func (s *GroupsTestSuite) TestDerivativePoliciesAreInheritCorrectly(c *C) {
	cb := func(ctx context.Context, group *api.ToGroups) ([]net.IP, error) {
		return []net.IP{net.ParseIP("192.168.1.1")}, nil
	}

	egressRule := []api.EgressRule{
		{
			ToPorts: []api.PortRule{
				{
					Ports: []api.PortProtocol{
						{Port: "5555"},
					},
				},
			},
			ToGroups: []api.ToGroups{
				{
					AWS: &api.AWSGroup{
						Labels: map[string]string{
							"test": "a",
						},
					},
				},
			},
		},
	}

	api.RegisterToGroupsProvider(api.AWSProvider, cb)

	name := "test"
	cnp := getSamplePolicy(name, "testns")

	cnp.Spec.Egress = egressRule

	cnpDerivedPolicy, err := createDerivativeCNP(context.TODO(), cnp)
	c.Assert(err, IsNil)
	c.Assert(cnpDerivedPolicy.Spec, IsNil)
	c.Assert(len(cnpDerivedPolicy.Specs), Equals, 1)
	c.Assert(cnpDerivedPolicy.Specs[0].Egress[0].ToPorts, checker.DeepEquals, cnp.Spec.Egress[0].ToPorts)
	c.Assert(len(cnpDerivedPolicy.Specs[0].Egress[0].ToGroups), Equals, 0)

	// Clusterwide policies
	ccnpName := "ccnp-test"
	ccnp := getSamplePolicy(ccnpName, "")
	ccnp.Spec.Egress = egressRule

	ccnpDerivedPolicy, err := createDerivativeCCNP(context.TODO(), ccnp)
	c.Assert(err, IsNil)
	c.Assert(ccnpDerivedPolicy.Spec, IsNil)
	c.Assert(len(ccnpDerivedPolicy.Specs), Equals, 1)
	c.Assert(ccnpDerivedPolicy.Specs[0].Egress[0].ToPorts, checker.DeepEquals, ccnp.Spec.Egress[0].ToPorts)
	c.Assert(len(ccnpDerivedPolicy.Specs[0].Egress[0].ToGroups), Equals, 0)
}
<|endoftext|>


<commit_before>// +build !linux,!windows,!freebsd,!darwin

package reexec // import "github.com/docker/docker/pkg/reexec"

import (
	"os/exec"
)

// Command is unsupported on operating systems apart from Linux, Windows, and Darwin.
func Command(args ...string) *exec.Cmd {
	return nil
}
<commit_msg>fix build on OpenBSD by defining Self()<commit_after>// +build !linux,!windows,!freebsd,!darwin

package reexec // import "github.com/docker/docker/pkg/reexec"

import (
	"os/exec"
)

func Self() string {
	return ""
}

// Command is unsupported on operating systems apart from Linux, Windows, and Darwin.
func Command(args ...string) *exec.Cmd {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pai

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"text/template"

	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/tensorflow"
)

// TFTrainAndSave generates PAI-TF train program.
func TFTrainAndSave(ir *ir.TrainStmt, session *pb.Session, modelPath string, cc *ClusterConfig) (string, error) {
	// Distributed training must call train_and_evaluate, which need the user to specify validation.select
	valSelect, valOK := ir.Attributes["validation.select"]
	hasVal := true
	if !valOK || valSelect.(string) == "" {
		hasVal = false
	}
	if cc.Worker.Count > 1 && !hasVal {
		return "", fmt.Errorf("Distributed training must specify WITH validation.select")
	}
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ckptDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}

	code, err := tensorflow.Train(ir, session)
	if err != nil {
		return "", err
	}

	// append code snippet to save model
	var tpl = template.Must(template.New("SaveModel").Parse(tfSaveModelTmplText))
	filler := saveModelFiller{
		OSSModelDir: ckptDir,
		Estimator:   ir.Estimator,
		NumWorkers:  cc.Worker.Count,
	}
	var saveCode bytes.Buffer
	if err = tpl.Execute(&saveCode, filler); err != nil {
		return "", err
	}
	return code + saveCode.String(), nil
}

// TFLoadAndPredict generates PAI-TF prediction program.
func TFLoadAndPredict(ir *ir.PredictStmt, session *pb.Session, modelPath string) (string, error) {
	var tpl = template.Must(template.New("Predict").Parse(tfPredictTmplText))
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ossModelDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}
	paiPredictTable := ""
	if tensorflow.IsPAI() && ir.TmpPredictTable != "" {
		paiPredictTable = ir.TmpPredictTable
	}
	filler := predictFiller{
		OSSModelDir: ossModelDir,
		DataSource:  session.DbConnStr,
		Select:      ir.Select,
		ResultTable: ir.ResultTable,
		IsPAI:       tensorflow.IsPAI(),
		PAITable:    paiPredictTable,
		Using:       ir.Using,
	}
	var code bytes.Buffer
	if err := tpl.Execute(&code, filler); err != nil {
		return "", err
	}
	return code.String(), nil
}

// TFLoadAndExplain generates PAI-TF explain program.
func TFLoadAndExplain(ir *ir.ExplainStmt, session *pb.Session, modelPath string, expn *ExplainRender) (string, error) {
	var tpl = template.Must(template.New("Explain").Parse(tfExplainTmplText))
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ossModelDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}
	paiExplainTable := ""
	if tensorflow.IsPAI() && ir.TmpExplainTable != "" {
		paiExplainTable = ir.TmpExplainTable
	}
	filler := explainFiller{
		OSSModelDir: ossModelDir,
		DataSource:  session.DbConnStr,
		Select:      ir.Select,
		ResultTable: ir.Into,
		IsPAI:       tensorflow.IsPAI(),
		PAITable:    paiExplainTable,
		// TODO(weiguo): use GFile to write oss without ak/sk
		// ref: https://yuque.antfin-inc.com/pai-user/manual/tf_oss_by_gfile
		ResultOSSDest:     expn.key,
		ResultOSSAK:       expn.ak,
		ResultOSSSK:       expn.sk,
		ResultOSSEndpoint: expn.endpoint,
		ResultOSSBucket:   expn.bucket,
	}
	var code bytes.Buffer
	if err := tpl.Execute(&code, filler); err != nil {
		return "", err
	}
	return code.String(), nil
}

func getTFPAICmd(cc *ClusterConfig, tarball, modelName, ossModelPath, trainTable, valTable, resTable, project string) (string, error) {
	jobName := strings.Replace(strings.Join([]string{"sqlflow", modelName}, "_"), ".", "_", 0)
	cfString, err := json.Marshal(cc)
	if err != nil {
		return "", err
	}
	cfQuote := strconv.Quote(string(cfString))
	ckpDir, err := checkpointURL(ossModelPath, project)
	if err != nil {
		return "", err
	}

	// submit table should format as: odps://<project>/tables/<table>,odps://<project>/tables/<table>...
	submitTables, err := maxComputeTableURL(trainTable)
	if err != nil {
		return "", err
	}
	if trainTable != valTable && valTable != "" {
		valTable, err := maxComputeTableURL(valTable)
		if err != nil {
			return "", err
		}
		submitTables = fmt.Sprintf("%s,%s", submitTables, valTable)
	}
	outputTables := ""
	if resTable != "" {
		table, err := maxComputeTableURL(resTable)
		if err != nil {
			return "", err
		}
		outputTables = fmt.Sprintf("-Doutputs=%s", table)
	}

	cmd := fmt.Sprintf("pai -name tensorflow1150 -project algo_public_dev -DmaxHungTimeBeforeGCInSeconds=0 -DjobName=%s -Dtags=dnn -Dscript=%s -DentryFile=entry.py -Dtables=%s %s -DcheckpointDir=\"%s\"", jobName, tarball, submitTables, outputTables, ckpDir)
	if cc.Worker.Count > 1 {
		cmd = fmt.Sprintf("%s -Dcluster=%s", cmd, cfQuote)
	} else {
		// TODO(typhoonzero): configure single node gpu resource if needed.
		cmd = fmt.Sprintf("%s -DgpuRequired='0'", cmd)
	}
	return cmd, nil
}
<commit_msg>single node gpu training for pai (#1947)<commit_after>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pai

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"text/template"

	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/tensorflow"
)

// TFTrainAndSave generates PAI-TF train program.
func TFTrainAndSave(ir *ir.TrainStmt, session *pb.Session, modelPath string, cc *ClusterConfig) (string, error) {
	// Distributed training must call train_and_evaluate, which need the user to specify validation.select
	valSelect, valOK := ir.Attributes["validation.select"]
	hasVal := true
	if !valOK || valSelect.(string) == "" {
		hasVal = false
	}
	if cc.Worker.Count > 1 && !hasVal {
		return "", fmt.Errorf("Distributed training must specify WITH validation.select")
	}
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ckptDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}

	code, err := tensorflow.Train(ir, session)
	if err != nil {
		return "", err
	}

	// append code snippet to save model
	var tpl = template.Must(template.New("SaveModel").Parse(tfSaveModelTmplText))
	filler := saveModelFiller{
		OSSModelDir: ckptDir,
		Estimator:   ir.Estimator,
		NumWorkers:  cc.Worker.Count,
	}
	var saveCode bytes.Buffer
	if err = tpl.Execute(&saveCode, filler); err != nil {
		return "", err
	}
	return code + saveCode.String(), nil
}

// TFLoadAndPredict generates PAI-TF prediction program.
func TFLoadAndPredict(ir *ir.PredictStmt, session *pb.Session, modelPath string) (string, error) {
	var tpl = template.Must(template.New("Predict").Parse(tfPredictTmplText))
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ossModelDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}
	paiPredictTable := ""
	if tensorflow.IsPAI() && ir.TmpPredictTable != "" {
		paiPredictTable = ir.TmpPredictTable
	}
	filler := predictFiller{
		OSSModelDir: ossModelDir,
		DataSource:  session.DbConnStr,
		Select:      ir.Select,
		ResultTable: ir.ResultTable,
		IsPAI:       tensorflow.IsPAI(),
		PAITable:    paiPredictTable,
		Using:       ir.Using,
	}
	var code bytes.Buffer
	if err := tpl.Execute(&code, filler); err != nil {
		return "", err
	}
	return code.String(), nil
}

// TFLoadAndExplain generates PAI-TF explain program.
func TFLoadAndExplain(ir *ir.ExplainStmt, session *pb.Session, modelPath string, expn *ExplainRender) (string, error) {
	var tpl = template.Must(template.New("Explain").Parse(tfExplainTmplText))
	currProject, err := database.GetDatabaseName(session.DbConnStr)
	if err != nil {
		return "", err
	}
	ossModelDir, err := checkpointURL(modelPath, currProject)
	if err != nil {
		return "", err
	}
	paiExplainTable := ""
	if tensorflow.IsPAI() && ir.TmpExplainTable != "" {
		paiExplainTable = ir.TmpExplainTable
	}
	filler := explainFiller{
		OSSModelDir: ossModelDir,
		DataSource:  session.DbConnStr,
		Select:      ir.Select,
		ResultTable: ir.Into,
		IsPAI:       tensorflow.IsPAI(),
		PAITable:    paiExplainTable,
		// TODO(weiguo): use GFile to write oss without ak/sk
		// ref: https://yuque.antfin-inc.com/pai-user/manual/tf_oss_by_gfile
		ResultOSSDest:     expn.key,
		ResultOSSAK:       expn.ak,
		ResultOSSSK:       expn.sk,
		ResultOSSEndpoint: expn.endpoint,
		ResultOSSBucket:   expn.bucket,
	}
	var code bytes.Buffer
	if err := tpl.Execute(&code, filler); err != nil {
		return "", err
	}
	return code.String(), nil
}

func getTFPAICmd(cc *ClusterConfig, tarball, modelName, ossModelPath, trainTable, valTable, resTable, project string) (string, error) {
	jobName := strings.Replace(strings.Join([]string{"sqlflow", modelName}, "_"), ".", "_", 0)
	cfString, err := json.Marshal(cc)
	if err != nil {
		return "", err
	}
	cfQuote := strconv.Quote(string(cfString))
	ckpDir, err := checkpointURL(ossModelPath, project)
	if err != nil {
		return "", err
	}

	// submit table should format as: odps://<project>/tables/<table>,odps://<project>/tables/<table>...
	submitTables, err := maxComputeTableURL(trainTable)
	if err != nil {
		return "", err
	}
	if trainTable != valTable && valTable != "" {
		valTable, err := maxComputeTableURL(valTable)
		if err != nil {
			return "", err
		}
		submitTables = fmt.Sprintf("%s,%s", submitTables, valTable)
	}
	outputTables := ""
	if resTable != "" {
		table, err := maxComputeTableURL(resTable)
		if err != nil {
			return "", err
		}
		outputTables = fmt.Sprintf("-Doutputs=%s", table)
	}

	cmd := fmt.Sprintf("pai -name tensorflow1150 -project algo_public_dev -DmaxHungTimeBeforeGCInSeconds=0 -DjobName=%s -Dtags=dnn -Dscript=%s -DentryFile=entry.py -Dtables=%s %s -DcheckpointDir=\"%s\"", jobName, tarball, submitTables, outputTables, ckpDir)
	if cc.Worker.Count > 1 {
		cmd = fmt.Sprintf("%s -Dcluster=%s", cmd, cfQuote)
	} else {
		cmd = fmt.Sprintf("%s -DgpuRequired='%d'", cmd, cc.Worker.GPU)
	}
	return cmd, nil
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/databus23/guttle"
	"github.com/go-kit/kit/log"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	informers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	utilexec "k8s.io/utils/exec"

	"github.com/sapcc/kubernikus/pkg/util/iptables"
)

const (
	KUBERNIKUS_TUNNELS iptables.Chain = "KUBERNIKUS-TUNNELS"
)

type Controller struct {
	nodes       informers.NodeInformer
	tunnel      *guttle.Server
	queue       workqueue.RateLimitingInterface
	store       map[string][]route
	iptables    iptables.Interface
	hijackPort  int
	serviceCIDR string

	Logger log.Logger
}

type route struct {
	cidr       string
	identifier string
}

func NewController(informer informers.NodeInformer, serviceCIDR string, tunnel *guttle.Server, logger log.Logger) *Controller {
	logger = log.With(logger, "controller", "tunnel")
	c := &Controller{
		nodes:       informer,
		tunnel:      tunnel,
		queue:       workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		store:       make(map[string][]route),
		iptables:    iptables.New(utilexec.New(), iptables.ProtocolIpv4, logger),
		hijackPort:  9191,
		serviceCIDR: serviceCIDR,
		Logger:      logger,
	}

	//Always forward requests to the serviceCIDR range
	c.tunnel.AddRoute(serviceCIDR)

	c.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
	})

	return c
}

func (c *Controller) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer c.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	c.Logger.Log(
		"msg", "starting WormholeGenerator",
		"threadiness", threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	ticker := time.NewTicker(5 * time.Minute)
	go func() {
		for {
			select {
			case <-ticker.C:
				c.Logger.Log(
					"msg", "Running periodic recheck. Queuing all known nodes...",
					"v", 5)
				for key := range c.store {
					c.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (c *Controller) runWorker() {
	for c.processNextWorkItem() {
	}
}

func (c *Controller) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	// Invoke the method containing the business logic
	err := c.reconcile(key.(string))
	c.handleErr(err, key)
	return true
}

func (c *Controller) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		c.queue.Forget(key)
		return
	}
	c.Logger.Log(
		"msg", "requeuing because of error",
		"key", key,
		"err", err)

	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if c.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		c.queue.AddRateLimited(key)
		return
	}

	c.Logger.Log(
		"msg", "dropping because of too many error",
		"key", key)
	c.queue.Forget(key)
}

func (c *Controller) reconcile(key string) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return c.delNode(key)
	}

	return c.addNode(key, obj.(*v1.Node))
}

func (c *Controller) addNode(key string, node *v1.Node) error {

	identifier := fmt.Sprintf("system:node:%v", node.GetName())
	c.Logger.Log(
		"msg", "adding tunnel routes",
		"node", identifier)

	podCIDR := node.Spec.PodCIDR

	ip, err := GetNodeHostIP(node)
	if err != nil {
		return err
	}
	nodeCIDR := ip.String() + "/32"

	if err := c.tunnel.AddClientRoute(podCIDR, identifier); err != nil {
		return err
	}
	c.store[key] = append(c.store[key], route{cidr: podCIDR, identifier: identifier})
	if err := c.tunnel.AddRoute(podCIDR); err != nil {
		return err
	}
	if err := c.tunnel.AddClientRoute(nodeCIDR, identifier); err != nil {
		return err
	}
	c.store[key] = append(c.store[key], route{cidr: nodeCIDR, identifier: identifier})
	if err := c.tunnel.AddRoute(nodeCIDR); err != nil {
		return err
	}

	return c.redoIPTablesSpratz()
}

func (c *Controller) delNode(key string) error {
	routes := c.store[key]
	for _, route := range routes {
		c.tunnel.DeleteClientRoute(route.cidr, route.identifier)
		c.tunnel.DeleteRoute(route.cidr)
	}
	return c.redoIPTablesSpratz()
}

func (c *Controller) redoIPTablesSpratz() error {
	table := iptables.TableNAT

	if _, err := c.iptables.EnsureChain(table, KUBERNIKUS_TUNNELS); err != nil {
		c.Logger.Log(
			"msg", "failed to ensure that chain exists",
			"table", table,
			"chain", KUBERNIKUS_TUNNELS,
			"err", err)
		return err
	}

	args := []string{"-m", "comment", "--comment", "kubernikus tunnels", "-j", string(KUBERNIKUS_TUNNELS)}
	if _, err := c.iptables.EnsureRule(iptables.Append, table, iptables.ChainOutput, args...); err != nil {
		c.Logger.Log(
			"msg", "failed to ensure jump",
			"table", table,
			"target", iptables.ChainOutput,
			"chain", KUBERNIKUS_TUNNELS,
			"err", err)
		return err
	}

	iptablesSaveRaw := bytes.NewBuffer(nil)
	existingNatChains := make(map[iptables.Chain]string)
	err := c.iptables.SaveInto(table, iptablesSaveRaw)
	if err != nil {
		c.Logger.Log(
			"msg", "failed to execute iptables-save, syncing all rules",
			"err", err)
	} else {
		existingNatChains = iptables.GetChainLines(table, iptablesSaveRaw.Bytes())
	}

	natChains := bytes.NewBuffer(nil)
	natRules := bytes.NewBuffer(nil)
	writeLine(natChains, "*nat")
	if chain, ok := existingNatChains[KUBERNIKUS_TUNNELS]; ok {
		writeLine(natChains, chain)
	} else {
		writeLine(natChains, iptables.MakeChainLine(KUBERNIKUS_TUNNELS))
	}

	for key := range c.store {
		err := c.writeTunnelRedirect(key, natRules)
		if err != nil {
			return err
		}
	}

	writeLine(natRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", `"cluster service CIDR tunnel redirect"`,
		"--dst", c.serviceCIDR,
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", strconv.Itoa(c.hijackPort),
	)

	writeLine(natRules, "COMMIT")

	lines := append(natChains.Bytes(), natRules.Bytes()...)
	c.Logger.Log(
		"msg", "Restoring iptables rules",
		"rules", lines,
		"v", 6)

	err = c.iptables.RestoreAll(lines, iptables.NoFlushTables, iptables.RestoreCounters)
	if err != nil {
		c.Logger.Log(
			"msg", "Failed to execute iptables-restore",
			"err", err)
		return err
	}

	return nil
}

func (c *Controller) writeTunnelRedirect(key string, filterRules *bytes.Buffer) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return nil
	}

	node := obj.(*v1.Node)
	ip, err := GetNodeHostIP(node)
	if err != nil {
		return err
	}

	port := strconv.Itoa(c.hijackPort)

	writeLine(filterRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", fmt.Sprintf(`"node ip tunnel redirect for %s"`, key),
		"--dst", ip.String(),
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", port,
	)

	writeLine(filterRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", fmt.Sprintf(`"pod cidr tunnel redirect for %s"`, key),
		"--dst", node.Spec.PodCIDR,
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", port,
	)

	return nil
}

func writeLine(buf *bytes.Buffer, words ...string) {
	buf.WriteString(strings.Join(words, " ") + "\n")
}

func GetNodeHostIP(node *v1.Node) (net.IP, error) {
	addresses := node.Status.Addresses
	addressMap := make(map[v1.NodeAddressType][]v1.NodeAddress)
	for i := range addresses {
		addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i])
	}
	if addresses, ok := addressMap[v1.NodeInternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	if addresses, ok := addressMap[v1.NodeExternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	return nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses)
}
<commit_msg>wormhole: skip nodes with empty podCIDR<commit_after>package server

import (
	"bytes"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/databus23/guttle"
	"github.com/go-kit/kit/log"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	informers "k8s.io/client-go/informers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	utilexec "k8s.io/utils/exec"

	"github.com/sapcc/kubernikus/pkg/util/iptables"
)

const (
	KUBERNIKUS_TUNNELS iptables.Chain = "KUBERNIKUS-TUNNELS"
)

type Controller struct {
	nodes       informers.NodeInformer
	tunnel      *guttle.Server
	queue       workqueue.RateLimitingInterface
	store       map[string][]route
	iptables    iptables.Interface
	hijackPort  int
	serviceCIDR string

	Logger log.Logger
}

type route struct {
	cidr       string
	identifier string
}

func NewController(informer informers.NodeInformer, serviceCIDR string, tunnel *guttle.Server, logger log.Logger) *Controller {
	logger = log.With(logger, "controller", "tunnel")
	c := &Controller{
		nodes:       informer,
		tunnel:      tunnel,
		queue:       workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		store:       make(map[string][]route),
		iptables:    iptables.New(utilexec.New(), iptables.ProtocolIpv4, logger),
		hijackPort:  9191,
		serviceCIDR: serviceCIDR,
		Logger:      logger,
	}

	//Always forward requests to the serviceCIDR range
	c.tunnel.AddRoute(serviceCIDR)

	c.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				c.queue.Add(key)
			}
		},
	})

	return c
}

func (c *Controller) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer c.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	c.Logger.Log(
		"msg", "starting WormholeGenerator",
		"threadiness", threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	c.redoIPTablesSpratz() // do the iptables thing at least once to make the service forward is set up

	ticker := time.NewTicker(5 * time.Minute)
	go func() {
		for {
			select {
			case <-ticker.C:
				c.Logger.Log(
					"msg", "Running periodic recheck. Queuing all known nodes...",
					"v", 5)
				for key := range c.store {
					c.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (c *Controller) runWorker() {
	for c.processNextWorkItem() {
	}
}

func (c *Controller) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	// Invoke the method containing the business logic
	err := c.reconcile(key.(string))
	c.handleErr(err, key)
	return true
}

func (c *Controller) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		c.queue.Forget(key)
		return
	}
	c.Logger.Log(
		"msg", "requeuing because of error",
		"key", key,
		"err", err)

	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if c.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		c.queue.AddRateLimited(key)
		return
	}

	c.Logger.Log(
		"msg", "dropping because of too many error",
		"key", key)
	c.queue.Forget(key)
}

func (c *Controller) reconcile(key string) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return c.delNode(key)
	}

	return c.addNode(key, obj.(*v1.Node))
}

func (c *Controller) addNode(key string, node *v1.Node) error {

	identifier := fmt.Sprintf("system:node:%v", node.GetName())

	podCIDR := node.Spec.PodCIDR
	if podCIDR == "" {
		c.Logger.Log(
			"msg", "skipping node with empty spec.PodCIDR",
			"node", identifier,
		)
		return nil
	}

	c.Logger.Log(
		"msg", "adding tunnel routes",
		"node", identifier)

	ip, err := GetNodeHostIP(node)
	if err != nil {
		return err
	}
	nodeCIDR := ip.String() + "/32"

	if err := c.tunnel.AddClientRoute(podCIDR, identifier); err != nil {
		return err
	}
	c.store[key] = append(c.store[key], route{cidr: podCIDR, identifier: identifier})
	if err := c.tunnel.AddRoute(podCIDR); err != nil {
		return err
	}
	if err := c.tunnel.AddClientRoute(nodeCIDR, identifier); err != nil {
		return err
	}
	c.store[key] = append(c.store[key], route{cidr: nodeCIDR, identifier: identifier})
	if err := c.tunnel.AddRoute(nodeCIDR); err != nil {
		return err
	}

	return c.redoIPTablesSpratz()
}

func (c *Controller) delNode(key string) error {
	routes := c.store[key]
	for _, route := range routes {
		c.tunnel.DeleteClientRoute(route.cidr, route.identifier)
		c.tunnel.DeleteRoute(route.cidr)
	}
	return c.redoIPTablesSpratz()
}

func (c *Controller) redoIPTablesSpratz() error {
	table := iptables.TableNAT

	if _, err := c.iptables.EnsureChain(table, KUBERNIKUS_TUNNELS); err != nil {
		c.Logger.Log(
			"msg", "failed to ensure that chain exists",
			"table", table,
			"chain", KUBERNIKUS_TUNNELS,
			"err", err)
		return err
	}

	args := []string{"-m", "comment", "--comment", "kubernikus tunnels", "-j", string(KUBERNIKUS_TUNNELS)}
	if _, err := c.iptables.EnsureRule(iptables.Append, table, iptables.ChainOutput, args...); err != nil {
		c.Logger.Log(
			"msg", "failed to ensure jump",
			"table", table,
			"target", iptables.ChainOutput,
			"chain", KUBERNIKUS_TUNNELS,
			"err", err)
		return err
	}

	iptablesSaveRaw := bytes.NewBuffer(nil)
	existingNatChains := make(map[iptables.Chain]string)
	err := c.iptables.SaveInto(table, iptablesSaveRaw)
	if err != nil {
		c.Logger.Log(
			"msg", "failed to execute iptables-save, syncing all rules",
			"err", err)
	} else {
		existingNatChains = iptables.GetChainLines(table, iptablesSaveRaw.Bytes())
	}

	natChains := bytes.NewBuffer(nil)
	natRules := bytes.NewBuffer(nil)
	writeLine(natChains, "*nat")
	if chain, ok := existingNatChains[KUBERNIKUS_TUNNELS]; ok {
		writeLine(natChains, chain)
	} else {
		writeLine(natChains, iptables.MakeChainLine(KUBERNIKUS_TUNNELS))
	}

	for key := range c.store {
		err := c.writeTunnelRedirect(key, natRules)
		if err != nil {
			return err
		}
	}

	writeLine(natRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", `"cluster service CIDR tunnel redirect"`,
		"--dst", c.serviceCIDR,
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", strconv.Itoa(c.hijackPort),
	)

	writeLine(natRules, "COMMIT")

	lines := append(natChains.Bytes(), natRules.Bytes()...)
	c.Logger.Log(
		"msg", "Restoring iptables rules",
		"rules", lines,
		"v", 6)

	err = c.iptables.RestoreAll(lines, iptables.NoFlushTables, iptables.RestoreCounters)
	if err != nil {
		c.Logger.Log(
			"msg", "Failed to execute iptables-restore",
			"err", err)
		return err
	}

	return nil
}

func (c *Controller) writeTunnelRedirect(key string, filterRules *bytes.Buffer) error {
	obj, exists, err := c.nodes.Informer().GetIndexer().GetByKey(key)
	if err != nil {
		return err
	}

	if !exists {
		return nil
	}

	node := obj.(*v1.Node)
	ip, err := GetNodeHostIP(node)
	if err != nil {
		return err
	}

	port := strconv.Itoa(c.hijackPort)

	writeLine(filterRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", fmt.Sprintf(`"node ip tunnel redirect for %s"`, key),
		"--dst", ip.String(),
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", port,
	)

	writeLine(filterRules,
		"-A", string(KUBERNIKUS_TUNNELS),
		"-m", "comment", "--comment", fmt.Sprintf(`"pod cidr tunnel redirect for %s"`, key),
		"--dst", node.Spec.PodCIDR,
		"-p", "tcp",
		"-j", "REDIRECT",
		"--to-ports", port,
	)

	return nil
}

func writeLine(buf *bytes.Buffer, words ...string) {
	buf.WriteString(strings.Join(words, " ") + "\n")
}

func GetNodeHostIP(node *v1.Node) (net.IP, error) {
	addresses := node.Status.Addresses
	addressMap := make(map[v1.NodeAddressType][]v1.NodeAddress)
	for i := range addresses {
		addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i])
	}
	if addresses, ok := addressMap[v1.NodeInternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	if addresses, ok := addressMap[v1.NodeExternalIP]; ok {
		return net.ParseIP(addresses[0].Address), nil
	}
	return nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses)
}
<|endoftext|>


<commit_before>package api

import (
	"flag"
	"net"
	"net/http/httputil"
	"net/url"

	"github.com/raintank/dur"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
)

var (
	maxPointsPerReqSoft int
	maxPointsPerReqHard int
	logMinDurStr        string
	logMinDur           uint32

	Addr             string
	UseSSL           bool
	useGzip          bool
	certFile         string
	keyFile          string
	multiTenant      bool
	fallbackGraphite string

	graphiteProxy *httputil.ReverseProxy
)

func ConfigSetup() {
	apiCfg := flag.NewFlagSet("http", flag.ExitOnError)
	apiCfg.IntVar(&maxPointsPerReqSoft, "max-points-per-req-soft", 1000000, "lower resolution rollups will be used to try and keep requests below this number of datapoints. (0 disables limit)")
	apiCfg.IntVar(&maxPointsPerReqHard, "max-points-per-req-hard", 20000000, "limit of number of datapoints a request can return. Requests that exceed this limit will be rejected. (0 disables limit)")
	apiCfg.StringVar(&logMinDurStr, "log-min-dur", "5min", "only log incoming requests if their timerange is at least this duration. Use 0 to disable")

	apiCfg.StringVar(&Addr, "listen", ":6060", "http listener address.")
	apiCfg.BoolVar(&UseSSL, "ssl", false, "use HTTPS")
	apiCfg.BoolVar(&useGzip, "gzip", true, "use GZIP compression of all responses")
	apiCfg.StringVar(&certFile, "cert-file", "", "SSL certificate file")
	apiCfg.StringVar(&keyFile, "key-file", "", "SSL key file")
	apiCfg.BoolVar(&multiTenant, "multi-tenant", true, "require x-org-id authentication to auth as a specific org. otherwise orgId 1 is assumed")
	apiCfg.StringVar(&fallbackGraphite, "fallback-graphite-addr", "http://localhost:8080", "in case our /render endpoint does not support the requested processing, proxy the request to this graphite")
	globalconf.Register("http", apiCfg)
}

func ConfigProcess() {
	logMinDur = dur.MustParseUsec("log-min-dur", logMinDurStr)

	//validate the addr
	_, err := net.ResolveTCPAddr("tcp", Addr)
	if err != nil {
		log.Fatal(4, "API listen address is not a valid TCP address.")
	}

	u, err := url.Parse(fallbackGraphite)
	if err != nil {
		log.Fatal(4, "API Cannot parse fallback-graphite-addr: %s", err)
	}
	graphiteProxy = httputil.NewSingleHostReverseProxy(u)
}
<commit_msg>fix duplicate access control headers leading to browser blocking requests<commit_after>package api

import (
	"flag"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/raintank/dur"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
)

var (
	maxPointsPerReqSoft int
	maxPointsPerReqHard int
	logMinDurStr        string
	logMinDur           uint32

	Addr             string
	UseSSL           bool
	useGzip          bool
	certFile         string
	keyFile          string
	multiTenant      bool
	fallbackGraphite string

	graphiteProxy *httputil.ReverseProxy
)

func ConfigSetup() {
	apiCfg := flag.NewFlagSet("http", flag.ExitOnError)
	apiCfg.IntVar(&maxPointsPerReqSoft, "max-points-per-req-soft", 1000000, "lower resolution rollups will be used to try and keep requests below this number of datapoints. (0 disables limit)")
	apiCfg.IntVar(&maxPointsPerReqHard, "max-points-per-req-hard", 20000000, "limit of number of datapoints a request can return. Requests that exceed this limit will be rejected. (0 disables limit)")
	apiCfg.StringVar(&logMinDurStr, "log-min-dur", "5min", "only log incoming requests if their timerange is at least this duration. Use 0 to disable")

	apiCfg.StringVar(&Addr, "listen", ":6060", "http listener address.")
	apiCfg.BoolVar(&UseSSL, "ssl", false, "use HTTPS")
	apiCfg.BoolVar(&useGzip, "gzip", true, "use GZIP compression of all responses")
	apiCfg.StringVar(&certFile, "cert-file", "", "SSL certificate file")
	apiCfg.StringVar(&keyFile, "key-file", "", "SSL key file")
	apiCfg.BoolVar(&multiTenant, "multi-tenant", true, "require x-org-id authentication to auth as a specific org. otherwise orgId 1 is assumed")
	apiCfg.StringVar(&fallbackGraphite, "fallback-graphite-addr", "http://localhost:8080", "in case our /render endpoint does not support the requested processing, proxy the request to this graphite")
	globalconf.Register("http", apiCfg)
}

func ConfigProcess() {
	logMinDur = dur.MustParseUsec("log-min-dur", logMinDurStr)

	//validate the addr
	_, err := net.ResolveTCPAddr("tcp", Addr)
	if err != nil {
		log.Fatal(4, "API listen address is not a valid TCP address.")
	}

	u, err := url.Parse(fallbackGraphite)
	if err != nil {
		log.Fatal(4, "API Cannot parse fallback-graphite-addr: %s", err)
	}
	graphiteProxy = httputil.NewSingleHostReverseProxy(u)
	// remove these headers from upstream
	// we will set our own correct ones (and duplicate headers are illegal)
	graphiteProxy.ModifyResponse = func(resp *http.Response) error {
		resp.Header.Del("access-control-allow-credentials")
		resp.Header.Del("Access-Control-Allow-Origin")
		return nil
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package api

import (
	"net/http"

	"code.google.com/p/go-uuid/uuid"
	"github.com/pagodabox/nanobox-server/config"
	"github.com/pagodabox/nanobox-server/data"
)

// CreateEVar
func (api *API) CreateDeploy(rw http.ResponseWriter, req *http.Request) {
	config.Log.Debug("[NANOBOX :: API] Deploy create\n")

	sync := data.Sync{
		Id:    uuid.New(),
		Reset: req.FormValue("reset") == "true",
	}
	api.Worker.QueueAndProcess(&sync)

	rw.Write([]byte("{\"id\":\"" + sync.Id + "\", \"status\":\"created\"}"))
}
<commit_msg>new uuid<commit_after>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package api

import (
	"net/http"

	"github.com/pagodabox/nanobox-server/config"
	"github.com/pagodabox/nanobox-server/data"
)

// CreateEVar
func (api *API) CreateDeploy(rw http.ResponseWriter, req *http.Request) {
	config.Log.Debug("[NANOBOX :: API] Deploy create\n")

	sync := data.Sync{
		Id:    newUUID(),
		Reset: req.FormValue("reset") == "true",
	}
	api.Worker.QueueAndProcess(&sync)

	rw.Write([]byte("{\"id\":\"" + sync.Id + "\", \"status\":\"created\"}"))
}
<|endoftext|>


<commit_before>package pruner_test

import (
	"errors"
	"time"

	"autoscaler/metricscollector/fakes"
	. "autoscaler/pruner"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("MetricdsDB Prune", func() {
	var (
		metricsDB  *fakes.FakeMetricsDB
		pruner     *MetricsDBPruner
		fclock     *fakeclock.FakeClock
		cutoffDays int
		buffer     *gbytes.Buffer
	)

	BeforeEach(func() {

		cutoffDays = 20
		logger := lagertest.NewTestLogger("prune-test")
		buffer = logger.Buffer()

		metricsDB = &fakes.FakeMetricsDB{}
		fclock = fakeclock.NewFakeClock(time.Now())

		pruner = NewMetricsDBPruner(logger, metricsDB, TestRefreshInterval, cutoffDays, fclock)
	})

	Describe("Start", func() {
		JustBeforeEach(func() {
			pruner.Start()
		})

		AfterEach(func() {
			pruner.Stop()
		})

		Context("when pruning metrics records from metrics db", func() {
			It("prunes at given interval and cutoff days", func() {
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(1))
				Expect(metricsDB.PruneMetricsArgsForCall(0)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))
				Expect(metricsDB.PruneMetricsArgsForCall(1)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(3))
				Expect(metricsDB.PruneMetricsArgsForCall(2)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))
			})
		})

		Context("when pruning records from metrics db fails", func() {
			BeforeEach(func() {
				metricsDB.PruneMetricsReturns(errors.New("test pruner error"))
			})

			It("should error", func() {
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(1))
				Eventually(buffer).Should(gbytes.Say("test pruner error"))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))
				Eventually(buffer).Should(gbytes.Say("test pruner error"))
			})
		})
	})

	Describe("Stop", func() {
		JustBeforeEach(func() {
			pruner.Start()
		})

		It("Stop the pruner", func() {
			fclock.Increment(TestRefreshInterval)
			Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))

			pruner.Stop()
			Eventually(buffer).Should(gbytes.Say("metrics-db-pruner-stopped"))

			fclock.Increment(TestRefreshInterval)
			Consistently(metricsDB.PruneMetricsCallCount).Should(Equal(2))
		})
	})
})
<commit_msg>fix test failure in MetricsDBPruner.Stop() (#103)<commit_after>package pruner_test

import (
	"errors"
	"time"

	"autoscaler/metricscollector/fakes"
	. "autoscaler/pruner"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("MetricdsDB Prune", func() {
	var (
		metricsDB  *fakes.FakeMetricsDB
		pruner     *MetricsDBPruner
		fclock     *fakeclock.FakeClock
		cutoffDays int
		buffer     *gbytes.Buffer
	)

	BeforeEach(func() {

		cutoffDays = 20
		logger := lagertest.NewTestLogger("prune-test")
		buffer = logger.Buffer()

		metricsDB = &fakes.FakeMetricsDB{}
		fclock = fakeclock.NewFakeClock(time.Now())

		pruner = NewMetricsDBPruner(logger, metricsDB, TestRefreshInterval, cutoffDays, fclock)
	})

	Describe("Start", func() {
		JustBeforeEach(func() {
			pruner.Start()
		})

		AfterEach(func() {
			pruner.Stop()
		})

		Context("when pruning metrics records from metrics db", func() {
			It("prunes at given interval and cutoff days", func() {
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(1))
				Expect(metricsDB.PruneMetricsArgsForCall(0)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))
				Expect(metricsDB.PruneMetricsArgsForCall(1)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(3))
				Expect(metricsDB.PruneMetricsArgsForCall(2)).To(BeNumerically("==", fclock.Now().AddDate(0, 0, -cutoffDays).UnixNano()))
			})
		})

		Context("when pruning records from metrics db fails", func() {
			BeforeEach(func() {
				metricsDB.PruneMetricsReturns(errors.New("test pruner error"))
			})

			It("should error", func() {
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(1))
				Eventually(buffer).Should(gbytes.Say("test pruner error"))

				fclock.Increment(TestRefreshInterval)
				Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))
				Eventually(buffer).Should(gbytes.Say("test pruner error"))
			})
		})
	})

	Describe("Stop", func() {
		JustBeforeEach(func() {
			pruner.Start()
			Eventually(fclock.WatcherCount).Should(Equal(1))
		})

		It("Stop the pruner", func() {
			fclock.Increment(TestRefreshInterval)
			Eventually(metricsDB.PruneMetricsCallCount).Should(Equal(2))

			pruner.Stop()
			Eventually(buffer).Should(gbytes.Say("metrics-db-pruner-stopped"))

			fclock.Increment(TestRefreshInterval)
			Consistently(metricsDB.PruneMetricsCallCount).Should(Equal(2))
		})
	})
})
<|endoftext|>