<commit_before>package front

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kendellfab/milo"
	"github.com/kendellfab/publish/domain"
	"log"
	"net/http"
	"time"
)

type FrontPosts struct {
	*FrontBase
}

func NewFrontPosts(base *FrontBase) FrontPosts {
	posts := FrontPosts{FrontBase: base}
	return posts
}

func (f FrontPosts) RegisterRoutes(app *milo.Milo) {
	app.Route("/{year:[0-9]+}", []string{"Get"}, f.handleYear)
	app.Route("/{year:[0-9]+}/{month:[0-9]+}", []string{"Get"}, f.handleMonth)
	app.Route("/{year:[0-9]+}/{month:[0-9]+}/{slug}", []string{"Get"}, f.handlePost)
}

func (f FrontPosts) handleYear(w http.ResponseWriter, r *http.Request) {
	year := mux.Vars(r)["year"]
	now := time.Now()

	series := make([]*domain.TimeSeries, 0)
	for i := int(now.Month()); i > 0; i-- {
		when, _ := time.Parse("2006-1", fmt.Sprintf("%s-%d", year, i))
		if posts, pErr := f.rm.PostRepo.FindByYearMonth(year, fmt.Sprintf("%d", i)); pErr == nil {
			series = append(series, &domain.TimeSeries{When: when, Posts: posts})
		}
	}
	data := make(map[string]interface{})
	data["series"] = series

	f.RenderTemplates(w, r, data, "series_time.html")
}

func (f FrontPosts) handleMonth(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	year := vars["year"]
	month := vars["month"]
	when, _ := time.Parse("2006-01", fmt.Sprintf("%s-%s", year, month))

	posts, pErr := f.rm.PostRepo.FindByYearMonth(year, month)
	series := make([]*domain.TimeSeries, 0)
	series = append(series, &domain.TimeSeries{When: when, Posts: posts})

	data := make(map[string]interface{})
	data["series"] = series
	data["error"] = pErr

	f.RenderTemplates(w, r, data, "series_time.html")
}

func (f FrontPosts) handlePost(w http.ResponseWriter, r *http.Request) {
	slug := mux.Vars(r)["slug"]

	// view := &domain.View{Who: f.getIp(r), At: time.Now(), TargetType: domain.TypePost, Target: slug}
	// vErr := f.rm.ViewRepo.Store(view)
	// if vErr != nil {
	// 	log.Println(vErr)
	// }

	post, err := f.rm.PostRepo.FindBySlug(slug)
	data := make(map[string]interface{})
	if err == nil {
		data["Post"] = post
	} else {
		log.Println("Post Error:", err)
		f.RenderTemplates(w, r, nil, "404.html")
		return
	}

	if series, sErr := f.rm.SeriesRepo.GetSeries(fmt.Sprintf("%d", post.SeriesId)); sErr == nil {
		data["series"] = series
	}

	f.RenderTemplates(w, r, data, "post.html")
}
<commit_msg>Work on showing the proper posts for the selected year.<commit_after>package front

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/kendellfab/milo"
	"github.com/kendellfab/publish/domain"
	"log"
	"net/http"
	"strconv"
	"time"
)

type FrontPosts struct {
	*FrontBase
}

func NewFrontPosts(base *FrontBase) FrontPosts {
	posts := FrontPosts{FrontBase: base}
	return posts
}

func (f FrontPosts) RegisterRoutes(app *milo.Milo) {
	app.Route("/{year:[0-9]+}", []string{"Get"}, f.handleYear)
	app.Route("/{year:[0-9]+}/{month:[0-9]+}", []string{"Get"}, f.handleMonth)
	app.Route("/{year:[0-9]+}/{month:[0-9]+}/{slug}", []string{"Get"}, f.handlePost)
}

func (f FrontPosts) handleYear(w http.ResponseWriter, r *http.Request) {
	year := mux.Vars(r)["year"]
	now := time.Now()

	start := int(now.Month())
	if yr, yrErr := strconv.Atoi(year); yrErr == nil {
		if yr != now.Year() {
			start = 12
		}
	}

	series := make([]*domain.TimeSeries, 0)
	for i := start; i > 0; i-- {
		when, _ := time.Parse("2006-1", fmt.Sprintf("%s-%d", year, i))
		if posts, pErr := f.rm.PostRepo.FindByYearMonth(year, fmt.Sprintf("%d", i)); pErr == nil {
			series = append(series, &domain.TimeSeries{When: when, Posts: posts})
		}
	}
	data := make(map[string]interface{})
	data["series"] = series

	f.RenderTemplates(w, r, data, "series_time.html")
}

func (f FrontPosts) handleMonth(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	year := vars["year"]
	month := vars["month"]
	when, _ := time.Parse("2006-1", fmt.Sprintf("%s-%s", year, month))

	posts, pErr := f.rm.PostRepo.FindByYearMonth(year, month)
	series := make([]*domain.TimeSeries, 0)
	series = append(series, &domain.TimeSeries{When: when, Posts: posts})

	data := make(map[string]interface{})
	data["series"] = series
	data["error"] = pErr

	f.RenderTemplates(w, r, data, "series_time.html")
}

func (f FrontPosts) handlePost(w http.ResponseWriter, r *http.Request) {
	slug := mux.Vars(r)["slug"]

	// view := &domain.View{Who: f.getIp(r), At: time.Now(), TargetType: domain.TypePost, Target: slug}
	// vErr := f.rm.ViewRepo.Store(view)
	// if vErr != nil {
	// 	log.Println(vErr)
	// }

	post, err := f.rm.PostRepo.FindBySlug(slug)
	data := make(map[string]interface{})
	if err == nil {
		data["Post"] = post
	} else {
		log.Println("Post Error:", err)
		f.RenderTemplates(w, r, nil, "404.html")
		return
	}

	if series, sErr := f.rm.SeriesRepo.GetSeries(fmt.Sprintf("%d", post.SeriesId)); sErr == nil {
		data["series"] = series
	}

	f.RenderTemplates(w, r, data, "post.html")
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"fmt"
	"net"
)

type THandle interface {
	Handle(conn *net.TCPConn)
}

// Wrap a panic, only print it, but ignore it.
//
// handle is a function, whose type is `func (*net.TCPConn)`, or a struct, which
// has implemented the interface `THandle`.
func WrapError(conn *net.TCPConn, handle interface{}) {
	yes := true
	defer func() {
		if err := recover(); err != nil {
			fmt.Printf("Get a error: %v\n", err)
			if !yes {
				panic(err)
			}
		}
	}()
	defer conn.Close()

	if handler, ok := handle.(THandle); ok {
		handler.Handle(conn)
	} else if handler, ok := handle.(func(*net.TCPConn)); ok {
		handler(conn)
	} else {
		yes = false
		panic("Don't support the handler")
	}
}

// Start a TCP server and never return. Return a error when returns.
//
// network MUST be "tcp", "tcp4", "tcp6".
// addr is like "host:port", such as "127.0.0.1:8000", and host or port
// may be omitted.
func TCPServerForever(network, addr string, handle interface{}) (e error) {
	ln, err := net.Listen(network, addr)
	if err != nil {
		return err
	}

	tcpln, ok := ln.(*net.TCPListener)
	if !ok {
		return errors.New("Must be a TCP Listener")
	}

	fmt.Printf("Listen on %v\n", addr)
	for {
		conn, err := tcpln.AcceptTCP()
		if err != nil {
			e = err
			fmt.Printf("Failed to AcceptTCP: %v\n", err)
		} else {
			e = nil
			go WrapError(conn, handle)
		}
	}
	return nil
}
<commit_msg>Replace fmt with log<commit_after>package server

import (
	"errors"
	"log"
	"net"
	"os"
)

var (
	_logger *log.Logger
)

func init() {
	_logger = log.New(os.Stderr, "[TCP Server] ", log.LstdFlags)
}

func SetLogger(logger *log.Logger) {
	_logger = logger
}

func GetLogger() *log.Logger {
	return _logger
}

type THandle interface {
	Handle(conn *net.TCPConn)
}

// Wrap a panic, only print it, but ignore it.
//
// handle is a function, whose type is `func (*net.TCPConn)`, or a struct, which
// has implemented the interface `THandle`.
func WrapError(conn *net.TCPConn, handle interface{}) {
	yes := true
	defer func() {
		if err := recover(); err != nil {
			_logger.Printf("[Error] Get a error: %v", err)
			if !yes {
				panic(err)
			}
		}
	}()
	defer conn.Close()

	if handler, ok := handle.(THandle); ok {
		handler.Handle(conn)
	} else if handler, ok := handle.(func(*net.TCPConn)); ok {
		handler(conn)
	} else {
		yes = false
		panic("Don't support the handler")
	}
}

// Start a TCP server and never return. Return a error when returns.
//
// network MUST be "tcp", "tcp4", "tcp6".
// addr is like "host:port", such as "127.0.0.1:8000", and host or port
// may be omitted.
func TCPServerForever(network, addr string, handle interface{}) (e error) {
	ln, err := net.Listen(network, addr)
	if err != nil {
		return err
	}

	tcpln, ok := ln.(*net.TCPListener)
	if !ok {
		return errors.New("Must be a TCP Listener")
	}

	_logger.Printf("[Debug] Listen on %v", addr)
	for {
		conn, err := tcpln.AcceptTCP()
		if err != nil {
			e = err
			_logger.Printf("[Error] Failed to AcceptTCP: %v", err)
		} else {
			e = nil
			go WrapError(conn, handle)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package socketio

import (
	"log"
	"strings"
)

func Subscribe(ch chan<- string, url, channel string) {
	socket, err := NewSocket(url, channel)
	if err != nil {
		log.Fatal(err)
	}

	// Receive loop
	var rawJsonMsg string
	for {
		// Receive the message through websocket
		if err := socket.Receive(&rawJsonMsg); err != nil {
			log.Fatal(err)
		}

		// Remove the socketio message headers
		rawJsonMsg = strings.TrimLeftFunc(rawJsonMsg, func(r rune) bool {
			if r == '{' {
				return false
			}
			return true
		})

		// ignore emtpy data and handshakes
		if len(rawJsonMsg) > 2 {
			ch <- rawJsonMsg
		}
	}
}
<commit_msg>Remove Subscribe()<commit_after><|endoftext|>


<commit_before>package prettyprint

import (
	"bytes"
	"fmt"
	"strings"
	"unicode"
)

var templateFuncMap = map[string]interface{}{
	"capitalize": func(str string) string {
		if len(str) == 0 {
			return str
		}

		runes := []rune(str)
		runes[0] = unicode.ToUpper(runes[0])
		return string(runes)
	},
	"gibtib": func(size int) string {
		// lt is a less than sign if < 1GiB
		lt := ""
		if size < 1024 {
			lt = "< "
		}
		size /= 1024
		gt := 'G'
		if size >= 1024 {
			size /= 1024
			gt = 'T'
		}
		return fmt.Sprintf("%s%d%ciB", lt, size, gt)
	},
	"mibgib": func(size int) string {
		mg := 'M'
		if size >= 1024 {
			size /= 1024
			mg = 'G'
		}
		return fmt.Sprintf("%d%ciB", size, mg)
	},
	"percentage": func(num int, denom int) string {
		return fmt.Sprintf("%d%%", int(100*float64(num)/float64(denom)))
	},
	"pluralize": func(single string, plural string, num int) string {
		if num == 1 {
			return fmt.Sprintf("%d %s", num, single)
		}
		return fmt.Sprintf("%d %s", num, plural)
	},
	"prettysprint": func(pp PrettyPrinter, detail DetailLevel) string {
		b := new(bytes.Buffer)
		pp.PrettyPrint(b, detail)
		return b.String()
	},
	"join": strings.Join,
}

// Funcs is not for general usage, only while in transition to PrettyPrint.
var Funcs = templateFuncMap
<commit_msg>prettysprint now checks for errors<commit_after>package prettyprint

import (
	"bytes"
	"fmt"
	"strings"
	"unicode"
)

var templateFuncMap = map[string]interface{}{
	"capitalize": func(str string) string {
		if len(str) == 0 {
			return str
		}

		runes := []rune(str)
		runes[0] = unicode.ToUpper(runes[0])
		return string(runes)
	},
	"gibtib": func(size int) string {
		// lt is a less than sign if < 1GiB
		lt := ""
		if size < 1024 {
			lt = "< "
		}
		size /= 1024
		gt := 'G'
		if size >= 1024 {
			size /= 1024
			gt = 'T'
		}
		return fmt.Sprintf("%s%d%ciB", lt, size, gt)
	},
	"mibgib": func(size int) string {
		mg := 'M'
		if size >= 1024 {
			size /= 1024
			mg = 'G'
		}
		return fmt.Sprintf("%d%ciB", size, mg)
	},
	"percentage": func(num int, denom int) string {
		return fmt.Sprintf("%d%%", int(100*float64(num)/float64(denom)))
	},
	"pluralize": func(single string, plural string, num int) string {
		if num == 1 {
			return fmt.Sprintf("%d %s", num, single)
		}
		return fmt.Sprintf("%d %s", num, plural)
	},
	"prettysprint": func(pp PrettyPrinter, detail DetailLevel) (string, error) {
		b := new(bytes.Buffer)
		err := pp.PrettyPrint(b, detail)
		if err != nil {
			return "", err
		}
		return b.String(), nil
	},
	"join": strings.Join,
}

// Funcs is not for general usage, only while in transition to PrettyPrint.
var Funcs = templateFuncMap
<|endoftext|>


<commit_before>// run

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that locks don't go quadratic due to runtime hash table collisions.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"
)

const debug = false

// checkLinear asserts that the running time of f(n) is at least linear but sub-quadratic.
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	n := tries
	fails := 0
	var buf bytes.Buffer
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)
		if debug {
			println(n, t1.String(), 2*n, t2.String())
		}
		fmt.Fprintf(&buf, "%d %v %d %v\n", n, t1, 2*n, t2)
		// should be 2x (linear); allow up to 2.5x
		if t1*3/2 < t2 && t2 < t1*5/2 {
			return
		}
		// If 2n ops run in under a second and the ratio
		// doesn't work out, make n bigger, trying to reduce
		// the effect that a constant amount of overhead has
		// on the computed ratio.
		if t2 < 1*time.Second {
			n *= 2
			continue
		}
		// Once the test runs long enough for n ops,
		// try to get the right ratio at least once.
		// If many in a row all fail, give up.
		if fails++; fails >= 10 {
			panic(fmt.Sprintf("%s: too slow: %d ops: %v; %d ops: %v\n\n%s",
				typ, n, t1, 2*n, t2, buf.String()))
		}
	}
}

const offset = 251 // known size of runtime hash table

const profile = false

func main() {
	if profile {
		f, err := os.Create("lock.prof")
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	checkLinear("lockone", 1000, func(n int) {
		ch := make(chan int)
		locks := make([]sync.RWMutex, offset+1)
		for i := 0; i < n; i++ {
			go func() {
				locks[0].Lock()
				ch <- 1
			}()
		}
		time.Sleep(1 * time.Millisecond)

		go func() {
			for j := 0; j < n; j++ {
				locks[1].Lock()
				locks[offset].Lock()
				locks[1].Unlock()
				runtime.Gosched()
				locks[offset].Unlock()
			}
		}()

		for j := 0; j < n; j++ {
			locks[1].Lock()
			locks[offset].Lock()
			locks[1].Unlock()
			runtime.Gosched()
			locks[offset].Unlock()
		}

		for i := 0; i < n; i++ {
			<-ch
			locks[0].Unlock()
		}
	})

	checkLinear("lockmany", 1000, func(n int) {
		locks := make([]sync.RWMutex, n*offset+1)

		var wg sync.WaitGroup
		for i := 0; i < n; i++ {
			wg.Add(1)
			go func(i int) {
				locks[(i+1)*offset].Lock()
				wg.Done()
				locks[(i+1)*offset].Lock()
				locks[(i+1)*offset].Unlock()
			}(i)
		}
		wg.Wait()

		go func() {
			for j := 0; j < n; j++ {
				locks[1].Lock()
				locks[0].Lock()
				locks[1].Unlock()
				runtime.Gosched()
				locks[0].Unlock()
			}
		}()

		for j := 0; j < n; j++ {
			locks[1].Lock()
			locks[0].Lock()
			locks[1].Unlock()
			runtime.Gosched()
			locks[0].Unlock()
		}

		for i := 0; i < n; i++ {
			locks[(i+1)*offset].Unlock()
		}
	})
}
<commit_msg>test/locklinear: deflake again<commit_after>// run

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that locks don't go quadratic due to runtime hash table collisions.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"
)

const debug = false

// checkLinear asserts that the running time of f(n) is at least linear but sub-quadratic.
// tries is the initial number of iterations.
func checkLinear(typ string, tries int, f func(n int)) {
	// Depending on the machine and OS, this test might be too fast
	// to measure with accurate enough granularity. On failure,
	// make it run longer, hoping that the timing granularity
	// is eventually sufficient.

	timeF := func(n int) time.Duration {
		t1 := time.Now()
		f(n)
		return time.Since(t1)
	}

	n := tries
	fails := 0
	var buf bytes.Buffer
	for {
		t1 := timeF(n)
		t2 := timeF(2 * n)
		if debug {
			println(n, t1.String(), 2*n, t2.String())
		}
		fmt.Fprintf(&buf, "%d %v %d %v\n", n, t1, 2*n, t2)
		// should be 2x (linear); allow up to 2.5x
		if t1*3/2 < t2 && t2 < t1*5/2 {
			return
		}
		// Once the test runs long enough for n ops,
		// try to get the right ratio at least once.
		// If many in a row all fail, give up.
		if fails++; fails >= 5 {
			// If 2n ops run in under a second and the ratio
			// doesn't work out, make n bigger, trying to reduce
			// the effect that a constant amount of overhead has
			// on the computed ratio.
			if t2 < time.Second*4/10 {
				fails = 0
				n *= 2
				continue
			}
			panic(fmt.Sprintf("%s: too slow: %d ops: %v; %d ops: %v\n\n%s",
				typ, n, t1, 2*n, t2, buf.String()))
		}
	}
}

const offset = 251 // known size of runtime hash table

const profile = false

func main() {
	if profile {
		f, err := os.Create("lock.prof")
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	checkLinear("lockone", 1000, func(n int) {
		ch := make(chan int)
		locks := make([]sync.RWMutex, offset+1)
		for i := 0; i < n; i++ {
			go func() {
				locks[0].Lock()
				ch <- 1
			}()
		}
		time.Sleep(1 * time.Millisecond)

		go func() {
			for j := 0; j < n; j++ {
				locks[1].Lock()
				locks[offset].Lock()
				locks[1].Unlock()
				runtime.Gosched()
				locks[offset].Unlock()
			}
		}()

		for j := 0; j < n; j++ {
			locks[1].Lock()
			locks[offset].Lock()
			locks[1].Unlock()
			runtime.Gosched()
			locks[offset].Unlock()
		}

		for i := 0; i < n; i++ {
			<-ch
			locks[0].Unlock()
		}
	})

	checkLinear("lockmany", 1000, func(n int) {
		locks := make([]sync.RWMutex, n*offset+1)

		var wg sync.WaitGroup
		for i := 0; i < n; i++ {
			wg.Add(1)
			go func(i int) {
				locks[(i+1)*offset].Lock()
				wg.Done()
				locks[(i+1)*offset].Lock()
				locks[(i+1)*offset].Unlock()
			}(i)
		}
		wg.Wait()

		go func() {
			for j := 0; j < n; j++ {
				locks[1].Lock()
				locks[0].Lock()
				locks[1].Unlock()
				runtime.Gosched()
				locks[0].Unlock()
			}
		}()

		for j := 0; j < n; j++ {
			locks[1].Lock()
			locks[0].Lock()
			locks[1].Unlock()
			runtime.Gosched()
			locks[0].Unlock()
		}

		for i := 0; i < n; i++ {
			locks[(i+1)*offset].Unlock()
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/absolute8511/glog"
	"github.com/absolute8511/redigo/redis"
	"github.com/youzan/ZanRedisDB/common"
	"github.com/youzan/ZanRedisDB/rockredis"
)

var ipProxy = flag.String("ip_proxy", "127.0.0.1", "server ip")
var portProxy = flag.Int("port_proxy", 3803, "server port")
var toolType = flag.String("type", "", "data tool type: gen_delrange/compactdb/scan_localttl")
var ns = flag.String("ns", "", "namespace for data")
var table = flag.String("table", "", "table for data")
var key = flag.String("key", "", "key for action")
var cleanMatched = flag.Bool("clean_matched", false, "do del to proxy if matched")
var maxScan = flag.Int("max_scan", 10000, "max scan db keys")
var sleepBetween = flag.Duration("sleep_between", time.Millisecond, "")
var startFrom = flag.String("start_from", "", "key range for data")
var endTo = flag.String("end_to", "", "key range for data")
var dbFile = flag.String("dbfile", "", "file path for rocksdb parent, the final will be dbFile/rocksdb")

type DeleteTableRange struct {
	Table     string `json:"table,omitempty"`
	StartFrom []byte `json:"start_from,omitempty"`
	EndTo     []byte `json:"end_to,omitempty"`
	// to avoid drop all table data, this is needed to delete all data in table
	DeleteAll bool `json:"delete_all,omitempty"`
	Dryrun    bool `json:"dryrun,omitempty"`
}

func getRangeStr(dr DeleteTableRange) string {
	b, _ := json.Marshal(dr)
	return string(b)
}

func main() {
	flag.Parse()
	common.InitDefaultForGLogger("./")
	defer glog.Flush()

	switch *toolType {
	case "gen_delrange":
		log.Printf("test gen del range")
		dr := DeleteTableRange{
			StartFrom: []byte(*startFrom),
			EndTo:     []byte(*endTo),
		}
		log.Printf("%s", getRangeStr(dr))
	case "compact_db":
		compactDB()
	case "scan_localttl":
		checkLocalTTL()
	default:
		log.Printf("unknown action: %v", *toolType)
	}
}

var (
	errExpTimeKey = errors.New("invalid expire time key")
)

const (
	logTimeFormatStr = "2006-01-02 15:04:05"
)

/*
the coded format of expire time key:
bytes:  -0-|-1-2-3-4-5-6-7-8-|----9---|-10-11--------x-|
data :  103|       when      |dataType|       key      |
*/
func expEncodeTimeKey(dataType byte, key []byte, when int64) []byte {
	buf := make([]byte, len(key)+1+8+1)

	pos := 0
	buf[pos] = rockredis.ExpTimeType
	pos++

	binary.BigEndian.PutUint64(buf[pos:], uint64(when))
	pos += 8

	buf[pos] = dataType
	pos++

	copy(buf[pos:], key)

	return buf
}

//decode the expire 'time key', the return values are: dataType, key, whenToExpire, error
func expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) {
	pos := 0
	if pos+10 > len(tk) || tk[pos] != rockredis.ExpTimeType {
		return 0, nil, 0, errExpTimeKey
	}

	return tk[pos+9], tk[pos+10:], int64(binary.BigEndian.Uint64(tk[pos+1:])), nil
}

func checkLocalTTL() {
	log.Printf("begin check ttl")
	now := time.Now().Unix()

	cfg := rockredis.NewRockRedisDBConfig()
	cfg.DataDir = *dbFile
	cfg.ReadOnly = true
	cfg.DataTool = true
	db, err := rockredis.OpenRockDB(cfg)
	if err != nil {
		log.Printf("open db failed: %s", err.Error())
		return
	}
	defer db.Close()
	minKey := expEncodeTimeKey(rockredis.NoneType, nil, 0)
	maxKey := expEncodeTimeKey(100, nil, now+600)

	var eCount int64
	var scanned int64
	checkStart := time.Now()

	it, err := db.NewDBRangeLimitIterator(minKey, maxKey,
		common.RangeROpen, 0, -1, false)
	if err != nil {
		log.Printf("open db iterator failed: %s", err.Error())
		return
	}
	defer it.Close()

	clientConn, err := redis.Dial("tcp", net.JoinHostPort(*ipProxy, strconv.Itoa(*portProxy)), redis.DialConnectTimeout(time.Second))
	if *cleanMatched {
		if err != nil {
			log.Printf("connect redis failed: %s", err.Error())
			return
		}
	}

	for ; it.Valid(); it.Next() {
		if scanned > int64(*maxScan) {
			break
		}
		tk := it.Key()
		if tk == nil {
			continue
		}

		dt, k, nt, dErr := expDecodeTimeKey(tk)
		if dErr != nil {
			continue
		}

		scanned += 1
		if nt > now+600 {
			//the next ttl check time is nt!
			log.Printf("ttl check end at key:[%s] of type:%s whose expire time is: %s", string(k),
				rockredis.TypeName[dt], time.Unix(nt, 0).Format(logTimeFormatStr))
			break
		}

		eCount += 1

		if *key != "" {
			if strings.HasPrefix(string(k), *key) {
				log.Printf("found key %s(type: %v), expire time: %s\n", string(k), rockredis.TypeName[dt],
					time.Unix(nt, 0).Format(logTimeFormatStr))
				if *cleanMatched {
					delKey := fmt.Sprintf("%s:%s", *ns, string(k))
					n, err := redis.Int(clientConn.Do("del", delKey))
					if err != nil {
						log.Printf("del %s failed: %s", delKey, err.Error())
					} else if n == 0 {
						log.Printf("del %s not exist: %v", delKey, n)
					}
					if *sleepBetween > 0 {
						time.Sleep(*sleepBetween)
					}
				}
			}
			continue
		}
		log.Printf("scanned ttl: key %s(type: %v), expire time: %s\n", string(k), rockredis.TypeName[dt],
			time.Unix(nt, 0).Format(logTimeFormatStr))
	}

	checkCost := time.Since(checkStart)
	log.Printf("[%d/%d] keys have expired during ttl checking, cost:%s", eCount, scanned, checkCost)
}

func compactDB() {
	cfg := rockredis.NewRockRedisDBConfig()
	cfg.DataDir = *dbFile
	db, err := rockredis.OpenRockDB(cfg)
	if err != nil {
		log.Printf("open db failed: %s", err.Error())
		return
	}
	defer db.Close()
	db.CompactAllRange()
}
<commit_msg>adjust check ttl time<commit_after>package main

import (
	"encoding/binary"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/absolute8511/glog"
	"github.com/absolute8511/redigo/redis"
	"github.com/youzan/ZanRedisDB/common"
	"github.com/youzan/ZanRedisDB/rockredis"
)

var ipProxy = flag.String("ip_proxy", "127.0.0.1", "server ip")
var portProxy = flag.Int("port_proxy", 3803, "server port")
var toolType = flag.String("type", "", "data tool type: gen_delrange/compactdb/scan_localttl")
var ns = flag.String("ns", "", "namespace for data")
var table = flag.String("table", "", "table for data")
var key = flag.String("key", "", "key for action")
var cleanMatched = flag.Bool("clean_matched", false, "do del to proxy if matched")
var maxScan = flag.Int("max_scan", 10000, "max scan db keys")
var sleepBetween = flag.Duration("sleep_between", time.Millisecond, "")
var startFrom = flag.String("start_from", "", "key range for data")
var endTo = flag.String("end_to", "", "key range for data")
var dbFile = flag.String("dbfile", "", "file path for rocksdb parent, the final will be dbFile/rocksdb")

type DeleteTableRange struct {
	Table     string `json:"table,omitempty"`
	StartFrom []byte `json:"start_from,omitempty"`
	EndTo     []byte `json:"end_to,omitempty"`
	// to avoid drop all table data, this is needed to delete all data in table
	DeleteAll bool `json:"delete_all,omitempty"`
	Dryrun    bool `json:"dryrun,omitempty"`
}

func getRangeStr(dr DeleteTableRange) string {
	b, _ := json.Marshal(dr)
	return string(b)
}

func main() {
	flag.Parse()
	common.InitDefaultForGLogger("./")
	defer glog.Flush()

	switch *toolType {
	case "gen_delrange":
		log.Printf("test gen del range")
		dr := DeleteTableRange{
			StartFrom: []byte(*startFrom),
			EndTo:     []byte(*endTo),
		}
		log.Printf("%s", getRangeStr(dr))
	case "compact_db":
		compactDB()
	case "scan_localttl":
		checkLocalTTL()
	default:
		log.Printf("unknown action: %v", *toolType)
	}
}

var (
	errExpTimeKey = errors.New("invalid expire time key")
)

const (
	logTimeFormatStr = "2006-01-02 15:04:05"
)

/*
the coded format of expire time key:
bytes:  -0-|-1-2-3-4-5-6-7-8-|----9---|-10-11--------x-|
data :  103|       when      |dataType|       key      |
*/
func expEncodeTimeKey(dataType byte, key []byte, when int64) []byte {
	buf := make([]byte, len(key)+1+8+1)

	pos := 0
	buf[pos] = rockredis.ExpTimeType
	pos++

	binary.BigEndian.PutUint64(buf[pos:], uint64(when))
	pos += 8

	buf[pos] = dataType
	pos++

	copy(buf[pos:], key)

	return buf
}

//decode the expire 'time key', the return values are: dataType, key, whenToExpire, error
func expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) {
	pos := 0
	if pos+10 > len(tk) || tk[pos] != rockredis.ExpTimeType {
		return 0, nil, 0, errExpTimeKey
	}

	return tk[pos+9], tk[pos+10:], int64(binary.BigEndian.Uint64(tk[pos+1:])), nil
}

func checkLocalTTL() {
	log.Printf("begin check ttl")
	now := time.Now().Unix() - 3600

	cfg := rockredis.NewRockRedisDBConfig()
	cfg.DataDir = *dbFile
	cfg.ReadOnly = true
	cfg.DataTool = true
	db, err := rockredis.OpenRockDB(cfg)
	if err != nil {
		log.Printf("open db failed: %s", err.Error())
		return
	}
	defer db.Close()
	minKey := expEncodeTimeKey(rockredis.NoneType, nil, 0)
	maxKey := expEncodeTimeKey(100, nil, now)

	var eCount int64
	var scanned int64
	checkStart := time.Now()

	it, err := db.NewDBRangeLimitIterator(minKey, maxKey,
		common.RangeROpen, 0, -1, false)
	if err != nil {
		log.Printf("open db iterator failed: %s", err.Error())
		return
	}
	defer it.Close()

	clientConn, err := redis.Dial("tcp", net.JoinHostPort(*ipProxy, strconv.Itoa(*portProxy)), redis.DialConnectTimeout(time.Second))
	if *cleanMatched {
		if err != nil {
			log.Printf("connect redis failed: %s", err.Error())
			return
		}
	}

	for ; it.Valid(); it.Next() {
		if scanned > int64(*maxScan) {
			break
		}
		tk := it.Key()
		if tk == nil {
			continue
		}

		dt, k, nt, dErr := expDecodeTimeKey(tk)
		if dErr != nil {
			continue
		}

		scanned += 1
		if nt > now {
			//the next ttl check time is nt!
			log.Printf("ttl check end at key:[%s] of type:%s whose expire time is: %s", string(k),
				rockredis.TypeName[dt], time.Unix(nt, 0).Format(logTimeFormatStr))
			break
		}

		eCount += 1

		if *key != "" {
			if strings.HasPrefix(string(k), *key) {
				log.Printf("found key %s(type: %v), expire time: %s\n", string(k), rockredis.TypeName[dt],
					time.Unix(nt, 0).Format(logTimeFormatStr))
				if *cleanMatched {
					delKey := fmt.Sprintf("%s:%s", *ns, string(k))
					n, err := redis.Int(clientConn.Do("del", delKey))
					if err != nil {
						log.Printf("del %s failed: %s", delKey, err.Error())
					} else if n == 0 {
						log.Printf("del %s not exist: %v", delKey, n)
					}
					if *sleepBetween > 0 {
						time.Sleep(*sleepBetween)
					}
				}
			}
			continue
		}
		log.Printf("scanned ttl: key %s(type: %v), expire time: %s\n", string(k), rockredis.TypeName[dt],
			time.Unix(nt, 0).Format(logTimeFormatStr))
	}

	checkCost := time.Since(checkStart)
	log.Printf("[%d/%d] keys have expired during ttl checking, cost:%s", eCount, scanned, checkCost)
}

func compactDB() {
	cfg := rockredis.NewRockRedisDBConfig()
	cfg.DataDir = *dbFile
	db, err := rockredis.OpenRockDB(cfg)
	if err != nil {
		log.Printf("open db failed: %s", err.Error())
		return
	}
	defer db.Close()
	db.CompactAllRange()
}
<|endoftext|>


<commit_before>package story_tester

import (
	"encoding/json"
	"encoding/hex"
	"github.com/tildedave/go-hpack-impl/hpack"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"testing"
	"fmt"
)

type Case struct {
	HeaderTableSize int `json:"header_table_size"`
	Seqno int
	Wire string
	Headers [](map[string]string)
}

type Story struct {
	Cases []Case
	Description string
	Draft int
}

const (
	Success = iota
	Failure
	Skipped
)

func testStoryFile(t *testing.T, filePath string) int {
	storyBytes, err := ioutil.ReadFile(filePath)
	assert.Nil(t, err)

	var story Story

	json.Unmarshal(storyBytes, &story)
	context := hpack.NewEncodingContext()
	context.HeaderTable.MaxSize = story.Cases[0].HeaderTableSize

	if story.Draft != 7 {
		fmt.Printf("Skipped %s (Draft %d)\n", filePath, story.Draft)
		return Skipped
	}

	for _, c := range story.Cases {
		wire, err := hex.DecodeString(c.Wire)
		if err != nil {
			return Failure
		}

		hs, err := context.Decode(string(wire))
		if err != nil {
			return Failure
		}

		expectedHeaders := make([]hpack.HeaderField, 0)
		for _, h := range c.Headers {
			// only one in here usually
			for k, v := range h {
				header := hpack.HeaderField{k, v}
				expectedHeaders = append(expectedHeaders, header)
			}
		}

		expectedSet := hpack.HeaderSet{expectedHeaders}

		sort.Sort(expectedSet)
		sort.Sort(hs)

		if !reflect.DeepEqual(expectedHeaders, hs.Headers) {
			return Failure
		}
	}

	return Success
}

func TestStory(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test in short mode.")
	}

	attempts := 0
	fails := 0
	crashes := 0
	skipped := 0
	success := 0

	filepath.Walk("../hpack-test-case", func (path string, info os.FileInfo, err error) error {
		defer func() {
			if r := recover(); r != nil {
				crashes += 1
				fails += 1
			}
		}()

		if strings.HasSuffix(path, ".json") {
			attempts += 1
			result := testStoryFile(t, path)
			if result == Failure {
				fmt.Println("Failed", path)
				fails += 1
			} else if result == Success {
				success += 1
			} else if result == Skipped {
				skipped += 1
			}
		}
		return nil
	})

	fmt.Printf("Attempted %d test cases\n", attempts)
	fmt.Printf("\tFailures: %d (%d crashed)\n", fails, crashes)
	fmt.Printf("\tSuccesses: %d\n", success)
	fmt.Printf("\tSkipped: %d\n", skipped)
}
<commit_msg>Print out which test files crashed<commit_after>package story_tester

import (
	"encoding/json"
	"encoding/hex"
	"github.com/tildedave/go-hpack-impl/hpack"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"testing"
	"fmt"
)

type Case struct {
	HeaderTableSize int `json:"header_table_size"`
	Seqno int
	Wire string
	Headers [](map[string]string)
}

type Story struct {
	Cases []Case
	Description string
	Draft int
}

const (
	Success = iota
	Failure
	Skipped
)

func testStoryFile(t *testing.T, filePath string) int {
	storyBytes, err := ioutil.ReadFile(filePath)
	assert.Nil(t, err)

	var story Story

	json.Unmarshal(storyBytes, &story)
	context := hpack.NewEncodingContext()
	context.HeaderTable.MaxSize = story.Cases[0].HeaderTableSize

	if story.Draft != 7 {
		fmt.Printf("Skipped %s (Draft %d)\n", filePath, story.Draft)
		return Skipped
	}

	for _, c := range story.Cases {
		wire, err := hex.DecodeString(c.Wire)
		if err != nil {
			return Failure
		}

		hs, err := context.Decode(string(wire))
		if err != nil {
			return Failure
		}

		expectedHeaders := make([]hpack.HeaderField, 0)
		for _, h := range c.Headers {
			// only one in here usually
			for k, v := range h {
				header := hpack.HeaderField{k, v}
				expectedHeaders = append(expectedHeaders, header)
			}
		}

		expectedSet := hpack.HeaderSet{expectedHeaders}

		sort.Sort(expectedSet)
		sort.Sort(hs)

		if !reflect.DeepEqual(expectedHeaders, hs.Headers) {
			return Failure
		}
	}

	return Success
}

func TestStory(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping test in short mode.")
	}

	attempts := 0
	fails := 0
	crashes := 0
	skipped := 0
	success := 0

	filepath.Walk("../hpack-test-case", func (path string, info os.FileInfo, err error) error {
		defer func() {
			if r := recover(); r != nil {
				fmt.Println("Failed", path, " (Crashed)")
				crashes += 1
				fails += 1
			}
		}()

		if strings.HasSuffix(path, ".json") {
			attempts += 1
			result := testStoryFile(t, path)
			if result == Failure {
				fmt.Println("Failed", path)
				fails += 1
			} else if result == Success {
				success += 1
			} else if result == Skipped {
				skipped += 1
			}
		}
		return nil
	})

	fmt.Printf("Attempted %d test cases\n", attempts)
	fmt.Printf("\tFailures: %d (%d crashed)\n", fails, crashes)
	fmt.Printf("\tSuccesses: %d\n", success)
	fmt.Printf("\tSkipped: %d\n", skipped)
}
<|endoftext|>


<commit_before>package logger

import (
	"io"

	"github.com/Clever/kayvee-go/v7/router"
)

/////////////////////////////
//
//	KayveeLogger interface
//
/////////////////////////////

// KayveeLogger is the main logging interface, providing customization of log messages.
type KayveeLogger interface {

	//
	// Configuration
	//

	// AddContext adds a new key-val to be logged with all log messages.
	AddContext(key, val string)

	// GetContext reads a key-val from the global map of data that will be logged with all log messages.
	GetContext(key string) (interface{}, bool)

	// SetConfig allows configuration changes in one go
	SetConfig(source string, logLvl LogLevel, formatter Formatter, output io.Writer)

	// SetFormatter sets the formatter function to use
	SetFormatter(formatter Formatter)

	// SetLogLevel sets the default log level threshold
	SetLogLevel(logLvl LogLevel)

	// SetOutput changes the output destination of the logger
	SetOutput(output io.Writer)

	// setFormatLogger use for to implemente the mock
	setFormatLogger(fl formatLogger)

	// SetRouter changes the router for this logger instance.  Once set, logs produced by this
	// logger will not be touched by the global router.  Mostly used for testing and benchmarking.
	SetRouter(router router.Router)

	//
	// Logging
	//

	// Counter takes a string and logs with LogLevel = Info
	Counter(title string)

	// CounterD takes a string, value, and data map. It logs with LogLevel = Info
	CounterD(title string, value int, data map[string]interface{})

	// Critical takes a string and logs with LogLevel = Critical
	Critical(title string)

	// CriticalD takes a string and data map. It logs with LogLevel = Critical
	CriticalD(title string, data map[string]interface{})

	// Trace takes a string and logs with LogLevel = Trace
	Trace(title string)

	// TraceD takes a string and data map. It logs with LogLevel = Trace
	TraceD(title string, data map[string]interface{})

	// Debug takes a string and logs with LogLevel = Debug
	Debug(title string)

	// DebugD takes a string and data map. It logs with LogLevel = Debug
	DebugD(title string, data map[string]interface{})

	// Error takes a string and logs with LogLevel = Error
	Error(title string)

	// ErrorD takes a string and data map. It logs with LogLevel = Error
	ErrorD(title string, data map[string]interface{})

	// GaugeFloat takes a string and float value. It logs with LogLevel = Info
	GaugeFloat(title string, value float64)

	// GaugeFloatD takes a string, a float value, and data map. It logs with LogLevel = Info
	GaugeFloatD(title string, value float64, data map[string]interface{})

	// GaugeInt takes a string and integer value. It logs with LogLevel = Info
	GaugeInt(title string, value int)

	// GaugeIntD takes a string, an integer value, and data map. It logs with LogLevel = Info
	GaugeIntD(title string, value int, data map[string]interface{})

	// Info takes a string and logs with LogLevel = Info
	Info(title string)

	// InfoD takes a string and data map. It logs with LogLevel = Info
	InfoD(title string, data map[string]interface{})

	// Timer takes a string and logs with LogLevel = Debug
	Timer(tittle string) *Timer

	// TimerD takes a string and data map. It logs with LogLevel = Debug
	TimerD(tittle string, data map[string]interface{}) *Timer

	// Warn takes a string and logs with LogLevel = Warning
	Warn(title string)

	// WarnD takes a string and data map. It logs with LogLevel = Warning
	WarnD(title string, data map[string]interface{})
}
<commit_msg>typo fix<commit_after>package logger

import (
	"io"

	"github.com/Clever/kayvee-go/v7/router"
)

/////////////////////////////
//
//	KayveeLogger interface
//
/////////////////////////////

// KayveeLogger is the main logging interface, providing customization of log messages.
type KayveeLogger interface {

	//
	// Configuration
	//

	// AddContext adds a new key-val to be logged with all log messages.
	AddContext(key, val string)

	// GetContext reads a key-val from the global map of data that will be logged with all log messages.
	GetContext(key string) (interface{}, bool)

	// SetConfig allows configuration changes in one go
	SetConfig(source string, logLvl LogLevel, formatter Formatter, output io.Writer)

	// SetFormatter sets the formatter function to use
	SetFormatter(formatter Formatter)

	// SetLogLevel sets the default log level threshold
	SetLogLevel(logLvl LogLevel)

	// SetOutput changes the output destination of the logger
	SetOutput(output io.Writer)

	// setFormatLogger use for to implemente the mock
	setFormatLogger(fl formatLogger)

	// SetRouter changes the router for this logger instance.  Once set, logs produced by this
	// logger will not be touched by the global router.  Mostly used for testing and benchmarking.
	SetRouter(router router.Router)

	//
	// Logging
	//

	// Counter takes a string and logs with LogLevel = Info
	Counter(title string)

	// CounterD takes a string, value, and data map. It logs with LogLevel = Info
	CounterD(title string, value int, data map[string]interface{})

	// Critical takes a string and logs with LogLevel = Critical
	Critical(title string)

	// CriticalD takes a string and data map. It logs with LogLevel = Critical
	CriticalD(title string, data map[string]interface{})

	// Trace takes a string and logs with LogLevel = Trace
	Trace(title string)

	// TraceD takes a string and data map. It logs with LogLevel = Trace
	TraceD(title string, data map[string]interface{})

	// Debug takes a string and logs with LogLevel = Debug
	Debug(title string)

	// DebugD takes a string and data map. It logs with LogLevel = Debug
	DebugD(title string, data map[string]interface{})

	// Error takes a string and logs with LogLevel = Error
	Error(title string)

	// ErrorD takes a string and data map. It logs with LogLevel = Error
	ErrorD(title string, data map[string]interface{})

	// GaugeFloat takes a string and float value. It logs with LogLevel = Info
	GaugeFloat(title string, value float64)

	// GaugeFloatD takes a string, a float value, and data map. It logs with LogLevel = Info
	GaugeFloatD(title string, value float64, data map[string]interface{})

	// GaugeInt takes a string and integer value. It logs with LogLevel = Info
	GaugeInt(title string, value int)

	// GaugeIntD takes a string, an integer value, and data map. It logs with LogLevel = Info
	GaugeIntD(title string, value int, data map[string]interface{})

	// Info takes a string and logs with LogLevel = Info
	Info(title string)

	// InfoD takes a string and data map. It logs with LogLevel = Info
	InfoD(title string, data map[string]interface{})

	// Timer takes a string and logs with LogLevel = Debug
	Timer(title string) *Timer

	// TimerD takes a string and data map. It logs with LogLevel = Debug
	TimerD(title string, data map[string]interface{}) *Timer

	// Warn takes a string and logs with LogLevel = Warning
	Warn(title string)

	// WarnD takes a string and data map. It logs with LogLevel = Warning
	WarnD(title string, data map[string]interface{})
}
<|endoftext|>


<commit_before>package camoproxy

import (
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	"github.com/gorilla/mux"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

type testStruct struct {
	Desc   string
	Url    string
	Status int
}

var camoConfig = Config{
	HmacKey:        "0x24FEEDFACEDEADBEEFCAFE",
	MaxSize:        5120 * 1024,
	RequestTimeout: time.Duration(6) * time.Second,
	MaxRedirects:   3}

func makeReq(testUrl string) (*http.Request, error) {
	k := []byte(camoConfig.HmacKey)
	hexurl := encoding.EncodeUrl(&k, testUrl)
	out := "http://example.com" + hexurl
	req, err := http.NewRequest("GET", out, nil)
	if err != nil {
		return nil, fmt.Errorf("Error building req url '%s': %s", testUrl, err.Error())
	}
	return req, nil
}

func processRequest(req *http.Request, status int) (*httptest.ResponseRecorder, error) {
	camoServer, err := New(camoConfig)
	if err != nil {
		return nil, fmt.Errorf("Error building CamoProxy: %s", err.Error())
	}

	router := mux.NewRouter()
	router.Handle("/{sigHash}/{encodedUrl}", camoServer).Methods("GET")

	record := httptest.NewRecorder()
	router.ServeHTTP(record, req)
	if got, want := record.Code, status; got != want {
		return nil, fmt.Errorf("response code = %d, wanted %d", got, want)
	}
	return record, nil
}

func makeTestReq(testUrl string, status int) (*httptest.ResponseRecorder, error) {
	req, err := makeReq(testUrl)
	if err != nil {
		return nil, err
	}
	record, err := processRequest(req, status)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func TestSimpleValidImageUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://media.ebaumsworld.com/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestGoogleChartUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://chart.apis.google.com/chart?chs=920x200&chxl=0:%7C2010-08-13%7C2010-09-12%7C2010-10-12%7C2010-11-11%7C1:%7C0%7C0%7C0%7C0%7C0%7C0&chm=B,EBF5FB,0,0,0&chco=008Cd6&chls=3,1,0&chg=8.3,20,1,4&chd=s:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&chxt=x,y&cht=lc"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestChunkedImageFile(t *testing.T) {
	t.Parallel()
	testUrl := "http://www.igvita.com/posts/12/spdyproxy-diagram.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/1K0X2Y2F1P0o3z140p0d/boom-headshot.gif"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestStrangeFormatRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/DPcp/Screen%20Shot%202012-01-17%20at%203.42.32%20PM.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestRedirectsWithPathOnly(t *testing.T) {
	t.Parallel()
	testUrl := "http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowTempRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://d.pr/i/rr7F+"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404InfiniRedirect(t *testing.T) {
	t.Parallel()
	testUrl := "http://modeselektor.herokuapp.com/"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404UrlWithoutHTTPHost(t *testing.T) {
	t.Parallel()
	testUrl := "/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404ImageLargerThan5MB(t *testing.T) {
	t.Parallel()
	testUrl := "http://apod.nasa.gov/apod/image/0505/larryslookout_spirit_big.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404HostNotFound(t *testing.T) {
	t.Parallel()
	testUrl := "http://flabergasted.cx"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnExcludes(t *testing.T) {
	t.Parallel()
	testUrl := "http://iphone.internal.example.org/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnNonImageContent(t *testing.T) {
	t.Parallel()
	testUrl := "https://github.com/atmos/cinderella/raw/master/bootstrap.sh"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On10xIpRange(t *testing.T) {
	t.Parallel()
	testUrl := "http://10.0.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On169Dot254Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://169.254.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On172Dot16Net(t *testing.T) {
	t.Parallel()
	for i := 16; i < 32; i++ {
		testUrl := "http://172.%d.0.1/foo.cgi"
		_, err := makeTestReq(fmt.Sprintf(testUrl, i), 404)
		if err != nil {
			t.Errorf(err.Error())
		}
	}
}

func Test404On192Dot168Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://192.168.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestSupplyAcceptIfNoneGiven(t *testing.T) {
	t.Parallel()
	testUrl := "http://images.anandtech.com/doci/6673/OpenMoboAMD30_575px.png"
	req, err := makeReq(testUrl)
	req.Header.Del("Accept")
	if err != nil {
		t.Errorf(err.Error())
	}
	_, err = processRequest(req, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}
<commit_msg>bump timeout for tests/travis<commit_after>package camoproxy

import (
	"fmt"
	"github.com/cactus/go-camo/camoproxy/encoding"
	"github.com/gorilla/mux"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"
)

type testStruct struct {
	Desc   string
	Url    string
	Status int
}

var camoConfig = Config{
	HmacKey:        "0x24FEEDFACEDEADBEEFCAFE",
	MaxSize:        5120 * 1024,
	RequestTimeout: time.Duration(10) * time.Second,
	MaxRedirects:   3}

func makeReq(testUrl string) (*http.Request, error) {
	k := []byte(camoConfig.HmacKey)
	hexurl := encoding.EncodeUrl(&k, testUrl)
	out := "http://example.com" + hexurl
	req, err := http.NewRequest("GET", out, nil)
	if err != nil {
		return nil, fmt.Errorf("Error building req url '%s': %s", testUrl, err.Error())
	}
	return req, nil
}

func processRequest(req *http.Request, status int) (*httptest.ResponseRecorder, error) {
	camoServer, err := New(camoConfig)
	if err != nil {
		return nil, fmt.Errorf("Error building CamoProxy: %s", err.Error())
	}

	router := mux.NewRouter()
	router.Handle("/{sigHash}/{encodedUrl}", camoServer).Methods("GET")

	record := httptest.NewRecorder()
	router.ServeHTTP(record, req)
	if got, want := record.Code, status; got != want {
		return nil, fmt.Errorf("response code = %d, wanted %d", got, want)
	}
	return record, nil
}

func makeTestReq(testUrl string, status int) (*httptest.ResponseRecorder, error) {
	req, err := makeReq(testUrl)
	if err != nil {
		return nil, err
	}
	record, err := processRequest(req, status)
	if err != nil {
		return nil, err
	}
	return record, nil
}

func TestSimpleValidImageUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://media.ebaumsworld.com/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestGoogleChartUrl(t *testing.T) {
	t.Parallel()
	testUrl := "http://chart.apis.google.com/chart?chs=920x200&chxl=0:%7C2010-08-13%7C2010-09-12%7C2010-10-12%7C2010-11-11%7C1:%7C0%7C0%7C0%7C0%7C0%7C0&chm=B,EBF5FB,0,0,0&chco=008Cd6&chls=3,1,0&chg=8.3,20,1,4&chd=s:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&chxt=x,y&cht=lc"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestChunkedImageFile(t *testing.T) {
	t.Parallel()
	testUrl := "http://www.igvita.com/posts/12/spdyproxy-diagram.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/1K0X2Y2F1P0o3z140p0d/boom-headshot.gif"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestStrangeFormatRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://cl.ly/DPcp/Screen%20Shot%202012-01-17%20at%203.42.32%20PM.png"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestRedirectsWithPathOnly(t *testing.T) {
	t.Parallel()
	testUrl := "http://blogs.msdn.com/photos/noahric/images/9948044/425x286.aspx"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestFollowTempRedirects(t *testing.T) {
	t.Parallel()
	testUrl := "http://d.pr/i/rr7F+"
	_, err := makeTestReq(testUrl, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404InfiniRedirect(t *testing.T) {
	t.Parallel()
	testUrl := "http://modeselektor.herokuapp.com/"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404UrlWithoutHTTPHost(t *testing.T) {
	t.Parallel()
	testUrl := "/picture/Mincemeat/Pimp.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404ImageLargerThan5MB(t *testing.T) {
	t.Parallel()
	testUrl := "http://apod.nasa.gov/apod/image/0505/larryslookout_spirit_big.jpg"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404HostNotFound(t *testing.T) {
	t.Parallel()
	testUrl := "http://flabergasted.cx"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnExcludes(t *testing.T) {
	t.Parallel()
	testUrl := "http://iphone.internal.example.org/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404OnNonImageContent(t *testing.T) {
	t.Parallel()
	testUrl := "https://github.com/atmos/cinderella/raw/master/bootstrap.sh"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On10xIpRange(t *testing.T) {
	t.Parallel()
	testUrl := "http://10.0.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On169Dot254Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://169.254.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func Test404On172Dot16Net(t *testing.T) {
	t.Parallel()
	for i := 16; i < 32; i++ {
		testUrl := "http://172.%d.0.1/foo.cgi"
		_, err := makeTestReq(fmt.Sprintf(testUrl, i), 404)
		if err != nil {
			t.Errorf(err.Error())
		}
	}
}

func Test404On192Dot168Net(t *testing.T) {
	t.Parallel()
	testUrl := "http://192.168.0.1/foo.cgi"
	_, err := makeTestReq(testUrl, 404)
	if err != nil {
		t.Errorf(err.Error())
	}
}

func TestSupplyAcceptIfNoneGiven(t *testing.T) {
	t.Parallel()
	testUrl := "http://images.anandtech.com/doci/6673/OpenMoboAMD30_575px.png"
	req, err := makeReq(testUrl)
	req.Header.Del("Accept")
	if err != nil {
		t.Errorf(err.Error())
	}
	_, err = processRequest(req, 200)
	if err != nil {
		t.Errorf(err.Error())
	}
}
<|endoftext|>


<commit_before>package master

import (
	"fmt"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy"
	pconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/proxy/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	etcdconfig "github.com/coreos/etcd/config"
	"github.com/coreos/etcd/etcd"
	etcdclient "github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"
	"github.com/google/cadvisor/client"
	"github.com/openshift/origin/pkg/api"
	"github.com/openshift/origin/pkg/service"
	"github.com/spf13/cobra"
)

func NewCommandStartAllInOne(name string) *cobra.Command {
	return &cobra.Command{
		Use:   name,
		Short: "Launch in all-in-one mode",
		Run: func(c *cobra.Command, args []string) {
			startAllInOne()
		},
	}
}

func startAllInOne() {
	dockerAddr := getDockerEndpoint("")
	dockerClient, err := docker.NewClient(dockerAddr)
	if err != nil {
		glog.Fatal("Couldn't connect to docker.")
	}
	if err := dockerClient.Ping(); err != nil {
		glog.Errorf("WARNING: Docker could not be reached at %s.  Docker must be installed and running to start containers.\n%v", dockerAddr, err)
	}

	cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
	if err != nil {
		glog.Errorf("Error on creating cadvisor client: %v", err)
	}

	// initialize etcd
	etcdAddr := "127.0.0.1:4001"
	etcdServers := []string{} // default
	etcdConfig := etcdconfig.New()
	etcdConfig.BindAddr = etcdAddr
	etcdServer := etcd.New(etcdConfig)
	go util.Forever(func() {
		glog.Infof("Started etcd at http://%s", etcdAddr)
		etcdServer.Run()
	}, 0)

	etcdClient := etcdclient.NewClient(etcdServers)
	for i := 0; ; i += 1 {
		_, err := etcdClient.Get("/", false, false)
		if err == nil || tools.IsEtcdNotFound(err) {
			break
		}
		if i > 100 {
			glog.Fatal("Could not reach etcd: %v", err)
		}
		time.Sleep(50 * time.Millisecond)
	}

	// initialize Kubelet
	minionHost := "127.0.0.1"
	minionPort := 10250
	rootDirectory := path.Clean("/var/lib/openshift")
	os.MkdirAll(rootDirectory, 0750)
	cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)
	kconfig.NewSourceEtcd(kconfig.EtcdKeyForHost(minionHost), etcdClient, 30*time.Second, cfg.Channel("etcd"))
	k := kubelet.NewMainKubelet(
		minionHost,
		dockerClient,
		cadvisorClient,
		etcdClient,
		rootDirectory,
		30*time.Second)
	go util.Forever(func() { k.Run(cfg.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(k, cfg.Channel("http"), http.DefaultServeMux, minionHost, uint(minionPort))
	}, 0)

	// initialize OpenShift API
	storage := map[string]apiserver.RESTStorage{
		"services": service.NewRESTStorage(service.MakeMemoryRegistry()),
	}
	osAddr := "127.0.0.1:8081"
	osPrefix := "/osapi/v1beta1"
	osApi := &http.Server{
		Addr:           osAddr,
		Handler:        apiserver.New(storage, api.Codec, osPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	go util.Forever(func() {
		glog.Infof("Started OpenShift API at http://%s%s", osAddr, osPrefix)
		glog.Fatal(osApi.ListenAndServe())
	}, 0)

	// initialize Kubernetes API
	kubeAddr := "127.0.0.1:8080"
	kubePrefix := "/api/v1beta1"
	kubeClient := client.New(fmt.Sprintf("http://%s", kubeAddr), nil)
	podInfoGetter := &client.HTTPPodInfoGetter{
		Client: http.DefaultClient,
		Port:   uint(minionPort),
	}
	masterConfig := &master.Config{
		Client:             kubeClient,
		EtcdServers:        etcdServers,
		HealthCheckMinions: true,
		Minions:            []string{minionHost},
		PodInfoGetter:      podInfoGetter,
	}
	m := master.New(masterConfig)
	go util.Forever(func() {
		glog.Infof("Started Kubernetes API at http://%s%s", kubeAddr, kubePrefix)
		m.Run(kubeAddr, kubePrefix)
	}, 0)

	// initialize kube proxy
	serviceConfig := pconfig.NewServiceConfig()
	endpointsConfig := pconfig.NewEndpointsConfig()
	pconfig.NewConfigSourceEtcd(etcdClient,
		serviceConfig.Channel("etcd"),
		endpointsConfig.Channel("etcd"))
	loadBalancer := proxy.NewLoadBalancerRR()
	proxier := proxy.NewProxier(loadBalancer)
	serviceConfig.RegisterHandler(proxier)
	endpointsConfig.RegisterHandler(loadBalancer)
	glog.Infof("Started Kubernetes Proxy")

	// initialize replication manager
	controllerManager := controller.MakeReplicationManager(kubeClient)
	controllerManager.Run(10 * time.Second)
	glog.Infof("Started Kubernetes Replication Manager")

	select {}
}

func getDockerEndpoint(dockerEndpoint string) string {
	var endpoint string
	if len(dockerEndpoint) > 0 {
		endpoint = dockerEndpoint
	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
		endpoint = os.Getenv("DOCKER_HOST")
	} else {
		endpoint = "unix:///var/run/docker.sock"
	}
	glog.Infof("Connecting to docker on %s", endpoint)

	return endpoint
}
<commit_msg>Integrate with BuildController<commit_after>package master

import (
	"fmt"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/build"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/proxy"
	pconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/proxy/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	etcdconfig "github.com/coreos/etcd/config"
	"github.com/coreos/etcd/etcd"
	etcdclient "github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	"github.com/golang/glog"
	"github.com/google/cadvisor/client"
	"github.com/openshift/origin/pkg/api"
	"github.com/openshift/origin/pkg/service"
	"github.com/spf13/cobra"
)

func NewCommandStartAllInOne(name string) *cobra.Command {
	return &cobra.Command{
		Use:   name,
		Short: "Launch in all-in-one mode",
		Run: func(c *cobra.Command, args []string) {
			startAllInOne()
		},
	}
}

func startAllInOne() {
	dockerAddr := getDockerEndpoint("")
	dockerClient, err := docker.NewClient(dockerAddr)
	if err != nil {
		glog.Fatal("Couldn't connect to docker.")
	}
	if err := dockerClient.Ping(); err != nil {
		glog.Errorf("WARNING: Docker could not be reached at %s.  Docker must be installed and running to start containers.\n%v", dockerAddr, err)
	}

	cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
	if err != nil {
		glog.Errorf("Error on creating cadvisor client: %v", err)
	}

	// initialize etcd
	etcdAddr := "127.0.0.1:4001"
	etcdServers := []string{} // default
	etcdConfig := etcdconfig.New()
	etcdConfig.BindAddr = etcdAddr
	etcdServer := etcd.New(etcdConfig)
	go util.Forever(func() {
		glog.Infof("Started etcd at http://%s", etcdAddr)
		etcdServer.Run()
	}, 0)

	etcdClient := etcdclient.NewClient(etcdServers)
	for i := 0; ; i += 1 {
		_, err := etcdClient.Get("/", false, false)
		if err == nil || tools.IsEtcdNotFound(err) {
			break
		}
		if i > 100 {
			glog.Fatal("Could not reach etcd: %v", err)
		}
		time.Sleep(50 * time.Millisecond)
	}

	// initialize Kubelet
	minionHost := "127.0.0.1"
	minionPort := 10250
	rootDirectory := path.Clean("/var/lib/openshift")
	os.MkdirAll(rootDirectory, 0750)
	cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)
	kconfig.NewSourceEtcd(kconfig.EtcdKeyForHost(minionHost), etcdClient, 30*time.Second, cfg.Channel("etcd"))
	k := kubelet.NewMainKubelet(
		minionHost,
		dockerClient,
		cadvisorClient,
		etcdClient,
		rootDirectory,
		30*time.Second)
	go util.Forever(func() { k.Run(cfg.Updates()) }, 0)
	go util.Forever(func() {
		kubelet.ListenAndServeKubeletServer(k, cfg.Channel("http"), http.DefaultServeMux, minionHost, uint(minionPort))
	}, 0)

	// initialize OpenShift API
	storage := map[string]apiserver.RESTStorage{
		"services": service.NewRESTStorage(service.MakeMemoryRegistry()),
	}
	osAddr := "127.0.0.1:8081"
	osPrefix := "/osapi/v1beta1"
	osApi := &http.Server{
		Addr:           osAddr,
		Handler:        apiserver.New(storage, api.Codec, osPrefix),
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	go util.Forever(func() {
		glog.Infof("Started OpenShift API at http://%s%s", osAddr, osPrefix)
		glog.Fatal(osApi.ListenAndServe())
	}, 0)

	// initialize Kubernetes API
	kubeAddr := "127.0.0.1:8080"
	kubePrefix := "/api/v1beta1"
	kubeClient := client.New(fmt.Sprintf("http://%s", kubeAddr), nil)
	podInfoGetter := &client.HTTPPodInfoGetter{
		Client: http.DefaultClient,
		Port:   uint(minionPort),
	}
	masterConfig := &master.Config{
		Client:             kubeClient,
		EtcdServers:        etcdServers,
		HealthCheckMinions: true,
		Minions:            []string{minionHost},
		PodInfoGetter:      podInfoGetter,
	}
	m := master.New(masterConfig)
	go util.Forever(func() {
		glog.Infof("Started Kubernetes API at http://%s%s", kubeAddr, kubePrefix)
		m.Run(kubeAddr, kubePrefix)
	}, 0)

	// initialize kube proxy
	serviceConfig := pconfig.NewServiceConfig()
	endpointsConfig := pconfig.NewEndpointsConfig()
	pconfig.NewConfigSourceEtcd(etcdClient,
		serviceConfig.Channel("etcd"),
		endpointsConfig.Channel("etcd"))
	loadBalancer := proxy.NewLoadBalancerRR()
	proxier := proxy.NewProxier(loadBalancer)
	serviceConfig.RegisterHandler(proxier)
	endpointsConfig.RegisterHandler(loadBalancer)
	glog.Infof("Started Kubernetes Proxy")

	// initialize replication manager
	controllerManager := controller.MakeReplicationManager(kubeClient)
	controllerManager.Run(10 * time.Second)
	glog.Infof("Started Kubernetes Replication Manager")

	// initialize build controller
	if env("BUILD_POD_CLEANUP_ENABLED", "true") == "true" {
		os.Setenv("BUILD_POD_CLEANUP_ENABLED", "true")
	}
	buildController := build.MakeBuildController(kubeClient, "openshift/docker-builder",
		env("DOCKER_REGISTRY_URL", ""), "openshift/sti-builder", 120)
	buildController.Run(10 * time.Second)

	select {}
}

func env(key string, defaultValue string) string {
	val := os.Getenv(key)
	if len(val) == 0 {
		return defaultValue
	} else {
		return val
	}
}

func getDockerEndpoint(dockerEndpoint string) string {
	var endpoint string
	if len(dockerEndpoint) > 0 {
		endpoint = dockerEndpoint
	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
		endpoint = os.Getenv("DOCKER_HOST")
	} else {
		endpoint = "unix:///var/run/docker.sock"
	}
	glog.Infof("Connecting to docker on %s", endpoint)

	return endpoint
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
	"github.com/gohttp/pprof"
	"github.com/meatballhat/negroni-logrus"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"
	"github.com/yadvendar/negroni-newrelic-go-agent"
)

// SetupGlobalMiddleware setup the global middleware
func SetupGlobalMiddleware(handler http.Handler) http.Handler {
	n := negroni.New()

	n.Use(negroni.NewRecovery())

	if Config.MiddlewareGzipEnabled {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}

	if Config.CORSEnabled {
		n.Use(cors.New(cors.Options{
			AllowedOrigins: []string{"*"},
			AllowedHeaders: []string{"Content-Type"},
			ExposedHeaders: []string{"Www-Authenticate"},
			AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"},
		}))
	}

	if Config.StatsdEnabled {
		n.Use(&statsdMiddleware{StatsdClient: Global.StatsdClient})
	}

	if Config.NewRelicEnabled {
		n.Use(&negroninewrelic.Newrelic{Application: &Global.NewrelicApp})
	}

	if Config.JWTAuthEnabled {
		n.Use(setupJWTAuthMiddleware())
	}

	if Config.MiddlewareVerboseLoggerEnabled {
		n.Use(negronilogrus.NewMiddlewareFromLogger(logrus.StandardLogger(), "flagr"))
	}

	n.Use(&negroni.Static{
		Dir:       http.Dir("./browser/flagr-ui/dist/"),
		Prefix:    Config.WebPrefix,
		IndexFile: "index.html",
	})

	if Config.PProfEnabled {
		n.UseHandler(pprof.New()(handler))
	} else {
		n.UseHandler(handler)
	}

	return n
}

/**
setupJWTAuthMiddleware setup an JWTMiddleware from the ENV config
*/
func setupJWTAuthMiddleware() *auth {
	var signingMethod jwt.SigningMethod
	var validationKey interface{}
	var errParsingKey error

	switch Config.JWTAuthSigningMethod {
	case "HS256":
		signingMethod = jwt.SigningMethodHS256
		validationKey = []byte(Config.JWTAuthSecret)
	case "RS256":
		signingMethod = jwt.SigningMethodRS256
		validationKey, errParsingKey = jwt.ParseRSAPublicKeyFromPEM([]byte(Config.JWTAuthSecret))
	default:
		signingMethod = jwt.SigningMethodHS256
		validationKey = []byte("")
	}

	return &auth{
		PrefixWhitelistPaths: Config.JWTAuthPrefixWhitelistPaths,
		ExactWhitelistPaths:  Config.JWTAuthExactWhitelistPaths,
		JWTMiddleware: jwtmiddleware.New(jwtmiddleware.Options{
			ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
				return validationKey, errParsingKey
			},
			SigningMethod: signingMethod,
			Extractor: jwtmiddleware.FromFirst(
				func(r *http.Request) (string, error) {
					c, err := r.Cookie(Config.JWTAuthCookieTokenName)
					if err != nil {
						return "", nil
					}
					return c.Value, nil
				},
				jwtmiddleware.FromAuthHeader,
			),
			UserProperty: Config.JWTAuthUserProperty,
			Debug:        Config.JWTAuthDebug,
			ErrorHandler: jwtErrorHandler,
		}),
	}
}

func jwtErrorHandler(w http.ResponseWriter, r *http.Request, err string) {
	switch Config.JWTAuthNoTokenStatusCode {
	case http.StatusTemporaryRedirect:
		http.Redirect(w, r, Config.JWTAuthNoTokenRedirectURL, http.StatusTemporaryRedirect)
		return
	default:
		w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, Config.JWTAuthNoTokenRedirectURL))
		http.Error(w, "Not authorized", http.StatusUnauthorized)
		return
	}
}

type auth struct {
	PrefixWhitelistPaths []string
	ExactWhitelistPaths  []string
	JWTMiddleware        *jwtmiddleware.JWTMiddleware
}

func (a *auth) whitelist(req *http.Request) bool {
	path := req.URL.Path

	// If we set to 401 unauthorized, let the client handles the 401 itself
	if Config.JWTAuthNoTokenStatusCode == http.StatusUnauthorized {
		for _, p := range a.ExactWhitelistPaths {
			if p == path {
				return true
			}
		}
	}

	for _, p := range a.PrefixWhitelistPaths {
		if p != "" && strings.HasPrefix(path, p) {
			return true
		}
	}
	return false
}

func (a *auth) ServeHTTP(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
	if a.whitelist(req) {
		next(w, req)
		return
	}
	a.JWTMiddleware.HandlerWithNext(w, req, next)
}

type statsdMiddleware struct {
	StatsdClient *statsd.Client
}

func (s *statsdMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer func(start time.Time) {
		response := w.(negroni.ResponseWriter)
		status := strconv.Itoa(response.Status())
		duration := float64(time.Since(start)) / float64(time.Millisecond)
		tags := []string{
			"status:" + status,
			"path:" + r.RequestURI,
			"method:" + r.Method,
		}

		s.StatsdClient.Incr("http.requests.count", tags, 1)
		s.StatsdClient.TimeInMilliseconds("http.requests.duration", duration, tags, 1)
	}(time.Now())

	next(w, r)
}
<commit_msg>Add headers to CORS middleware<commit_after>package config

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
	"github.com/gohttp/pprof"
	"github.com/meatballhat/negroni-logrus"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/sirupsen/logrus"
	"github.com/urfave/negroni"
	"github.com/yadvendar/negroni-newrelic-go-agent"
)

// SetupGlobalMiddleware setup the global middleware
func SetupGlobalMiddleware(handler http.Handler) http.Handler {
	n := negroni.New()

	n.Use(negroni.NewRecovery())

	if Config.MiddlewareGzipEnabled {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}

	if Config.CORSEnabled {
		n.Use(cors.New(cors.Options{
			AllowedOrigins:   []string{"*"},
			AllowedHeaders:   []string{"Origin", "Accept", "Content-Type", "X-Requested-With", "Authorization", "Time_Zone"},
			ExposedHeaders:   []string{"Www-Authenticate"},
			AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "PATCH"},
			AllowCredentials: true,
		}))
	}

	if Config.StatsdEnabled {
		n.Use(&statsdMiddleware{StatsdClient: Global.StatsdClient})
	}

	if Config.NewRelicEnabled {
		n.Use(&negroninewrelic.Newrelic{Application: &Global.NewrelicApp})
	}

	if Config.JWTAuthEnabled {
		n.Use(setupJWTAuthMiddleware())
	}

	if Config.MiddlewareVerboseLoggerEnabled {
		n.Use(negronilogrus.NewMiddlewareFromLogger(logrus.StandardLogger(), "flagr"))
	}

	n.Use(&negroni.Static{
		Dir:       http.Dir("./browser/flagr-ui/dist/"),
		Prefix:    Config.WebPrefix,
		IndexFile: "index.html",
	})

	if Config.PProfEnabled {
		n.UseHandler(pprof.New()(handler))
	} else {
		n.UseHandler(handler)
	}

	return n
}

/**
setupJWTAuthMiddleware setup an JWTMiddleware from the ENV config
*/
func setupJWTAuthMiddleware() *auth {
	var signingMethod jwt.SigningMethod
	var validationKey interface{}
	var errParsingKey error

	switch Config.JWTAuthSigningMethod {
	case "HS256":
		signingMethod = jwt.SigningMethodHS256
		validationKey = []byte(Config.JWTAuthSecret)
	case "RS256":
		signingMethod = jwt.SigningMethodRS256
		validationKey, errParsingKey = jwt.ParseRSAPublicKeyFromPEM([]byte(Config.JWTAuthSecret))
	default:
		signingMethod = jwt.SigningMethodHS256
		validationKey = []byte("")
	}

	return &auth{
		PrefixWhitelistPaths: Config.JWTAuthPrefixWhitelistPaths,
		ExactWhitelistPaths:  Config.JWTAuthExactWhitelistPaths,
		JWTMiddleware: jwtmiddleware.New(jwtmiddleware.Options{
			ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
				return validationKey, errParsingKey
			},
			SigningMethod: signingMethod,
			Extractor: jwtmiddleware.FromFirst(
				func(r *http.Request) (string, error) {
					c, err := r.Cookie(Config.JWTAuthCookieTokenName)
					if err != nil {
						return "", nil
					}
					return c.Value, nil
				},
				jwtmiddleware.FromAuthHeader,
			),
			UserProperty: Config.JWTAuthUserProperty,
			Debug:        Config.JWTAuthDebug,
			ErrorHandler: jwtErrorHandler,
		}),
	}
}

func jwtErrorHandler(w http.ResponseWriter, r *http.Request, err string) {
	switch Config.JWTAuthNoTokenStatusCode {
	case http.StatusTemporaryRedirect:
		http.Redirect(w, r, Config.JWTAuthNoTokenRedirectURL, http.StatusTemporaryRedirect)
		return
	default:
		w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, Config.JWTAuthNoTokenRedirectURL))
		http.Error(w, "Not authorized", http.StatusUnauthorized)
		return
	}
}

type auth struct {
	PrefixWhitelistPaths []string
	ExactWhitelistPaths  []string
	JWTMiddleware        *jwtmiddleware.JWTMiddleware
}

func (a *auth) whitelist(req *http.Request) bool {
	path := req.URL.Path

	// If we set to 401 unauthorized, let the client handles the 401 itself
	if Config.JWTAuthNoTokenStatusCode == http.StatusUnauthorized {
		for _, p := range a.ExactWhitelistPaths {
			if p == path {
				return true
			}
		}
	}

	for _, p := range a.PrefixWhitelistPaths {
		if p != "" && strings.HasPrefix(path, p) {
			return true
		}
	}
	return false
}

func (a *auth) ServeHTTP(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
	if a.whitelist(req) {
		next(w, req)
		return
	}
	a.JWTMiddleware.HandlerWithNext(w, req, next)
}

type statsdMiddleware struct {
	StatsdClient *statsd.Client
}

func (s *statsdMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer func(start time.Time) {
		response := w.(negroni.ResponseWriter)
		status := strconv.Itoa(response.Status())
		duration := float64(time.Since(start)) / float64(time.Millisecond)
		tags := []string{
			"status:" + status,
			"path:" + r.RequestURI,
			"method:" + r.Method,
		}

		s.StatsdClient.Incr("http.requests.count", tags, 1)
		s.StatsdClient.TimeInMilliseconds("http.requests.duration", duration, tags, 1)
	}(time.Now())

	next(w, r)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type state interface {
	Advance(txt string) (state, error)
}

type stateStart struct{}

func (s stateStart) Advance(txt string) (state, error) {
	if txt == "! Dark" || txt == "! Light" {
		return stateColorTitle{}, nil
	}
	return s, nil
}

type stateColorTitle struct{}

func (s stateColorTitle) Advance(txt string) (state, error) {
	if !strings.Contains(txt, "#define ") {
		return nil, errors.New("expected color definitions")
	}
	return stateColorBody{}, nil
}

type stateColorBody struct{}

func (s stateColorBody) Advance(txt string) (state, error) {
	if !strings.Contains(txt, "#define ") {
		return stateStart{}, nil
	}
	return s, nil
}

func main() {
	xresources := filepath.Join(os.Getenv("HOME"), ".Xresources")

	fx, err := os.OpenFile(xresources, os.O_RDWR, 0640)
	if err != nil {
		panic(err)
	}
	defer fx.Close()

	err = switchResources(fx)
	if err != nil {
		panic(err)
	}

	err = exec.Command("xrdb", "-merge", xresources).Run()
	if err != nil {
		panic(err)
	}
}

func switchResources(fx *os.File) error {
	var out bytes.Buffer
	scanner := bufio.NewScanner(fx)

	var s state = stateStart{}
	var err error
	for scanner.Scan() {
		txt := scanner.Text()
		s, err = s.Advance(txt)
		if err != nil {
			return fmt.Errorf("invalid state transition: %v", err)
		}
		if _, ok := s.(stateColorBody); ok {
			txt = flip(txt)
		}
		out.WriteString(txt + "\n")
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	d := bytes.TrimSuffix(out.Bytes(), []byte("\n"))

	fx.Seek(0, 0)
	n, err := fx.Write(d)
	if err == nil && n < len(d) {
		return io.ErrShortWrite
	}

	return err
}

func flip(txt string) string {
	if strings.HasPrefix(txt, "! ") {
		return txt[2:]
	}
	return "! " + txt
}
<commit_msg>Make solarswitcher more standards-compliant<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type state interface {
	Advance(txt string) (state, error)
}

type stateStart struct{}

func (s stateStart) Advance(txt string) (state, error) {
	if txt == "! Dark" || txt == "! Light" {
		return stateColorTitle{}, nil
	}
	return s, nil
}

type stateColorTitle struct{}

func (s stateColorTitle) Advance(txt string) (state, error) {
	if !strings.Contains(txt, "#define ") {
		return nil, errors.New("expected color definitions")
	}
	return stateColorBody{}, nil
}

type stateColorBody struct{}

func (s stateColorBody) Advance(txt string) (state, error) {
	if !strings.Contains(txt, "#define ") {
		return stateStart{}, nil
	}
	return s, nil
}

func main() {
	xresources := filepath.Join(os.Getenv("XDG_CONFIG_HOME"), "X11", "xresources")

	fx, err := os.OpenFile(xresources, os.O_RDWR, 0640)
	if err != nil {
		panic(err)
	}
	defer fx.Close()

	err = switchResources(fx)
	if err != nil {
		panic(err)
	}

	err = exec.Command("xrdb", "-merge", xresources).Run()
	if err != nil {
		panic(err)
	}
}

func switchResources(fx *os.File) error {
	var out bytes.Buffer
	scanner := bufio.NewScanner(fx)

	var s state = stateStart{}
	var err error
	for scanner.Scan() {
		txt := scanner.Text()
		s, err = s.Advance(txt)
		if err != nil {
			return fmt.Errorf("invalid state transition: %v", err)
		}
		if _, ok := s.(stateColorBody); ok {
			txt = flip(txt)
		}
		out.WriteString(txt + "\n")
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	d := bytes.TrimSuffix(out.Bytes(), []byte("\n"))

	fx.Seek(0, 0)
	n, err := fx.Write(d)
	if err == nil && n < len(d) {
		return io.ErrShortWrite
	}

	return err
}

func flip(txt string) string {
	if strings.HasPrefix(txt, "! ") {
		return txt[2:]
	}
	return "! " + txt
}
<|endoftext|>


<commit_before>package nv

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"
)

func check_array(t *testing.T, field string, value interface{}, l int, fn func([]string)) {
	expecteds := strings.Split(field, ";")
	if len(expecteds) < 2 {
		t.Fatal("field does not seem to be a list of expected values:", field)
	}
	if len(expecteds) != l {
		t.Fatal("length mismatch between expected and decoded arrays, expected:",
			len(expecteds), "decoded:", l)
	}
	fn(expecteds)
}

func check_boolean(t *testing.T, field string, value interface{}) {
	exp := false
	_, err := fmt.Sscanf(field, "%t", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := bool(value.(Boolean))
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_boolean_value(t *testing.T, field string, value interface{}) {
	exp := false
	_, err := fmt.Sscanf(field, "%t", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(bool)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_boolean_array(t *testing.T, field string, value interface{}) {
	arr := value.([]bool)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_boolean_value(t, expecteds[i], arr[i])
		}
	})
}

func check_byte(t *testing.T, field string, value interface{}) {
	d := int(0)
	_, err := fmt.Sscanf(field, "%d", &d)
	if err != nil {
		t.Fatal(err)
	}
	exp := byte(d)
	got := value.(byte)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_byte_array(t *testing.T, field string, value interface{}) {
	arr := value.([]byte)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_byte(t, expecteds[i], arr[i])
		}
	})
}

func check_double(t *testing.T, field string, value interface{}) {
	exp := float64(0)
	// is a hack, but good enough
	_, err := fmt.Sscanf(field, "%f", &exp)
	if err != nil {
		t.Fatal(err)
	}

	got := value.(float64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}

func check_hrtime(t *testing.T, field string, value interface{}) {
	d := int64(0)
	_, err := fmt.Sscanf(field, "%d", &d)
	if err != nil {
		t.Fatal(err)
	}
	exp := time.Duration(d)
	got := value.(time.Duration)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}

func check_int8(t *testing.T, field string, value interface{}) {
	exp := int8(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int8)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int8_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int8)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int8(t, expecteds[i], arr[i])
		}
	})
}

func check_int16(t *testing.T, field string, value interface{}) {
	exp := int16(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int16)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int16_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int16)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int16(t, expecteds[i], arr[i])
		}
	})
}

func check_int32(t *testing.T, field string, value interface{}) {
	exp := int32(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int32)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int32_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int32)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int32(t, expecteds[i], arr[i])
		}
	})
}

func check_int64(t *testing.T, field string, value interface{}) {
	exp := int64(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int64_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int64)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int64(t, expecteds[i], arr[i])
		}
	})
}

func check_uint8(t *testing.T, field string, value interface{}) {
	exp := uint8(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint8)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint8_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint8)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint8(t, expecteds[i], arr[i])
		}
	})
}

func check_uint16(t *testing.T, field string, value interface{}) {
	exp := uint16(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint16)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint16_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint16)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint16(t, expecteds[i], arr[i])
		}
	})
}

func check_uint32(t *testing.T, field string, value interface{}) {
	exp := uint32(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint32)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint32_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint32)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint32(t, expecteds[i], arr[i])
		}
	})
}

func check_uint64(t *testing.T, field string, value interface{}) {
	exp := uint64(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint64_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint64)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint64(t, expecteds[i], arr[i])
		}
	})
}

func check_nvlist(t *testing.T, field string, value interface{}) {
	list := value.(map[string]interface{})
	for f, v := range list {
		checkers[reflect.TypeOf(v).String()](t, f, v)
	}
}
func check_nvlist_array(t *testing.T, field string, value interface{}) {
	arr := value.([]map[string]interface{})
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_nvlist(t, expecteds[i], arr[i])
		}
	})
}

func check_string(t *testing.T, field string, value interface{}) {
	exp := field
	got := value.(string)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_string_array(t *testing.T, field string, value interface{}) {
	arr := value.([]string)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_string(t, expecteds[i], arr[i])
		}
	})
}

var checkers map[string]func(*testing.T, string, interface{})

func init() {
	checkers = map[string]func(*testing.T, string, interface{}){
		"nv.Boolean":                check_boolean,
		"bool":                      check_boolean_value,
		"byte":                      check_byte,
		"int16":                     check_int16,
		"uint16":                    check_uint16,
		"int32":                     check_int32,
		"uint32":                    check_uint32,
		"int64":                     check_int64,
		"uint64":                    check_uint64,
		"string":                    check_string,
		"[]byte":                    check_byte_array,
		"[]int16":                   check_int16_array,
		"[]uint16":                  check_uint16_array,
		"[]int32":                   check_int32_array,
		"[]uint32":                  check_uint32_array,
		"[]int64":                   check_int64_array,
		"[]uint64":                  check_uint64_array,
		"[]string":                  check_string_array,
		"time.Duration":             check_hrtime,
		"map[string]interface {}":   check_nvlist,
		"[]map[string]interface {}": check_nvlist_array,
		"int8":    check_int8,
		"uint8":   check_uint8,
		"[]bool":  check_boolean_array,
		"[]int8":  check_int8_array,
		"[]uint8": check_uint8_array,
		"float64": check_double,
	}
}

func struct2map(s interface{}) map[string]interface{} {
	m := map[string]interface{}{}
	val := reflect.ValueOf(s).Elem()
	for name, index := range fieldIndexMap(val) {
		m[name] = val.Field(index).Interface()
	}
	return m
}

func assertFields(t *testing.T, name string, m map[string]interface{}) {
	for field, value := range m {
		valueType := reflect.TypeOf(value).String()
		fn, ok := checkers[valueType]
		if !ok {
			t.Fatal(name, "unknown type:", valueType)
		}
		fn(t, field, value)
	}
}

//go:generate make -C _test ../known_good_data_test.go
func TestDecodeGood(t *testing.T) {
	for _, test := range good {
		t.Log(test.name)

		m := map[string]interface{}{}
		err := Decode(test.payload, &m)
		if err != nil {
			t.Fatal(test.name, "decode as map failed:", err)
		}

		assertFields(t, test.name, m)

		s := test.ptr()
		err = Decode(test.payload, s)
		if err != nil {
			t.Fatal(test.name, "decode as struct failed:", err)
		}

		m = struct2map(s)
		if len(m) != reflect.ValueOf(s).Elem().NumField() {
			t.Fatalf("incorrect number of fields, got: %d %+v want: %d %+v\n",
				len(m), m, reflect.ValueOf(s).Elem().NumField(), s)
		}
		assertFields(t, test.name, m)
	}
}

func TestDecodeBad(t *testing.T) {
	for _, test := range decode_bad {
		t.Log(test.err)

		m := map[string]interface{}{}
		err := Decode(test.payload, &m)
		if err == nil {
			t.Fatalf("expected an error, wanted:|%s| payload:|%v|\n",
				test.err, test.payload)
		}
		if test.err != err.Error() {
			t.Fatalf("error mismatch, want:|%s| got:|%s| payload:|%v|\n",
				test.err, err.Error(), test.payload)
		}
	}
}

func TestDecodeBadArgs(t *testing.T) {
	bad_args := []struct {
		arg interface{}
		err string
	}{
		{
			arg: struct{}{},
			err: "cannot decode into non-pointer: reflect.Value",
		},
	}
	for _, test := range bad_args {
		err := Decode([]byte(enc_dec_name_typ), test.arg)
		if err == nil {
			t.Fatalf("expected an error, wanted:|%s|\n", test.err)
		}
		if test.err != err.Error() {
			t.Fatalf("error mismatch, want:|%s| got:|%s|\n",
				test.err, err.Error())
		}
	}
}
<commit_msg>set make to be silent<commit_after>package nv

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"
)

func check_array(t *testing.T, field string, value interface{}, l int, fn func([]string)) {
	expecteds := strings.Split(field, ";")
	if len(expecteds) < 2 {
		t.Fatal("field does not seem to be a list of expected values:", field)
	}
	if len(expecteds) != l {
		t.Fatal("length mismatch between expected and decoded arrays, expected:",
			len(expecteds), "decoded:", l)
	}
	fn(expecteds)
}

func check_boolean(t *testing.T, field string, value interface{}) {
	exp := false
	_, err := fmt.Sscanf(field, "%t", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := bool(value.(Boolean))
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_boolean_value(t *testing.T, field string, value interface{}) {
	exp := false
	_, err := fmt.Sscanf(field, "%t", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(bool)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_boolean_array(t *testing.T, field string, value interface{}) {
	arr := value.([]bool)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_boolean_value(t, expecteds[i], arr[i])
		}
	})
}

func check_byte(t *testing.T, field string, value interface{}) {
	d := int(0)
	_, err := fmt.Sscanf(field, "%d", &d)
	if err != nil {
		t.Fatal(err)
	}
	exp := byte(d)
	got := value.(byte)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_byte_array(t *testing.T, field string, value interface{}) {
	arr := value.([]byte)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_byte(t, expecteds[i], arr[i])
		}
	})
}

func check_double(t *testing.T, field string, value interface{}) {
	exp := float64(0)
	// is a hack, but good enough
	_, err := fmt.Sscanf(field, "%f", &exp)
	if err != nil {
		t.Fatal(err)
	}

	got := value.(float64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}

func check_hrtime(t *testing.T, field string, value interface{}) {
	d := int64(0)
	_, err := fmt.Sscanf(field, "%d", &d)
	if err != nil {
		t.Fatal(err)
	}
	exp := time.Duration(d)
	got := value.(time.Duration)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}

func check_int8(t *testing.T, field string, value interface{}) {
	exp := int8(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int8)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int8_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int8)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int8(t, expecteds[i], arr[i])
		}
	})
}

func check_int16(t *testing.T, field string, value interface{}) {
	exp := int16(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int16)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int16_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int16)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int16(t, expecteds[i], arr[i])
		}
	})
}

func check_int32(t *testing.T, field string, value interface{}) {
	exp := int32(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int32)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int32_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int32)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int32(t, expecteds[i], arr[i])
		}
	})
}

func check_int64(t *testing.T, field string, value interface{}) {
	exp := int64(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(int64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_int64_array(t *testing.T, field string, value interface{}) {
	arr := value.([]int64)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_int64(t, expecteds[i], arr[i])
		}
	})
}

func check_uint8(t *testing.T, field string, value interface{}) {
	exp := uint8(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint8)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint8_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint8)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint8(t, expecteds[i], arr[i])
		}
	})
}

func check_uint16(t *testing.T, field string, value interface{}) {
	exp := uint16(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint16)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint16_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint16)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint16(t, expecteds[i], arr[i])
		}
	})
}

func check_uint32(t *testing.T, field string, value interface{}) {
	exp := uint32(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint32)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint32_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint32)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint32(t, expecteds[i], arr[i])
		}
	})
}

func check_uint64(t *testing.T, field string, value interface{}) {
	exp := uint64(0)
	_, err := fmt.Sscanf(field, "%d", &exp)
	if err != nil {
		t.Fatal(err)
	}
	got := value.(uint64)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_uint64_array(t *testing.T, field string, value interface{}) {
	arr := value.([]uint64)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_uint64(t, expecteds[i], arr[i])
		}
	})
}

func check_nvlist(t *testing.T, field string, value interface{}) {
	list := value.(map[string]interface{})
	for f, v := range list {
		checkers[reflect.TypeOf(v).String()](t, f, v)
	}
}
func check_nvlist_array(t *testing.T, field string, value interface{}) {
	arr := value.([]map[string]interface{})
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_nvlist(t, expecteds[i], arr[i])
		}
	})
}

func check_string(t *testing.T, field string, value interface{}) {
	exp := field
	got := value.(string)
	if got != exp {
		t.Fatal("expected:", exp, "got:", got)
	}
}
func check_string_array(t *testing.T, field string, value interface{}) {
	arr := value.([]string)
	check_array(t, field, value, len(arr), func(expecteds []string) {
		for i := range arr {
			check_string(t, expecteds[i], arr[i])
		}
	})
}

var checkers map[string]func(*testing.T, string, interface{})

func init() {
	checkers = map[string]func(*testing.T, string, interface{}){
		"nv.Boolean":                check_boolean,
		"bool":                      check_boolean_value,
		"byte":                      check_byte,
		"int16":                     check_int16,
		"uint16":                    check_uint16,
		"int32":                     check_int32,
		"uint32":                    check_uint32,
		"int64":                     check_int64,
		"uint64":                    check_uint64,
		"string":                    check_string,
		"[]byte":                    check_byte_array,
		"[]int16":                   check_int16_array,
		"[]uint16":                  check_uint16_array,
		"[]int32":                   check_int32_array,
		"[]uint32":                  check_uint32_array,
		"[]int64":                   check_int64_array,
		"[]uint64":                  check_uint64_array,
		"[]string":                  check_string_array,
		"time.Duration":             check_hrtime,
		"map[string]interface {}":   check_nvlist,
		"[]map[string]interface {}": check_nvlist_array,
		"int8":    check_int8,
		"uint8":   check_uint8,
		"[]bool":  check_boolean_array,
		"[]int8":  check_int8_array,
		"[]uint8": check_uint8_array,
		"float64": check_double,
	}
}

func struct2map(s interface{}) map[string]interface{} {
	m := map[string]interface{}{}
	val := reflect.ValueOf(s).Elem()
	for name, index := range fieldIndexMap(val) {
		m[name] = val.Field(index).Interface()
	}
	return m
}

func assertFields(t *testing.T, name string, m map[string]interface{}) {
	for field, value := range m {
		valueType := reflect.TypeOf(value).String()
		fn, ok := checkers[valueType]
		if !ok {
			t.Fatal(name, "unknown type:", valueType)
		}
		fn(t, field, value)
	}
}

//go:generate make -s -C _test ../known_good_data_test.go
func TestDecodeGood(t *testing.T) {
	for _, test := range good {
		t.Log(test.name)

		m := map[string]interface{}{}
		err := Decode(test.payload, &m)
		if err != nil {
			t.Fatal(test.name, "decode as map failed:", err)
		}

		assertFields(t, test.name, m)

		s := test.ptr()
		err = Decode(test.payload, s)
		if err != nil {
			t.Fatal(test.name, "decode as struct failed:", err)
		}

		m = struct2map(s)
		if len(m) != reflect.ValueOf(s).Elem().NumField() {
			t.Fatalf("incorrect number of fields, got: %d %+v want: %d %+v\n",
				len(m), m, reflect.ValueOf(s).Elem().NumField(), s)
		}
		assertFields(t, test.name, m)
	}
}

func TestDecodeBad(t *testing.T) {
	for _, test := range decode_bad {
		t.Log(test.err)

		m := map[string]interface{}{}
		err := Decode(test.payload, &m)
		if err == nil {
			t.Fatalf("expected an error, wanted:|%s| payload:|%v|\n",
				test.err, test.payload)
		}
		if test.err != err.Error() {
			t.Fatalf("error mismatch, want:|%s| got:|%s| payload:|%v|\n",
				test.err, err.Error(), test.payload)
		}
	}
}

func TestDecodeBadArgs(t *testing.T) {
	bad_args := []struct {
		arg interface{}
		err string
	}{
		{
			arg: struct{}{},
			err: "cannot decode into non-pointer: reflect.Value",
		},
	}
	for _, test := range bad_args {
		err := Decode([]byte(enc_dec_name_typ), test.arg)
		if err == nil {
			t.Fatalf("expected an error, wanted:|%s|\n", test.err)
		}
		if test.err != err.Error() {
			t.Fatalf("error mismatch, want:|%s| got:|%s|\n",
				test.err, err.Error())
		}
	}
}
<|endoftext|>