<commit_before>package namedwebsockets

import (
	"time"

	"github.com/gorilla/websocket"

	"log"
	"strings"
	"github.com/notificator"

	"net/http"
	"container/list"
	"fmt"

	"runtime"
)

var notify *notificator.Notificator

var peerNotifiedList list.List

type PeerConnection struct {
	// Unique identifier for this peer connection
	id int

	// WebSocket connection object
	ws *websocket.Conn
}

type Message struct {
	// The source peer connection of the message
	source int

	// The destination peer connection id target
	target int

	// The message payload
	payload string

	// Whether this message originated from a ProxyConnection object
	fromProxy bool
}

func NewPeerConnection(id int, socket *websocket.Conn) *PeerConnection {
	peerConn := &PeerConnection{
		id: id,
		ws: socket,
	}

	return peerConn
}

// Send a message to the target websocket connection
func (conn *PeerConnection) send(payload string) {
	conn.ws.SetWriteDeadline(time.Now().Add(writeWait))
	conn.ws.WriteMessage(websocket.TextMessage, []byte(payload))
}

// readConnectionPump pumps messages from an individual websocket connection to the dispatcher
func (peer *PeerConnection) readConnectionPump(sock *NamedWebSocket) {
	defer func() {
		peer.removeConnection(sock)
	}()
	peer.ws.SetReadLimit(maxMessageSize)
	peer.ws.SetReadDeadline(time.Now().Add(pongWait))
	peer.ws.SetPongHandler(func(string) error { peer.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		opCode, message, err := peer.ws.ReadMessage()
		if err != nil || opCode != websocket.TextMessage {
			break
		}

		//log.Printf("Received Peer Message: %s",message)

		wsBroadcast := &Message{
			source:    peer.id,
			target:    0, // target all connections
			payload:   string(message),
			fromProxy: false,
		}
		var notiRecived bool = false

		for e := peerNotifiedList.Front(); e != nil; e=e.Next() {
			if(string(message)==e.Value.(string)){
				notiRecived = true
			}
		}

		//create notifications with the url for linux and android
		if(strings.Contains(string(message),"http://")&&!notiRecived){

			path, err2 := filepath.Abs("Mediascape.png")
			if err2 != nil {
				log.Fatal(err)
			}
			//log.Printf("Notification %s", path)

			notify = notificator.New(notificator.Options{
				DefaultIcon: path,
				AppName:     "Mediascape",
			})

			if(string(runtime.GOOS)=="linux"&&string(runtime.GOARCH)!="arm"){
				notify.Push("Mediascape", string(message), path)
			}else if(string(runtime.GOOS)=="linux"&&string(runtime.GOARCH)=="arm"){
				completUrl := fmt.Sprintf("http://localhost:8182/discoveryagent/notification?url=%s", string(message))
				response, err := http.Get(completUrl)
				if err != nil {
					// handle error
					log.Printf("Error: %s",err)
				}else{
					defer response.Body.Close()
				}
			}
		}
		sock.broadcastBuffer <- wsBroadcast
	}
}

// writeConnectionPump keeps an individual websocket connection alive
func (peer *PeerConnection) writeConnectionPump(sock *NamedWebSocket) {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		peer.removeConnection(sock)
	}()
	for {
		select {
		case <-ticker.C:
			peer.ws.SetWriteDeadline(time.Now().Add(writeWait))
			peer.ws.WriteMessage(websocket.PingMessage, []byte{})
		}
	}
}

// Set up a new NamedWebSocket connection instance
func (peer *PeerConnection) addConnection(sock *NamedWebSocket) {
	// Add this websocket instance to Named WebSocket broadcast list
	sock.peers = append(sock.peers, peer)

	// Inform all control connections that we now own this peer connection
	for _, control := range sock.controllers {
		// don't notify controller if its id matches the peer's id
		if control.id != peer.id {
			control.send("connect", control.id, peer.id, "")
		}
	}

	// Inform all proxy connections that we now own this peer connection
	for _, proxy := range sock.proxies {
		if proxy.writeable {
			proxy.send("connect", proxy.id, peer.id, "")
		}
	}

	// Start connection read/write pumps
	go peer.writeConnectionPump(sock)
	go peer.readConnectionPump(sock)
}

// Tear down an existing NamedWebSocket connection instance
func (peer *PeerConnection) removeConnection(sock *NamedWebSocket) {
	for i, conn := range sock.peers {
		if conn.id == peer.id {
			sock.peers = append(sock.peers[:i], sock.peers[i+1:]...)
			break
		}
	}

	// Find associated control connection and close also
	for _, control := range sock.controllers {
		if control.id == peer.id {
			control.removeConnection(sock)
			break
		}
	}

	// Inform all control connections that we no longer own this peer connection
	for _, control := range sock.controllers {
		// don't notify controller if its id matches the peer's id
		if control.id != peer.id {
			control.send("disconnect", control.id, peer.id, "")
		}
	}

	// Inform all proxy connections that we no longer own this peer connection
	for _, proxy := range sock.proxies {
		if proxy.writeable {
			proxy.send("disconnect", proxy.id, peer.id, "")
		}
	}

	peer.ws.Close()
}
<commit_msg>Update peer.go<commit_after>package namedwebsockets

import (
	"time"

	"github.com/gorilla/websocket"

	"log"
	"strings"
	"github.com/notificator"

	"net/http"
	"container/list"
	"fmt"

	"runtime"
)

var notify *notificator.Notificator

var peerNotifiedList list.List

type PeerConnection struct {
	// Unique identifier for this peer connection
	id int

	// WebSocket connection object
	ws *websocket.Conn
}

type Message struct {
	// The source peer connection of the message
	source int

	// The destination peer connection id target
	target int

	// The message payload
	payload string

	// Whether this message originated from a ProxyConnection object
	fromProxy bool
}

func NewPeerConnection(id int, socket *websocket.Conn) *PeerConnection {
	peerConn := &PeerConnection{
		id: id,
		ws: socket,
	}

	return peerConn
}

// Send a message to the target websocket connection
func (conn *PeerConnection) send(payload string) {
	conn.ws.SetWriteDeadline(time.Now().Add(writeWait))
	conn.ws.WriteMessage(websocket.TextMessage, []byte(payload))
}

// readConnectionPump pumps messages from an individual websocket connection to the dispatcher
func (peer *PeerConnection) readConnectionPump(sock *NamedWebSocket) {
	defer func() {
		peer.removeConnection(sock)
	}()
	peer.ws.SetReadLimit(maxMessageSize)
	peer.ws.SetReadDeadline(time.Now().Add(pongWait))
	peer.ws.SetPongHandler(func(string) error { peer.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		opCode, message, err := peer.ws.ReadMessage()
		if err != nil || opCode != websocket.TextMessage {
			break
		}

		//log.Printf("Received Peer Message: %s",message)

		wsBroadcast := &Message{
			source:    peer.id,
			target:    0, // target all connections
			payload:   string(message),
			fromProxy: false,
		}
		var notiRecived bool = false

		for e := peerNotifiedList.Front(); e != nil; e=e.Next() {
			if(string(message)==e.Value.(string)){
				notiRecived = true
			}
		}

		//create notifications with the url for linux and android
		if(strings.Contains(string(message),"http://")&&!notiRecived){

			peerNotifiedList.PushBack(message.Payload)

			path, err2 := filepath.Abs("Mediascape.png")
			if err2 != nil {
				log.Fatal(err)
			}
			//log.Printf("Notification %s", path)

			notify = notificator.New(notificator.Options{
				DefaultIcon: path,
				AppName:     "Mediascape",
			})

			if(string(runtime.GOOS)=="linux"&&string(runtime.GOARCH)!="arm"){
				notify.Push("Mediascape", string(message), path)
			}else if(string(runtime.GOOS)=="linux"&&string(runtime.GOARCH)=="arm"){
				completUrl := fmt.Sprintf("http://localhost:8182/discoveryagent/notification?url=%s", string(message))
				response, err := http.Get(completUrl)
				if err != nil {
					// handle error
					log.Printf("Error: %s",err)
				}else{
					defer response.Body.Close()
				}
			}
		}
		sock.broadcastBuffer <- wsBroadcast
	}
}

// writeConnectionPump keeps an individual websocket connection alive
func (peer *PeerConnection) writeConnectionPump(sock *NamedWebSocket) {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		peer.removeConnection(sock)
	}()
	for {
		select {
		case <-ticker.C:
			peer.ws.SetWriteDeadline(time.Now().Add(writeWait))
			peer.ws.WriteMessage(websocket.PingMessage, []byte{})
		}
	}
}

// Set up a new NamedWebSocket connection instance
func (peer *PeerConnection) addConnection(sock *NamedWebSocket) {
	// Add this websocket instance to Named WebSocket broadcast list
	sock.peers = append(sock.peers, peer)

	// Inform all control connections that we now own this peer connection
	for _, control := range sock.controllers {
		// don't notify controller if its id matches the peer's id
		if control.id != peer.id {
			control.send("connect", control.id, peer.id, "")
		}
	}

	// Inform all proxy connections that we now own this peer connection
	for _, proxy := range sock.proxies {
		if proxy.writeable {
			proxy.send("connect", proxy.id, peer.id, "")
		}
	}

	// Start connection read/write pumps
	go peer.writeConnectionPump(sock)
	go peer.readConnectionPump(sock)
}

// Tear down an existing NamedWebSocket connection instance
func (peer *PeerConnection) removeConnection(sock *NamedWebSocket) {
	for i, conn := range sock.peers {
		if conn.id == peer.id {
			sock.peers = append(sock.peers[:i], sock.peers[i+1:]...)
			break
		}
	}

	// Find associated control connection and close also
	for _, control := range sock.controllers {
		if control.id == peer.id {
			control.removeConnection(sock)
			break
		}
	}

	// Inform all control connections that we no longer own this peer connection
	for _, control := range sock.controllers {
		// don't notify controller if its id matches the peer's id
		if control.id != peer.id {
			control.send("disconnect", control.id, peer.id, "")
		}
	}

	// Inform all proxy connections that we no longer own this peer connection
	for _, proxy := range sock.proxies {
		if proxy.writeable {
			proxy.send("disconnect", proxy.id, peer.id, "")
		}
	}

	peer.ws.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func main() {
	log.SetOutput(os.Stderr)

	url1 := "https://homebrew.bintray.com/bottles/youtube-dl-2016.12.12.sierra.bottle.tar.gz"
	url2 := "https://homebrew.bintray.com/bottles/libtiff-4.0.7.sierra.bottle.tar.gz"

	var wg sync.WaitGroup
	p := mpb.New(mpb.WithWaitGroup(&wg))

	for i, url := range [...]string{url1, url2} {
		wg.Add(1)
		name := fmt.Sprintf("url%d:", i+1)
		go download(&wg, p, name, url, i)
	}

	p.Wait()
	fmt.Println("done")
}

func download(wg *sync.WaitGroup, p *mpb.Progress, name, url string, n int) {
	defer wg.Done()
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("%s: %v", name, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("non-200 status: %s", resp.Status)
		log.Printf("%s: %v", name, err)
		return
	}

	size := resp.ContentLength

	// create dest
	destName := filepath.Base(url)
	dest, err := os.Create(destName)
	if err != nil {
		err = fmt.Errorf("Can't create %s: %v", destName, err)
		log.Printf("%s: %v", name, err)
		return
	}

	// create bar with appropriate decorators
	bar := p.AddBar(size,
		mpb.PrependDecorators(
			decor.StaticName(name, 0, 0),
			decor.CountersKibiByte("%6.1f / %6.1f", 18, 0),
		),
		mpb.AppendDecorators(decor.ETA(5, decor.DwidthSync)),
	)
	// Respect the order
	p.UpdateBarPriority(bar, n)

	// create proxy reader
	reader := bar.ProxyReader(resp.Body)
	// and copy from reader
	_, err = io.Copy(dest, reader)

	if e := dest.Close(); err == nil {
		err = e
	}
	if err != nil {
		log.Printf("%s: %v", name, err)
	}
}
<commit_msg>use BarPriority option<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"github.com/vbauerster/mpb"
	"github.com/vbauerster/mpb/decor"
)

func main() {
	log.SetOutput(os.Stderr)

	url1 := "https://homebrew.bintray.com/bottles/youtube-dl-2016.12.12.sierra.bottle.tar.gz"
	url2 := "https://homebrew.bintray.com/bottles/libtiff-4.0.7.sierra.bottle.tar.gz"

	var wg sync.WaitGroup
	p := mpb.New(mpb.WithWaitGroup(&wg))

	for i, url := range [...]string{url1, url2} {
		wg.Add(1)
		name := fmt.Sprintf("url%d:", i+1)
		go download(&wg, p, name, url, i)
	}

	p.Wait()
	fmt.Println("done")
}

func download(wg *sync.WaitGroup, p *mpb.Progress, name, url string, n int) {
	defer wg.Done()
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("%s: %v", name, err)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("non-200 status: %s", resp.Status)
		log.Printf("%s: %v", name, err)
		return
	}

	size := resp.ContentLength

	// create dest
	destName := filepath.Base(url)
	dest, err := os.Create(destName)
	if err != nil {
		err = fmt.Errorf("Can't create %s: %v", destName, err)
		log.Printf("%s: %v", name, err)
		return
	}

	// create bar with appropriate decorators
	bar := p.AddBar(size, mpb.BarPriority(n),
		mpb.PrependDecorators(
			decor.StaticName(name, len(name)+1, decor.DidentRight),
			decor.CountersKibiByte("%6.1f / %6.1f", 0, decor.DwidthSync),
		),
		mpb.AppendDecorators(decor.ETA(3, 0)),
	)

	// create proxy reader
	reader := bar.ProxyReader(resp.Body)
	// and copy from reader
	_, err = io.Copy(dest, reader)

	if e := dest.Close(); err == nil {
		err = e
	}
	if err != nil {
		log.Printf("%s: %v", name, err)
	}
}
<|endoftext|>


<commit_before>// Response to Data Compression Challenge
// http://gafferongames.com/2015/03/14/the-networked-physics-data-compression-challenge/
//
// The main insight that this compression uses is
// given some previous state P and current state S
// the ordering by some attribute is very similar
//   order(P, attr) ~= order(S, attr)
//
// so we use the previous state to determine the order
// and delta encode the current attributes in that order
//
// Note that the ordering must give the same result on both computers
// but, it doesn't have to guarantee perfect ordering (e.g. see improveApprox)
//
// Anyways, the all the interesting stats
//
// MIN     17.760 kbps
// P05     27.360 kbps
// P10    152.160 kbps
// P25    293.760 kbps
// P50    416.640 kbps
// P75    525.120 kbps
// P90    617.760 kbps
// P95    666.720 kbps
// MAX    764.640 kbps
//
// TOTAL   18928.200 kb
//   AVG       6.672 kb per frame
//   AVG       7.405 bits per cube
//
// TIMING:
//                   MIN        10%        25%        50%        75%        90%        MAX
//    improve  611.469µs  742.339µs  810.231µs  879.909µs   949.14µs 1.013905ms 6.925824ms
//     encode  770.925µs  996.932µs 1.110829ms 1.181401ms 1.230533ms 1.269838ms 7.492182ms
//     decode  891.969µs  963.433µs  983.533µs 1.012565ms 1.031771ms 1.047851ms 1.308697ms
package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"sort"

	"github.com/egonelbre/exp/qpc"

	"github.com/montanaflynn/stats"
)

func check(err error) {
	if err != nil {
		panic(err)
	}
}

type Delta struct {
	Largest     int32
	A, B, C     int32
	X, Y, Z     int32
	Interacting int32
}

type Deltas []Delta

var previous_nochange []bool

func Encode(order *Ordering, baseline, current Deltas) (snapshot []byte) {
	wr := NewWriter()

	nochange := make([]bool, len(baseline))
	for i := range baseline {
		nochange[i] = baseline[i] == current[i]
	}
	previous_nochange = nochange
	wr.WriteBools(order.Sameness, nochange)

	wr.WriteDelta(nochange, order.Largest, current, getLargest)
	wr.WriteDelta(nochange, order.Interacting, current, getInteracting)
	wr.WriteIndexed(nochange, order.ABC, baseline, current)
	wr.WriteIndexed(nochange, order.XYZ, baseline, current)

	wr.Close()
	return wr.Bytes()
}

// ignores reading errors
func Decode(order *Ordering, baseline, current Deltas, snapshot []byte) {
	rd := NewReader(snapshot)

	nochange := make([]bool, len(baseline))
	rd.ReadBools(order.Sameness, nochange)

	copy(current, baseline)

	rd.ReadDelta(nochange, order.Largest, current, setLargest)
	rd.ReadDelta(nochange, order.Interacting, current, setInteracting)
	rd.ReadIndexed(nochange, order.ABC, baseline, current)
	rd.ReadIndexed(nochange, order.XYZ, baseline, current)
}

func ReadDelta(r io.Reader, current Deltas) error {
	for i := range current {
		if err := current[i].ReadFrom(r); err != nil {
			return err
		}
	}
	return nil
}

func main() {
	verbose := flag.Bool("v", false, "verbose output")
	flag.Parse()

	file, err := os.Open("delta_data.bin")
	check(err)
	defer file.Close()

	buffer := bufio.NewReader(file)

	total := 0.0
	speeds := make([]float64, 0)

	improve := qpc.NewHistory("improve")
	encode := qpc.NewHistory("encode")
	decode := qpc.NewHistory("decode")

	const N = 901
	baseline := make(Deltas, N)
	order := NewOrdering(baseline)

	var history [9]Deltas
	for i := range history {
		history[i] = make(Deltas, N)
	}

	frame := 0
	for i := 0; i < 6; i += 1 {
		ReadDelta(buffer, history[frame])
		frame += 1
	}

	mirror := make(Deltas, N)

	for {
		historic := history[(frame-8+len(history))%len(history)]
		baseline := history[(frame-6+len(history))%len(history)]
		current := history[frame%len(history)]

		err = ReadDelta(buffer, current)
		if err == io.EOF {
			break
		}
		check(err)
		frame += 1

		runtime.GC()

		//===
		improve.Start()
		order.Improve(historic, baseline)
		improve.Stop()

		encode.Start()
		snapshot := Encode(order, baseline, current)
		encode.Stop()

		decode.Start()
		Decode(order, baseline, mirror, snapshot)
		decode.Stop()
		//===

		size := float64(len(snapshot)*8) / 1000.0
		total += size

		speed := size * 60.0
		speeds = append(speeds, speed)

		if *verbose {
			if !current.Equals(mirror) {
				fmt.Print("! ")
			}
			fmt.Printf("%04d %8.3fkbps %10s %10s %10s %10s\n", frame, speed, improve.Last(), encode.Last(), decode.Last())
		} else {
			if current.Equals(mirror) {
				fmt.Print(".")
			} else {
				fmt.Print("X")
			}
		}
	}
	fmt.Println()

	fmt.Printf("MIN %10.3f kbps\n", stats.Min(speeds))
	for _, p := range []float64{5, 10, 25, 50, 75, 90, 95} {
		fmt.Printf("P%02.f %10.3f kbps\n", p, stats.Percentile(speeds, p))
	}
	fmt.Printf("MAX %10.3f kbps\n", stats.Max(speeds))

	fmt.Println()

	fmt.Printf("TOTAL  %10.3f kb\n", total)
	fmt.Printf("  AVG  %10.3f kb per frame\n", total/float64(frame))
	fmt.Printf("  AVG  %10.3f bits per cube\n", total*1000/float64(frame*N))

	fmt.Println()
	fmt.Println("TIMING:")
	qpc.PrintSummary(improve, encode, decode)
}

// delta utilities
func (d *Delta) ReadFrom(r io.Reader) error {
	return read(r, &d.Largest, &d.A, &d.B, &d.C, &d.X, &d.Y, &d.Z, &d.Interacting)
}

func (d *Delta) WriteTo(w io.Writer) error {
	return write(w, &d.Largest, &d.A, &d.B, &d.C, &d.X, &d.Y, &d.Z, &d.Interacting)
}

// for validating content
func (a Deltas) Equals(b Deltas) bool {
	if len(a) != len(b) {
		return false
	}
	for i, ax := range a {
		if ax != b[i] {
			return false
		}
	}
	return true
}

// getters / setters
func getLargest(a *Delta) int32     { return a.Largest }
func getInteracting(a *Delta) int32 { return a.Interacting }

func setLargest(a *Delta, v int32)     { a.Largest = v }
func setInteracting(a *Delta, v int32) { a.Interacting = v }

// Order management

type Ordering struct {
	Sameness []int

	Largest     []int
	Interacting []int
	ABC         []IndexValue
	XYZ         []IndexValue
}

func NewOrdering(deltas Deltas) *Ordering {
	n := len(deltas)
	order := &Ordering{
		Sameness: make([]int, n),

		Largest:     make([]int, n),
		Interacting: make([]int, n),

		ABC: make([]IndexValue, n*3),
		XYZ: make([]IndexValue, n*3),
	}

	for i := range order.Largest {
		order.Sameness[i] = i
		order.Largest[i] = i
		order.Interacting[i] = i
	}

	for i := 0; i < n*3; i += 1 {
		order.ABC[i].Index = i % n
		order.ABC[i].Value = byte(i / n)

		order.XYZ[i].Index = i % n
		order.XYZ[i].Value = byte(i/n + 3)
	}

	return order
}

var improve = improveSort

func improveSort(order []int, deltas Deltas, get Getter) {
	sort.Sort(&byGetter{order, deltas, get})
}

func improveApprox(order []int, deltas Deltas, get Getter) {
	const max_steps = 10
	for k := 0; k < max_steps; k += 1 {
		prev := int32(-(1 << 31))
		swapped := false
		for i, idx := range order {
			curr := get(&deltas[idx])
			if prev > curr {
				order[i-1], order[i] = order[i], order[i-1]
				swapped = true
			} else {
				prev = curr
			}
		}
		if !swapped {
			return
		}
	}
}

func (order *Ordering) Improve(historic, baseline Deltas) {
	sort.Sort(&bySameness{order.Sameness, historic, baseline})
	sort.Sort(&byGetter{order.Largest, baseline, getLargest})
	sort.Sort(&byGetter{order.Largest, baseline, getLargest})
	sort.Sort(&byGetter{order.Interacting, baseline, getInteracting})
	sort.Sort(&byDelta{order.ABC, historic, baseline})
	sort.Sort(&byDelta{order.XYZ, historic, baseline})
}

// reading input/output
func read(r io.Reader, vs ...interface{}) error {
	for _, v := range vs {
		err := binary.Read(r, binary.LittleEndian, v)
		if err != nil {
			return err
		}
	}
	return nil
}

func write(w io.Writer, vs ...interface{}) error {
	for _, v := range vs {
		err := binary.Write(w, binary.LittleEndian, v)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Update stats.<commit_after>// Response to Data Compression Challenge
// http://gafferongames.com/2015/03/14/the-networked-physics-data-compression-challenge/
//
// The main insight that this compression uses is
// given some previous state P and current state S
// the ordering by some attribute is very similar
//   order(P, attr) ~= order(S, attr)
//
// so we use the previous state to determine the order
// and delta encode the current attributes in that order
//
// Note that the ordering must give the same result on both computers
// but, it doesn't have to guarantee perfect ordering (e.g. see improveApprox)
//
// Anyways, the all the interesting stats
//
// MIN      6.720 kbps
// P05     15.360 kbps
// P10    128.640 kbps
// P25    269.760 kbps
// P50    382.560 kbps
// P75    481.440 kbps
// P90    568.800 kbps
// P95    623.520 kbps
// MAX    706.080 kbps
//
// TOTAL   17390.264 kb
//   AVG       6.130 kb per frame
//   AVG       6.803 bits per cube
//
// TIMING:
//                   MIN        10%        25%        50%        75%        90%        MAX
//    improve  772.265µs 1.139862ms 1.175594ms 1.229193ms 1.301104ms 1.368102ms 1.661108ms
//     encode  271.565µs  428.341µs  528.392µs  611.023µs  686.507µs  756.632µs 1.444927ms
//     decode  129.083µs  186.254µs  245.659µs  296.578µs  339.904µs  374.296µs  472.113µs
package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"sort"

	"github.com/egonelbre/exp/qpc"

	"github.com/montanaflynn/stats"
)

func check(err error) {
	if err != nil {
		panic(err)
	}
}

type Delta struct {
	Largest     int32
	A, B, C     int32
	X, Y, Z     int32
	Interacting int32
}

type Deltas []Delta

var previous_nochange []bool

func Encode(order *Ordering, baseline, current Deltas) (snapshot []byte) {
	wr := NewWriter()

	nochange := make([]bool, len(baseline))
	for i := range baseline {
		nochange[i] = baseline[i] == current[i]
	}
	previous_nochange = nochange
	wr.WriteBools(order.Sameness, nochange)

	wr.WriteDelta(nochange, order.Largest, current, getLargest)
	wr.WriteDelta(nochange, order.Interacting, current, getInteracting)
	wr.WriteIndexed(nochange, order.ABC, baseline, current)
	wr.WriteIndexed(nochange, order.XYZ, baseline, current)

	wr.Close()
	return wr.Bytes()
}

// ignores reading errors
func Decode(order *Ordering, baseline, current Deltas, snapshot []byte) {
	rd := NewReader(snapshot)

	nochange := make([]bool, len(baseline))
	rd.ReadBools(order.Sameness, nochange)

	copy(current, baseline)

	rd.ReadDelta(nochange, order.Largest, current, setLargest)
	rd.ReadDelta(nochange, order.Interacting, current, setInteracting)
	rd.ReadIndexed(nochange, order.ABC, baseline, current)
	rd.ReadIndexed(nochange, order.XYZ, baseline, current)
}

func ReadDelta(r io.Reader, current Deltas) error {
	for i := range current {
		if err := current[i].ReadFrom(r); err != nil {
			return err
		}
	}
	return nil
}

func main() {
	verbose := flag.Bool("v", false, "verbose output")
	flag.Parse()

	file, err := os.Open("delta_data.bin")
	check(err)
	defer file.Close()

	buffer := bufio.NewReader(file)

	total := 0.0
	speeds := make([]float64, 0)

	improve := qpc.NewHistory("improve")
	encode := qpc.NewHistory("encode")
	decode := qpc.NewHistory("decode")

	const N = 901
	baseline := make(Deltas, N)
	order := NewOrdering(baseline)

	var history [9]Deltas
	for i := range history {
		history[i] = make(Deltas, N)
	}

	frame := 0
	for i := 0; i < 6; i += 1 {
		ReadDelta(buffer, history[frame])
		frame += 1
	}

	mirror := make(Deltas, N)

	for {
		historic := history[(frame-8+len(history))%len(history)]
		baseline := history[(frame-6+len(history))%len(history)]
		current := history[frame%len(history)]

		err = ReadDelta(buffer, current)
		if err == io.EOF {
			break
		}
		check(err)
		frame += 1

		runtime.GC()

		//===
		improve.Start()
		order.Improve(historic, baseline)
		improve.Stop()

		encode.Start()
		snapshot := Encode(order, baseline, current)
		encode.Stop()

		decode.Start()
		Decode(order, baseline, mirror, snapshot)
		decode.Stop()
		//===

		size := float64(len(snapshot)*8) / 1000.0
		total += size

		speed := size * 60.0
		speeds = append(speeds, speed)

		if *verbose {
			if !current.Equals(mirror) {
				fmt.Print("! ")
			}
			fmt.Printf("%04d %8.3fkbps %10s %10s %10s %10s\n", frame, speed, improve.Last(), encode.Last(), decode.Last())
		} else {
			if current.Equals(mirror) {
				fmt.Print(".")
			} else {
				fmt.Print("X")
			}
		}
	}
	fmt.Println()

	fmt.Printf("MIN %10.3f kbps\n", stats.Min(speeds))
	for _, p := range []float64{5, 10, 25, 50, 75, 90, 95} {
		fmt.Printf("P%02.f %10.3f kbps\n", p, stats.Percentile(speeds, p))
	}
	fmt.Printf("MAX %10.3f kbps\n", stats.Max(speeds))

	fmt.Println()

	fmt.Printf("TOTAL  %10.3f kb\n", total)
	fmt.Printf("  AVG  %10.3f kb per frame\n", total/float64(frame))
	fmt.Printf("  AVG  %10.3f bits per cube\n", total*1000/float64(frame*N))

	fmt.Println()
	fmt.Println("TIMING:")
	qpc.PrintSummary(improve, encode, decode)
}

// delta utilities
func (d *Delta) ReadFrom(r io.Reader) error {
	return read(r, &d.Largest, &d.A, &d.B, &d.C, &d.X, &d.Y, &d.Z, &d.Interacting)
}

func (d *Delta) WriteTo(w io.Writer) error {
	return write(w, &d.Largest, &d.A, &d.B, &d.C, &d.X, &d.Y, &d.Z, &d.Interacting)
}

// for validating content
func (a Deltas) Equals(b Deltas) bool {
	if len(a) != len(b) {
		return false
	}
	for i, ax := range a {
		if ax != b[i] {
			return false
		}
	}
	return true
}

// getters / setters
func getLargest(a *Delta) int32     { return a.Largest }
func getInteracting(a *Delta) int32 { return a.Interacting }

func setLargest(a *Delta, v int32)     { a.Largest = v }
func setInteracting(a *Delta, v int32) { a.Interacting = v }

// Order management

type Ordering struct {
	Sameness []int

	Largest     []int
	Interacting []int
	ABC         []IndexValue
	XYZ         []IndexValue
}

func NewOrdering(deltas Deltas) *Ordering {
	n := len(deltas)
	order := &Ordering{
		Sameness: make([]int, n),

		Largest:     make([]int, n),
		Interacting: make([]int, n),

		ABC: make([]IndexValue, n*3),
		XYZ: make([]IndexValue, n*3),
	}

	for i := range order.Largest {
		order.Sameness[i] = i
		order.Largest[i] = i
		order.Interacting[i] = i
	}

	for i := 0; i < n*3; i += 1 {
		order.ABC[i].Index = i % n
		order.ABC[i].Value = byte(i / n)

		order.XYZ[i].Index = i % n
		order.XYZ[i].Value = byte(i/n + 3)
	}

	return order
}

var improve = improveSort

func improveSort(order []int, deltas Deltas, get Getter) {
	sort.Sort(&byGetter{order, deltas, get})
}

func improveApprox(order []int, deltas Deltas, get Getter) {
	const max_steps = 10
	for k := 0; k < max_steps; k += 1 {
		prev := int32(-(1 << 31))
		swapped := false
		for i, idx := range order {
			curr := get(&deltas[idx])
			if prev > curr {
				order[i-1], order[i] = order[i], order[i-1]
				swapped = true
			} else {
				prev = curr
			}
		}
		if !swapped {
			return
		}
	}
}

func (order *Ordering) Improve(historic, baseline Deltas) {
	sort.Sort(&bySameness{order.Sameness, historic, baseline})
	sort.Sort(&byGetter{order.Largest, baseline, getLargest})
	sort.Sort(&byGetter{order.Largest, baseline, getLargest})
	sort.Sort(&byGetter{order.Interacting, baseline, getInteracting})
	sort.Sort(&byDelta{order.ABC, historic, baseline})
	sort.Sort(&byDelta{order.XYZ, historic, baseline})
}

// reading input/output
func read(r io.Reader, vs ...interface{}) error {
	for _, v := range vs {
		err := binary.Read(r, binary.LittleEndian, v)
		if err != nil {
			return err
		}
	}
	return nil
}

func write(w io.Writer, vs ...interface{}) error {
	for _, v := range vs {
		err := binary.Write(w, binary.LittleEndian, v)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example helpers file.
	HelpersName = "_helpers.tpl"
	// TemplatesTestsDir is the relative directory name for templates tests.
	TemplatesTestsDir = "templates/tests"
	// TestConnectionName is the name of the example connection test file.
	TestConnectionName = "test-connection.yaml"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent

nameOverride: ""
fullnameOverride: ""

service:
  type: ClusterIP
  port: 80

ingress:
  enabled: false
  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  paths: []
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #  cpu: 100m
  #  memory: 128Mi
  # requests:
  #  cpu: 100m
  #  memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}
`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $fullName := include "<CHARTNAME>.fullname" . -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ $fullName }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.ingress.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
{{- if .Values.ingress.tls }}
  tls:
  {{- range .Values.ingress.tls }}
    - hosts:
      {{- range .hosts }}
        - {{ . | quote }}
      {{- end }}
      secretName: {{ .secretName }}
  {{- end }}
{{- end }}
  rules:
  {{- range .Values.ingress.hosts }}
    - host: {{ . | quote }}
      http:
        paths:
	{{- range $ingressPaths }}
          - path: {{ . }}
            backend:
              serviceName: {{ $fullName }}
              servicePort: http
	{{- end }}
  {{- end }}
{{- end }}
`

const defaultDeployment = `apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
      app.kubernetes.io/instance: {{ .Release.Name }}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
        app.kubernetes.io/instance: {{ .Release.Name }}
    spec:
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /
              port: http
          readinessProbe:
            httpGet:
              path: /
              port: http
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
    {{- end }}
    {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
    {{- end }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  type: {{ .Values.service.type }}
  ports:
    - port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP
      name: http
  selector:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
  {{- range $.Values.ingress.paths }}
  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }}
  {{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "<CHARTNAME>.fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get svc -w {{ include "<CHARTNAME>.fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "<CHARTNAME>.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "<CHARTNAME>.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:80
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "<CHARTNAME>.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "<CHARTNAME>.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "<CHARTNAME>.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

const defaultTestConnection = `apiVersion: v1
kind: Pod
metadata:
  name: "{{ include "<CHARTNAME>.fullname" . }}-test-connection"
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  annotations:
    "helm.sh/hook": test-success
spec:
  containers:
    - name: wget
      image: busybox
      command: ['wget']
      args:  ['{{ include "<CHARTNAME>.fullname" . }}:{{ .Values.service.port }}']
  restartPolicy: Never
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile

	var updatedTemplates []*chart.Template

	for _, template := range schart.Templates {
		newData := Transform(string(template.Data), "<CHARTNAME>", schart.Metadata.Name)
		updatedTemplates = append(updatedTemplates, &chart.Template{Name: template.Name, Data: newData})
	}

	schart.Templates = updatedTemplates
	if schart.Values != nil {
		schart.Values = &chart.Config{Raw: string(Transform(schart.Values.Raw, "<CHARTNAME>", schart.Metadata.Name))}
	}
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, TemplatesTestsDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: Transform(defaultIngress, "<CHARTNAME>", chartfile.Name),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: Transform(defaultDeployment, "<CHARTNAME>", chartfile.Name),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: Transform(defaultService, "<CHARTNAME>", chartfile.Name),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: Transform(defaultNotes, "<CHARTNAME>", chartfile.Name),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: Transform(defaultHelpers, "<CHARTNAME>", chartfile.Name),
		},
		{
			// test-connection.yaml
			path:    filepath.Join(cdir, TemplatesTestsDir, TestConnectionName),
			content: Transform(defaultTestConnection, "<CHARTNAME>", chartfile.Name),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<commit_msg>Added extra padding to resources template (#4981)<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package chartutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"k8s.io/helm/pkg/proto/hapi/chart"
)

const (
	// ChartfileName is the default Chart file name.
	ChartfileName = "Chart.yaml"
	// ValuesfileName is the default values file name.
	ValuesfileName = "values.yaml"
	// TemplatesDir is the relative directory name for templates.
	TemplatesDir = "templates"
	// ChartsDir is the relative directory name for charts dependencies.
	ChartsDir = "charts"
	// IgnorefileName is the name of the Helm ignore file.
	IgnorefileName = ".helmignore"
	// IngressFileName is the name of the example ingress file.
	IngressFileName = "ingress.yaml"
	// DeploymentName is the name of the example deployment file.
	DeploymentName = "deployment.yaml"
	// ServiceName is the name of the example service file.
	ServiceName = "service.yaml"
	// NotesName is the name of the example NOTES.txt file.
	NotesName = "NOTES.txt"
	// HelpersName is the name of the example helpers file.
	HelpersName = "_helpers.tpl"
	// TemplatesTestsDir is the relative directory name for templates tests.
	TemplatesTestsDir = "templates/tests"
	// TestConnectionName is the name of the example connection test file.
	TestConnectionName = "test-connection.yaml"
)

const defaultValues = `# Default values for %s.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

replicaCount: 1

image:
  repository: nginx
  tag: stable
  pullPolicy: IfNotPresent

nameOverride: ""
fullnameOverride: ""

service:
  type: ClusterIP
  port: 80

ingress:
  enabled: false
  annotations: {}
    # kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  paths: []
  hosts:
    - chart-example.local
  tls: []
  #  - secretName: chart-example-tls
  #    hosts:
  #      - chart-example.local

resources: {}
  # We usually recommend not to specify default resources and to leave this as a conscious
  # choice for the user. This also increases chances charts run on environments with little
  # resources, such as Minikube. If you do want to specify resources, uncomment the following
  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
  # limits:
  #   cpu: 100m
  #   memory: 128Mi
  # requests:
  #   cpu: 100m
  #   memory: 128Mi

nodeSelector: {}

tolerations: []

affinity: {}
`

const defaultIgnore = `# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
`

const defaultIngress = `{{- if .Values.ingress.enabled -}}
{{- $fullName := include "<CHARTNAME>.fullname" . -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: {{ $fullName }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  {{- with .Values.ingress.annotations }}
  annotations:
    {{- toYaml . | nindent 4 }}
  {{- end }}
spec:
{{- if .Values.ingress.tls }}
  tls:
  {{- range .Values.ingress.tls }}
    - hosts:
      {{- range .hosts }}
        - {{ . | quote }}
      {{- end }}
      secretName: {{ .secretName }}
  {{- end }}
{{- end }}
  rules:
  {{- range .Values.ingress.hosts }}
    - host: {{ . | quote }}
      http:
        paths:
	{{- range $ingressPaths }}
          - path: {{ . }}
            backend:
              serviceName: {{ $fullName }}
              servicePort: http
	{{- end }}
  {{- end }}
{{- end }}
`

const defaultDeployment = `apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
      app.kubernetes.io/instance: {{ .Release.Name }}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
        app.kubernetes.io/instance: {{ .Release.Name }}
    spec:
      containers:
        - name: {{ .Chart.Name }}
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /
              port: http
          readinessProbe:
            httpGet:
              path: /
              port: http
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
    {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
    {{- end }}
    {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
    {{- end }}
`

const defaultService = `apiVersion: v1
kind: Service
metadata:
  name: {{ include "<CHARTNAME>.fullname" . }}
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
  type: {{ .Values.service.type }}
  ports:
    - port: {{ .Values.service.port }}
      targetPort: http
      protocol: TCP
      name: http
  selector:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
`

const defaultNotes = `1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
  {{- range $.Values.ingress.paths }}
  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }}
  {{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "<CHARTNAME>.fullname" . }})
  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
  echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
           You can watch the status of by running 'kubectl get svc -w {{ include "<CHARTNAME>.fullname" . }}'
  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "<CHARTNAME>.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
  echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "<CHARTNAME>.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
  echo "Visit http://127.0.0.1:8080 to use your application"
  kubectl port-forward $POD_NAME 8080:80
{{- end }}
`

const defaultHelpers = `{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "<CHARTNAME>.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "<CHARTNAME>.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "<CHARTNAME>.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
`

const defaultTestConnection = `apiVersion: v1
kind: Pod
metadata:
  name: "{{ include "<CHARTNAME>.fullname" . }}-test-connection"
  labels:
    app.kubernetes.io/name: {{ include "<CHARTNAME>.name" . }}
    helm.sh/chart: {{ include "<CHARTNAME>.chart" . }}
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
  annotations:
    "helm.sh/hook": test-success
spec:
  containers:
    - name: wget
      image: busybox
      command: ['wget']
      args:  ['{{ include "<CHARTNAME>.fullname" . }}:{{ .Values.service.port }}']
  restartPolicy: Never
`

// CreateFrom creates a new chart, but scaffolds it from the src chart.
func CreateFrom(chartfile *chart.Metadata, dest string, src string) error {
	schart, err := Load(src)
	if err != nil {
		return fmt.Errorf("could not load %s: %s", src, err)
	}

	schart.Metadata = chartfile

	var updatedTemplates []*chart.Template

	for _, template := range schart.Templates {
		newData := Transform(string(template.Data), "<CHARTNAME>", schart.Metadata.Name)
		updatedTemplates = append(updatedTemplates, &chart.Template{Name: template.Name, Data: newData})
	}

	schart.Templates = updatedTemplates
	if schart.Values != nil {
		schart.Values = &chart.Config{Raw: string(Transform(schart.Values.Raw, "<CHARTNAME>", schart.Metadata.Name))}
	}
	return SaveDir(schart, dest)
}

// Create creates a new chart in a directory.
//
// Inside of dir, this will create a directory based on the name of
// chartfile.Name. It will then write the Chart.yaml into this directory and
// create the (empty) appropriate directories.
//
// The returned string will point to the newly created directory. It will be
// an absolute path, even if the provided base directory was relative.
//
// If dir does not exist, this will return an error.
// If Chart.yaml or any directories cannot be created, this will return an
// error. In such a case, this will attempt to clean up by removing the
// new chart directory.
func Create(chartfile *chart.Metadata, dir string) (string, error) {
	path, err := filepath.Abs(dir)
	if err != nil {
		return path, err
	}

	if fi, err := os.Stat(path); err != nil {
		return path, err
	} else if !fi.IsDir() {
		return path, fmt.Errorf("no such directory %s", path)
	}

	n := chartfile.Name
	cdir := filepath.Join(path, n)
	if fi, err := os.Stat(cdir); err == nil && !fi.IsDir() {
		return cdir, fmt.Errorf("file %s already exists and is not a directory", cdir)
	}
	if err := os.MkdirAll(cdir, 0755); err != nil {
		return cdir, err
	}

	cf := filepath.Join(cdir, ChartfileName)
	if _, err := os.Stat(cf); err != nil {
		if err := SaveChartfile(cf, chartfile); err != nil {
			return cdir, err
		}
	}

	for _, d := range []string{TemplatesDir, TemplatesTestsDir, ChartsDir} {
		if err := os.MkdirAll(filepath.Join(cdir, d), 0755); err != nil {
			return cdir, err
		}
	}

	files := []struct {
		path    string
		content []byte
	}{
		{
			// values.yaml
			path:    filepath.Join(cdir, ValuesfileName),
			content: []byte(fmt.Sprintf(defaultValues, chartfile.Name)),
		},
		{
			// .helmignore
			path:    filepath.Join(cdir, IgnorefileName),
			content: []byte(defaultIgnore),
		},
		{
			// ingress.yaml
			path:    filepath.Join(cdir, TemplatesDir, IngressFileName),
			content: Transform(defaultIngress, "<CHARTNAME>", chartfile.Name),
		},
		{
			// deployment.yaml
			path:    filepath.Join(cdir, TemplatesDir, DeploymentName),
			content: Transform(defaultDeployment, "<CHARTNAME>", chartfile.Name),
		},
		{
			// service.yaml
			path:    filepath.Join(cdir, TemplatesDir, ServiceName),
			content: Transform(defaultService, "<CHARTNAME>", chartfile.Name),
		},
		{
			// NOTES.txt
			path:    filepath.Join(cdir, TemplatesDir, NotesName),
			content: Transform(defaultNotes, "<CHARTNAME>", chartfile.Name),
		},
		{
			// _helpers.tpl
			path:    filepath.Join(cdir, TemplatesDir, HelpersName),
			content: Transform(defaultHelpers, "<CHARTNAME>", chartfile.Name),
		},
		{
			// test-connection.yaml
			path:    filepath.Join(cdir, TemplatesTestsDir, TestConnectionName),
			content: Transform(defaultTestConnection, "<CHARTNAME>", chartfile.Name),
		},
	}

	for _, file := range files {
		if _, err := os.Stat(file.path); err == nil {
			// File exists and is okay. Skip it.
			continue
		}
		if err := ioutil.WriteFile(file.path, file.content, 0644); err != nil {
			return cdir, err
		}
	}
	return cdir, nil
}
<|endoftext|>


<commit_before>package dbg

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/sirupsen/logrus"
)

const (
	path     = "/var/cores/"
	fnameFmt = "2006-01-02T15:04:05.999999-0700MST"
)

// DumpGoMemoryTrace output memory profile to logs.
func DumpGoMemoryTrace() {
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	res := fmt.Sprintf("%#v", m)
	logrus.Infof("==== Dumping Memory Profile ===")
	logrus.Infof(res)
}

// DumpGoProfile output goroutines to file.
func DumpGoProfile() error {
	trace := make([]byte, 5120*1024)
	len := runtime.Stack(trace, true)
	return ioutil.WriteFile(path+time.Now().Format(fnameFmt)+".stack", trace[:len], 0644)
}

func DumpHeap() {
	f, err := os.Create(path + time.Now().Format(fnameFmt) + ".heap")
	if err != nil {
		logrus.Errorf("could not create memory profile: %v", err)
		return
	}
	defer f.Close()
	if err := pprof.WriteHeapProfile(f); err != nil {
		logrus.Errorf("could not write memory profile: %v", err)
	}
}
<commit_msg>PWX-18523: Change profile stack & heap names to include the hostname … (#1823)<commit_after>package dbg

import (
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/sirupsen/logrus"
)

const (
	path     = "/var/cores/"
	fnameFmt = "2006-01-02T15:04:05.999999-0700MST"
)

// GetTimeStamp returns 'readable' timestamp with no spaces 'YYYYMMDDHHMMSS'
func GetTimeStamp() string {
	tnow := time.Now()
	return fmt.Sprintf("%d%02d%02d%02d%02d%02d",
		tnow.Year(), tnow.Month(), tnow.Day(),
		tnow.Hour(), tnow.Minute(), tnow.Second())
}

// GetHostNamePrefix return '<hostname>' to use as prefix.
func GetHostNamePrefix() string {
	hname, _ := os.Hostname()
	return hname
}

// DumpGoMemoryTrace output memory profile to logs.
func DumpGoMemoryTrace() {
	m := &runtime.MemStats{}
	runtime.ReadMemStats(m)
	res := fmt.Sprintf("%#v", m)
	logrus.Infof("==== Dumping Memory Profile ===")
	logrus.Infof(res)
}

// DumpGoProfile output goroutines to file.
func DumpGoProfile() error {
	trace := make([]byte, 5120*1024)
	len := runtime.Stack(trace, true)
	return ioutil.WriteFile(path+GetHostNamePrefix()+"-"+GetTimeStamp()+".stack", trace[:len], 0644)
}

func DumpHeap() {
	f, err := os.Create(path + GetHostNamePrefix() + "-" + GetTimeStamp() + ".heap")
	if err != nil {
		logrus.Errorf("could not create memory profile: %v", err)
		return
	}
	defer f.Close()
	if err := pprof.WriteHeapProfile(f); err != nil {
		logrus.Errorf("could not write memory profile: %v", err)
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"github.com/tilezen/tapalcatl/pkg/cache"

	"github.com/tilezen/tapalcatl/pkg/buffer"
	"github.com/tilezen/tapalcatl/pkg/log"
	"github.com/tilezen/tapalcatl/pkg/metrics"
	"github.com/tilezen/tapalcatl/pkg/state"
	"github.com/tilezen/tapalcatl/pkg/storage"
	"github.com/tilezen/tapalcatl/pkg/tile"
)

const (
	// cacheTimeout is the amount of time to wait for tile cache to do it's job before timing out.
	cacheTimeout = 100 * time.Millisecond
)

func MetatileHandler(
	p state.Parser,
	metatileSize, tileSize, metatileMaxDetailZoom int,
	stg storage.Storage,
	bufferManager buffer.BufferManager,
	mw metrics.MetricsWriter,
	logger log.JsonLogger,
	tileCache cache.Cache) http.Handler {

	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		reqState := &state.RequestState{}

		startTime := time.Now()

		defer func() {
			totalDuration := time.Since(startTime)
			reqState.Duration.Total = totalDuration

			if reqState.ResponseState == state.ResponseState_Nil {
				logger.Error(log.LogCategory_InvalidCodeState, "handler did not set response state for tile %+v", reqState.Coord)
			}

			jsonReqData := reqState.AsJsonMap()
			logger.Metrics(jsonReqData)

			// write out metrics
			mw.WriteMetatileState(reqState)

		}()

		parseStart := time.Now()
		parseResult, err := p.Parse(req)
		reqState.Duration.Parse = time.Since(parseStart)
		if err != nil {
			var sc int
			var response string

			if pe, ok := err.(*ParseError); ok {
				logger.Warning(log.LogCategory_ParseError, err.Error())
				if pe.MimeError != nil {
					sc = http.StatusNotFound
					reqState.ResponseState = state.ResponseState_NotFound
					response = pe.MimeError.Error()
				} else if pe.CoordError != nil {
					sc = http.StatusBadRequest
					reqState.ResponseState = state.ResponseState_BadRequest
					response = pe.CoordError.Error()
				} else if pe.CondError != nil {
					reqState.IsCondError = true
					logger.Warning(log.LogCategory_ConditionError, pe.CondError.Error())
				}
			} else {
				logger.Error(log.LogCategory_ParseError, "Unknown parse error: %#v\n", err)
				sc = http.StatusInternalServerError
				response = "Internal server error"
				reqState.ResponseState = state.ResponseState_Error
			}

			// only return an error response when not a condition parse error
			// NOTE: maybe it's better to not consider this an error, but
			// capture it in the parse result state and handle it that way?
			if sc > 0 {
				http.Error(rw, response, sc)
				return
			}
		}

		metatileData := parseResult.AdditionalData.(*state.MetatileParseData)
		reqState.Coord = &metatileData.Coord
		reqState.Format = reqState.Coord.Format
		reqState.HttpData = parseResult.HttpData

		if tileCache == nil {
			tileCache = cache.NilCache
		}

		// Check for requested vector tile in cache before doing work to extract it from metatile
		vecCacheLookupStart := time.Now()
		timeoutCtx, cancel := context.WithTimeout(req.Context(), cacheTimeout)
		cached, err := tileCache.GetTile(timeoutCtx, parseResult)
		cancel()
		reqState.Duration.VectorCacheLookup = time.Since(vecCacheLookupStart)
		if err != nil {
			reqState.IsCacheLookupError = true
			logger.Warning(log.LogCategory_ResponseError, "Error checking cache: %+v", err)
		}

		if cached != nil {
			err := writeVectorTileResponse(reqState, rw, cached)
			if err != nil {
				logger.Error(log.LogCategory_ResponseError, "Failed to write cached response body: %#v", err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				reqState.ResponseState = state.ResponseState_Error
				return
			}

			reqState.Cache.VectorCacheHit = true
			reqState.ResponseState = state.ResponseState_Success
			return
		}

		metatileResponseData, err := fetchMetatile(reqState, stg, bufferManager, parseResult, metatileSize, tileSize, metatileMaxDetailZoom)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.ResponseState = state.ResponseState_Error
			return
		}

		if metatileResponseData.ResponseState == state.ResponseState_NotFound {
			http.NotFound(rw, req)
			reqState.ResponseState = state.ResponseState_NotFound
			return
		} else if metatileResponseData.ResponseState == state.ResponseState_NotModified {
			rw.WriteHeader(http.StatusNotModified)
			reqState.ResponseState = state.ResponseState_NotModified
			return
		}

		responseData, err := extractVectorTileFromMetatile(reqState, bufferManager, parseResult, metatileResponseData)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.ResponseState = state.ResponseState_Error
			return
		}

		// Copy some of the metatile response data over to the vector tile response data so that it is properly cached
		responseData.ETag = metatileResponseData.ETag
		responseData.LastModified = metatileResponseData.LastModified

		err = writeVectorTileResponse(reqState, rw, responseData)
		if err != nil {
			// TODO Context cancellation might happen here?
			logger.Error(log.LogCategory_ResponseError, "Failed to write response body: %#v", err)
			// Still want to set the cache in this case
		}

		// Cache the response
		// TODO Do this in a goroutine so the handler can exit faster?
		cacheSetStart := time.Now()
		timeoutCtx, cancel = context.WithTimeout(req.Context(), cacheTimeout)
		err = tileCache.SetTile(timeoutCtx, parseResult, responseData)
		cancel()
		reqState.Duration.CacheSet = time.Since(cacheSetStart)
		if err != nil {
			logger.Error(log.LogCategory_ResponseError, "Failed to set cache: %#v", err)
		}
	})
}

func fetchMetatile(reqState *state.RequestState, stg storage.Storage, bufferManager buffer.BufferManager, parseResult *state.ParseResult, metatileSize int, tileSize int, metatileMaxDetailZoom int) (*state.MetatileResponseData, error) {
	metatileData := parseResult.AdditionalData.(*state.MetatileParseData)
	responseData := &state.MetatileResponseData{}

	// Get the offset coordinate inside the metatile where we should be able to find the vector tile
	metaCoord, offset, err := metatileData.Coord.MetaAndOffset(metatileSize, tileSize, metatileMaxDetailZoom)
	if err != nil {
		// Note: FetchState is left as nil, since no fetch was performed
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("metaAndOffset could not be calculated: %w", err)
	}

	responseData.Offset = offset

	// Fetch the metatile zip file from storage
	storageFetchStart := time.Now()
	storageResult, err := stg.Fetch(metaCoord, parseResult.Cond, parseResult.BuildID)
	reqState.Duration.StorageFetch = time.Since(storageFetchStart)

	if err != nil || storageResult.NotFound {
		if err != nil {
			reqState.FetchState = state.FetchState_FetchError
			reqState.ResponseState = state.ResponseState_Error
			responseData.ResponseState = state.ResponseState_Error
			return responseData, fmt.Errorf("metatile storage fetch failure: %w", err)
		}

		reqState.FetchState = state.FetchState_NotFound
		reqState.ResponseState = state.ResponseState_NotFound
		responseData.ResponseState = state.ResponseState_NotFound
		return responseData, nil
	}

	reqState.Cache.MetatileCacheHit = storageResult.FetchCacheHit
	reqState.FetchState = state.FetchState_Success

	if storageResult.NotModified {
		reqState.ResponseState = state.ResponseState_NotModified
		responseData.ResponseState = state.ResponseState_NotModified
		return responseData, nil
	}

	// Copy the last-modified and etag headers from the metatile over to the vector tile
	if lastMod := storageResult.Response.LastModified; lastMod != nil {
		responseData.LastModified = lastMod
		reqState.StorageMetadata.HasLastModified = true
	}

	if etag := storageResult.Response.ETag; etag != nil {
		responseData.ETag = etag
		reqState.StorageMetadata.HasLastModified = true
	}

	storageResp := storageResult.Response
	reqState.FetchState = state.FetchState_Success

	storageBytes := storageResp.Body
	reqState.FetchSize.BodySize = int64(storageResp.Size)
	reqState.FetchSize.BytesLength = int64(len(storageBytes))
	reqState.FetchSize.BytesCap = int64(cap(storageBytes))

	responseData.Data = storageBytes
	responseData.BodySize = int64(len(storageBytes))

	return responseData, nil
}

func extractVectorTileFromMetatile(reqState *state.RequestState, bufferManager buffer.BufferManager, parseResult *state.ParseResult, data *state.MetatileResponseData) (*state.VectorTileResponseData, error) {
	responseData := &state.VectorTileResponseData{}
	responseData.ContentType = parseResult.ContentType

	// Set up the metatile reader to read the vector tile out of the metatile
	metatileReaderFindStart := time.Now()
	reader, formatSize, err := tile.NewMetatileReader(data.Offset, bytes.NewReader(data.Data), data.BodySize)
	reqState.Duration.MetatileFind = time.Since(metatileReaderFindStart)
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to read metatile: %w", err)
	}

	// Copy the bytes of the vector tile from the metatile into another buffer
	tileBuf := bufferManager.Get()
	defer bufferManager.Put(tileBuf)
	_, err = io.Copy(tileBuf, reader)
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to read tile out of metatile: %w", err)
	}

	err = reader.Close()
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to close vector tile reader: %w", err)
	}

	reqState.ResponseSize = int(formatSize)
	responseData.Data = tileBuf.Bytes()

	return responseData, nil
}

func writeVectorTileResponse(reqState *state.RequestState, rw http.ResponseWriter, vectorData *state.VectorTileResponseData) error {
	headers := rw.Header()

	headers.Set("Content-Type", vectorData.ContentType)
	headers.Set("Content-Length", fmt.Sprintf("%d", len(vectorData.Data)))

	if lastMod := vectorData.LastModified; lastMod != nil {
		// It's important to write the last-modified header in an HTTP-compliant way.
		// Go exposes http.TimeFormat for that, but hard-codes "GMT" at the end, though,
		// so we need to make sure we convert the time to UTC before formatting.
		lastModifiedFormatted := lastMod.UTC().Format(http.TimeFormat)
		headers.Set("Last-Modified", lastModifiedFormatted)
		reqState.StorageMetadata.HasLastModified = true
	}

	if etag := vectorData.ETag; etag != nil {
		headers.Set("ETag", *etag)
		reqState.StorageMetadata.HasEtag = true
	}

	rw.WriteHeader(http.StatusOK)
	reqState.ResponseState = state.ResponseState_Success
	respWriteStart := time.Now()
	_, err := rw.Write(vectorData.Data)
	reqState.Duration.RespWrite = time.Since(respWriteStart)
	if err != nil {
		reqState.IsResponseWriteError = true
		return fmt.Errorf("failed to write response body: %w", err)
	}

	return nil
}

type MetatileMuxParser struct {
	MimeMap map[string]string
}

func (mp *MetatileMuxParser) Parse(req *http.Request) (*state.ParseResult, error) {
	m := mux.Vars(req)

	var contentType string
	var err error
	var ok bool

	parseResult := &state.ParseResult{
		Type:     state.ParseResultType_Metatile,
		HttpData: ParseHttpData(req),
	}
	metatileData := &state.MetatileParseData{}
	parseResult.AdditionalData = metatileData

	fmt := m["fmt"]
	if contentType, ok = mp.MimeMap[fmt]; !ok {
		return parseResult, &ParseError{
			MimeError: &MimeParseError{
				BadFormat: fmt,
			},
		}
	}
	parseResult.ContentType = contentType
	t := &metatileData.Coord
	t.Format = fmt

	parseResult.BuildID = req.URL.Query().Get("buildid")

	var coordError CoordParseError
	z := m["z"]
	t.Z, err = strconv.Atoi(z)
	if err != nil {
		coordError.BadZ = z
	}

	x := m["x"]
	t.X, err = strconv.Atoi(x)
	if err != nil {
		coordError.BadX = x
	}

	y := m["y"]
	t.Y, err = strconv.Atoi(y)
	if err != nil {
		coordError.BadY = y
	}

	if coordError.IsError() {
		return parseResult, &ParseError{
			CoordError: &coordError,
		}
	}
	var condErr *CondParseError
	parseResult.Cond, condErr = ParseCondition(req)
	if condErr != nil {
		return parseResult, &ParseError{CondError: condErr}
	}

	return parseResult, nil
}
<commit_msg>Tighten up cache timeout<commit_after>package handler

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"github.com/tilezen/tapalcatl/pkg/cache"

	"github.com/tilezen/tapalcatl/pkg/buffer"
	"github.com/tilezen/tapalcatl/pkg/log"
	"github.com/tilezen/tapalcatl/pkg/metrics"
	"github.com/tilezen/tapalcatl/pkg/state"
	"github.com/tilezen/tapalcatl/pkg/storage"
	"github.com/tilezen/tapalcatl/pkg/tile"
)

const (
	// cacheTimeout is the amount of time to wait for tile cache to do it's job before timing out.
	cacheTimeout = 20 * time.Millisecond
)

func MetatileHandler(
	p state.Parser,
	metatileSize, tileSize, metatileMaxDetailZoom int,
	stg storage.Storage,
	bufferManager buffer.BufferManager,
	mw metrics.MetricsWriter,
	logger log.JsonLogger,
	tileCache cache.Cache) http.Handler {

	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		reqState := &state.RequestState{}

		startTime := time.Now()

		defer func() {
			totalDuration := time.Since(startTime)
			reqState.Duration.Total = totalDuration

			if reqState.ResponseState == state.ResponseState_Nil {
				logger.Error(log.LogCategory_InvalidCodeState, "handler did not set response state for tile %+v", reqState.Coord)
			}

			jsonReqData := reqState.AsJsonMap()
			logger.Metrics(jsonReqData)

			// write out metrics
			mw.WriteMetatileState(reqState)

		}()

		parseStart := time.Now()
		parseResult, err := p.Parse(req)
		reqState.Duration.Parse = time.Since(parseStart)
		if err != nil {
			var sc int
			var response string

			if pe, ok := err.(*ParseError); ok {
				logger.Warning(log.LogCategory_ParseError, err.Error())
				if pe.MimeError != nil {
					sc = http.StatusNotFound
					reqState.ResponseState = state.ResponseState_NotFound
					response = pe.MimeError.Error()
				} else if pe.CoordError != nil {
					sc = http.StatusBadRequest
					reqState.ResponseState = state.ResponseState_BadRequest
					response = pe.CoordError.Error()
				} else if pe.CondError != nil {
					reqState.IsCondError = true
					logger.Warning(log.LogCategory_ConditionError, pe.CondError.Error())
				}
			} else {
				logger.Error(log.LogCategory_ParseError, "Unknown parse error: %#v\n", err)
				sc = http.StatusInternalServerError
				response = "Internal server error"
				reqState.ResponseState = state.ResponseState_Error
			}

			// only return an error response when not a condition parse error
			// NOTE: maybe it's better to not consider this an error, but
			// capture it in the parse result state and handle it that way?
			if sc > 0 {
				http.Error(rw, response, sc)
				return
			}
		}

		metatileData := parseResult.AdditionalData.(*state.MetatileParseData)
		reqState.Coord = &metatileData.Coord
		reqState.Format = reqState.Coord.Format
		reqState.HttpData = parseResult.HttpData

		if tileCache == nil {
			tileCache = cache.NilCache
		}

		// Check for requested vector tile in cache before doing work to extract it from metatile
		vecCacheLookupStart := time.Now()
		timeoutCtx, cancel := context.WithTimeout(req.Context(), cacheTimeout)
		cached, err := tileCache.GetTile(timeoutCtx, parseResult)
		cancel()
		reqState.Duration.VectorCacheLookup = time.Since(vecCacheLookupStart)
		if err != nil {
			reqState.IsCacheLookupError = true
			logger.Warning(log.LogCategory_ResponseError, "Error checking cache: %+v", err)
		}

		if cached != nil {
			err := writeVectorTileResponse(reqState, rw, cached)
			if err != nil {
				logger.Error(log.LogCategory_ResponseError, "Failed to write cached response body: %#v", err)
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				reqState.ResponseState = state.ResponseState_Error
				return
			}

			reqState.Cache.VectorCacheHit = true
			reqState.ResponseState = state.ResponseState_Success
			return
		}

		metatileResponseData, err := fetchMetatile(reqState, stg, bufferManager, parseResult, metatileSize, tileSize, metatileMaxDetailZoom)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.ResponseState = state.ResponseState_Error
			return
		}

		if metatileResponseData.ResponseState == state.ResponseState_NotFound {
			http.NotFound(rw, req)
			reqState.ResponseState = state.ResponseState_NotFound
			return
		} else if metatileResponseData.ResponseState == state.ResponseState_NotModified {
			rw.WriteHeader(http.StatusNotModified)
			reqState.ResponseState = state.ResponseState_NotModified
			return
		}

		responseData, err := extractVectorTileFromMetatile(reqState, bufferManager, parseResult, metatileResponseData)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			reqState.ResponseState = state.ResponseState_Error
			return
		}

		// Copy some of the metatile response data over to the vector tile response data so that it is properly cached
		responseData.ETag = metatileResponseData.ETag
		responseData.LastModified = metatileResponseData.LastModified

		err = writeVectorTileResponse(reqState, rw, responseData)
		if err != nil {
			// TODO Context cancellation might happen here?
			logger.Error(log.LogCategory_ResponseError, "Failed to write response body: %#v", err)
			// Still want to set the cache in this case
		}

		// Cache the response
		// TODO Do this in a goroutine so the handler can exit faster?
		cacheSetStart := time.Now()
		timeoutCtx, cancel = context.WithTimeout(req.Context(), cacheTimeout)
		err = tileCache.SetTile(timeoutCtx, parseResult, responseData)
		cancel()
		reqState.Duration.CacheSet = time.Since(cacheSetStart)
		if err != nil {
			logger.Error(log.LogCategory_ResponseError, "Failed to set cache: %#v", err)
		}
	})
}

func fetchMetatile(reqState *state.RequestState, stg storage.Storage, bufferManager buffer.BufferManager, parseResult *state.ParseResult, metatileSize int, tileSize int, metatileMaxDetailZoom int) (*state.MetatileResponseData, error) {
	metatileData := parseResult.AdditionalData.(*state.MetatileParseData)
	responseData := &state.MetatileResponseData{}

	// Get the offset coordinate inside the metatile where we should be able to find the vector tile
	metaCoord, offset, err := metatileData.Coord.MetaAndOffset(metatileSize, tileSize, metatileMaxDetailZoom)
	if err != nil {
		// Note: FetchState is left as nil, since no fetch was performed
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("metaAndOffset could not be calculated: %w", err)
	}

	responseData.Offset = offset

	// Fetch the metatile zip file from storage
	storageFetchStart := time.Now()
	storageResult, err := stg.Fetch(metaCoord, parseResult.Cond, parseResult.BuildID)
	reqState.Duration.StorageFetch = time.Since(storageFetchStart)

	if err != nil || storageResult.NotFound {
		if err != nil {
			reqState.FetchState = state.FetchState_FetchError
			reqState.ResponseState = state.ResponseState_Error
			responseData.ResponseState = state.ResponseState_Error
			return responseData, fmt.Errorf("metatile storage fetch failure: %w", err)
		}

		reqState.FetchState = state.FetchState_NotFound
		reqState.ResponseState = state.ResponseState_NotFound
		responseData.ResponseState = state.ResponseState_NotFound
		return responseData, nil
	}

	reqState.Cache.MetatileCacheHit = storageResult.FetchCacheHit
	reqState.FetchState = state.FetchState_Success

	if storageResult.NotModified {
		reqState.ResponseState = state.ResponseState_NotModified
		responseData.ResponseState = state.ResponseState_NotModified
		return responseData, nil
	}

	// Copy the last-modified and etag headers from the metatile over to the vector tile
	if lastMod := storageResult.Response.LastModified; lastMod != nil {
		responseData.LastModified = lastMod
		reqState.StorageMetadata.HasLastModified = true
	}

	if etag := storageResult.Response.ETag; etag != nil {
		responseData.ETag = etag
		reqState.StorageMetadata.HasLastModified = true
	}

	storageResp := storageResult.Response
	reqState.FetchState = state.FetchState_Success

	storageBytes := storageResp.Body
	reqState.FetchSize.BodySize = int64(storageResp.Size)
	reqState.FetchSize.BytesLength = int64(len(storageBytes))
	reqState.FetchSize.BytesCap = int64(cap(storageBytes))

	responseData.Data = storageBytes
	responseData.BodySize = int64(len(storageBytes))

	return responseData, nil
}

func extractVectorTileFromMetatile(reqState *state.RequestState, bufferManager buffer.BufferManager, parseResult *state.ParseResult, data *state.MetatileResponseData) (*state.VectorTileResponseData, error) {
	responseData := &state.VectorTileResponseData{}
	responseData.ContentType = parseResult.ContentType

	// Set up the metatile reader to read the vector tile out of the metatile
	metatileReaderFindStart := time.Now()
	reader, formatSize, err := tile.NewMetatileReader(data.Offset, bytes.NewReader(data.Data), data.BodySize)
	reqState.Duration.MetatileFind = time.Since(metatileReaderFindStart)
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to read metatile: %w", err)
	}

	// Copy the bytes of the vector tile from the metatile into another buffer
	tileBuf := bufferManager.Get()
	defer bufferManager.Put(tileBuf)
	_, err = io.Copy(tileBuf, reader)
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to read tile out of metatile: %w", err)
	}

	err = reader.Close()
	if err != nil {
		reqState.IsZipError = true
		reqState.ResponseState = state.ResponseState_Error
		responseData.ResponseState = state.ResponseState_Error
		return responseData, fmt.Errorf("failed to close vector tile reader: %w", err)
	}

	reqState.ResponseSize = int(formatSize)
	responseData.Data = tileBuf.Bytes()

	return responseData, nil
}

func writeVectorTileResponse(reqState *state.RequestState, rw http.ResponseWriter, vectorData *state.VectorTileResponseData) error {
	headers := rw.Header()

	headers.Set("Content-Type", vectorData.ContentType)
	headers.Set("Content-Length", fmt.Sprintf("%d", len(vectorData.Data)))

	if lastMod := vectorData.LastModified; lastMod != nil {
		// It's important to write the last-modified header in an HTTP-compliant way.
		// Go exposes http.TimeFormat for that, but hard-codes "GMT" at the end, though,
		// so we need to make sure we convert the time to UTC before formatting.
		lastModifiedFormatted := lastMod.UTC().Format(http.TimeFormat)
		headers.Set("Last-Modified", lastModifiedFormatted)
		reqState.StorageMetadata.HasLastModified = true
	}

	if etag := vectorData.ETag; etag != nil {
		headers.Set("ETag", *etag)
		reqState.StorageMetadata.HasEtag = true
	}

	rw.WriteHeader(http.StatusOK)
	reqState.ResponseState = state.ResponseState_Success
	respWriteStart := time.Now()
	_, err := rw.Write(vectorData.Data)
	reqState.Duration.RespWrite = time.Since(respWriteStart)
	if err != nil {
		reqState.IsResponseWriteError = true
		return fmt.Errorf("failed to write response body: %w", err)
	}

	return nil
}

type MetatileMuxParser struct {
	MimeMap map[string]string
}

func (mp *MetatileMuxParser) Parse(req *http.Request) (*state.ParseResult, error) {
	m := mux.Vars(req)

	var contentType string
	var err error
	var ok bool

	parseResult := &state.ParseResult{
		Type:     state.ParseResultType_Metatile,
		HttpData: ParseHttpData(req),
	}
	metatileData := &state.MetatileParseData{}
	parseResult.AdditionalData = metatileData

	fmt := m["fmt"]
	if contentType, ok = mp.MimeMap[fmt]; !ok {
		return parseResult, &ParseError{
			MimeError: &MimeParseError{
				BadFormat: fmt,
			},
		}
	}
	parseResult.ContentType = contentType
	t := &metatileData.Coord
	t.Format = fmt

	parseResult.BuildID = req.URL.Query().Get("buildid")

	var coordError CoordParseError
	z := m["z"]
	t.Z, err = strconv.Atoi(z)
	if err != nil {
		coordError.BadZ = z
	}

	x := m["x"]
	t.X, err = strconv.Atoi(x)
	if err != nil {
		coordError.BadX = x
	}

	y := m["y"]
	t.Y, err = strconv.Atoi(y)
	if err != nil {
		coordError.BadY = y
	}

	if coordError.IsError() {
		return parseResult, &ParseError{
			CoordError: &coordError,
		}
	}
	var condErr *CondParseError
	parseResult.Cond, condErr = ParseCondition(req)
	if condErr != nil {
		return parseResult, &ParseError{CondError: condErr}
	}

	return parseResult, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/mitchellh/go-homedir"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"k8s.io/cloud-provider-openstack/pkg/ingress/config"
	"k8s.io/cloud-provider-openstack/pkg/ingress/controller"
	"k8s.io/klog/v2"
)

var (
	cfgFile string
	isDebug bool
	conf    config.Config
)

// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
	Use:   "ingress-openstack",
	Short: "Ingress controller for OpenStack",
	Long:  `Ingress controller for OpenStack`,

	Run: func(cmd *cobra.Command, args []string) {
		osIngress := controller.NewController(conf)
		osIngress.Start()

		sigterm := make(chan os.Signal, 1)
		signal.Notify(sigterm, syscall.SIGTERM)
		signal.Notify(sigterm, syscall.SIGINT)
		<-sigterm
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	log.SetOutput(os.Stdout)

	cobra.OnInitialize(initConfig)

	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ingress_openstack.yaml)")
	rootCmd.PersistentFlags().BoolVar(&isDebug, "debug", false, "Print more detailed information.")

	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	rootCmd.PersistentFlags().AddGoFlagSet(klogFlags)
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// Search config in home directory with name ".ingress_openstack" (without extension).
		viper.AddConfigPath(home)
		viper.SetConfigName(".ingress_openstack")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err != nil {
		log.WithFields(log.Fields{"error": err}).Fatal("Failed to read config file")
	}

	log.WithFields(log.Fields{"file": viper.ConfigFileUsed()}).Info("Using config file")

	if err := viper.Unmarshal(&conf); err != nil {
		log.WithFields(log.Fields{"error": err}).Fatal("Unable to decode the configuration")
	}
	if conf.ClusterName == "" {
		log.Fatal("clusterName configuration is required")
	}

	if isDebug {
		log.SetLevel(log.DebugLevel)
	}
}
<commit_msg>[octavia-ingress-controller] Set log formatter (#1427)<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"syscall"

	"github.com/mitchellh/go-homedir"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"k8s.io/cloud-provider-openstack/pkg/ingress/config"
	"k8s.io/cloud-provider-openstack/pkg/ingress/controller"
	"k8s.io/klog/v2"
)

var (
	cfgFile string
	isDebug bool
	conf    config.Config
)

// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
	Use:   "ingress-openstack",
	Short: "Ingress controller for OpenStack",
	Long:  `Ingress controller for OpenStack`,

	Run: func(cmd *cobra.Command, args []string) {
		osIngress := controller.NewController(conf)
		osIngress.Start()

		sigterm := make(chan os.Signal, 1)
		signal.Notify(sigterm, syscall.SIGTERM)
		signal.Notify(sigterm, syscall.SIGINT)
		<-sigterm
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	log.SetOutput(os.Stdout)
	log.SetFormatter(&log.TextFormatter{
		ForceColors:            true,
		FullTimestamp:          true,
		DisableLevelTruncation: true,
		PadLevelText:           true,
	})

	cobra.OnInitialize(initConfig)

	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.ingress_openstack.yaml)")
	rootCmd.PersistentFlags().BoolVar(&isDebug, "debug", false, "Print more detailed information.")

	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)
	rootCmd.PersistentFlags().AddGoFlagSet(klogFlags)
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// Search config in home directory with name ".ingress_openstack" (without extension).
		viper.AddConfigPath(home)
		viper.SetConfigName(".ingress_openstack")
	}

	viper.AutomaticEnv() // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err != nil {
		log.WithFields(log.Fields{"error": err}).Fatal("Failed to read config file")
	}

	log.WithFields(log.Fields{"file": viper.ConfigFileUsed()}).Info("Using config file")

	if err := viper.Unmarshal(&conf); err != nil {
		log.WithFields(log.Fields{"error": err}).Fatal("Unable to decode the configuration")
	}
	if conf.ClusterName == "" {
		log.Fatal("clusterName configuration is required")
	}

	if isDebug {
		log.SetLevel(log.DebugLevel)
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/stats"
)

type Stats struct {
	statsClient *stats.StatsClient
}

func NewStats(statsClient *stats.StatsClient) *Stats {
	return &Stats{statsClient}
}

func (m *Stats) Handler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Debug("Starting Stats middleware")

		timing := m.statsClient.StatsDClient.NewTiming()

		// reverse proxy replaces original request with target request, so keep required fields of the original one
		originalURL := &url.URL{}
		*originalURL = *r.URL
		originalRequest := &http.Request{Method: r.Method, URL: originalURL}

		handler.ServeHTTP(w, r)

		m.statsClient.TrackRequest(timing, originalRequest)
	})
}
<commit_msg>More debugging for stats stuff<commit_after>package middleware

import (
	"net/http"
	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/stats"
)

type Stats struct {
	statsClient *stats.StatsClient
}

func NewStats(statsClient *stats.StatsClient) *Stats {
	return &Stats{statsClient}
}

func (m *Stats) Handler(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.WithField("path", r.URL.Path).Debug("Starting Stats middleware")

		timing := m.statsClient.StatsDClient.NewTiming()

		// reverse proxy replaces original request with target request, so keep required fields of the original one
		originalURL := &url.URL{}
		*originalURL = *r.URL
		originalRequest := &http.Request{Method: r.Method, URL: originalURL}

		handler.ServeHTTP(w, r)

		log.WithFields(log.Fields{"original_path": originalURL.Path, "request_url": r.URL.Path}).Debug("Track request stats")
		m.statsClient.TrackRequest(timing, originalRequest)
	})
}
<|endoftext|>