<commit_before>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"log"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0005   // Minumum change required to update prices
	TRADENUM  = 10       // Number of trades to use in calculations
	AMOUNT    = 0.50     // Size to trade
	BIDEDGE   = 0.02     // Required edge for a buy order
	ASKEDGE   = 0.02     // Required edge for a sell order
)

var (
	api = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
)

func main() {
	fmt.Println("\nConnecting...")

	// Create channels
	bookChan := make(chan bitfinex.Book)
	tradesChan := make(chan bitfinex.Trades)
	bidChan := make(chan bitfinex.Order)
	askChan := make(chan bitfinex.Order)
	inputChan := make(chan rune)

	// Initial orders
	bid, ask := createOrders()

	// Check for input to break loop
	go checkStdin(inputChan)

	var (
		trades bitfinex.Trades
		book   bitfinex.Book
		start  time.Time
		theo   float64
	)

loop:
	for {
		start = time.Now()

		// Get data in separate goroutines
		go processBook(bookChan)
		go processTrades(tradesChan)

		// Modify orders in separate goroutines when trade data returns
		trades = <-tradesChan
		theo = calculateTheo(trades)
		go replaceBid(bid, bidChan, theo)
		go replaceAsk(ask, askChan, theo)

		// Print data and current orders when all communication is finished
		bid = <-bidChan
		ask = <-askChan
		book = <-bookChan
		printResults(book, trades, bid, ask)

		// Print processing time
		fmt.Printf("\n%v processing time...", time.Since(start))

		// Exit if anything entered by user
		select {
		case <-inputChan:
			cancelAll()
			break loop
		default:
		}
	}
}

// Create initial orders
func createOrders() (bitfinex.Order, bitfinex.Order) {
	// Get the current price
	trades, err := api.Trades(SYMBOL, 1)
	checkErr(err)
	price := trades[0].Price

	// Order parameters
	params := []bitfinex.OrderParams{
		{SYMBOL, AMOUNT, price - BIDEDGE, "bitfinex", "buy", "limit"},
		{SYMBOL, AMOUNT, price + ASKEDGE, "bitfinex", "sell", "limit"},
	}

	// Send new order request to the exchange
	orders, err := api.MultipleNewOrders(params)
	checkErr(err)

	return orders.Orders[0], orders.Orders[1]
}

func checkStdin(inputChan chan rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades(tradesChan chan<- bitfinex.Trades) {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	tradesChan <- trades
}

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Modify bid order and send to channel
func replaceBid(bid bitfinex.Order, bidChan chan<- bitfinex.Order, theo float64) {
	price := theo - BIDEDGE
	var err error
	if math.Abs(price-bid.Price) >= MINCHANGE {
		count := 0
		for id := 0; id == 0; id = bid.ID {
			fmt.Printf("Attempting to replace bid order %d", bid.ID)
			bid, err = api.ReplaceOrder(bid.ID, SYMBOL, AMOUNT, price, "bitfinex", "buy", "limit")
			checkErr(err)
			count++
			if count >= 5 {
				cancelAll()
				log.Fatalf("Could not replace bid order %d", bid.ID)
			}
		}
	}

	bidChan <- bid
}

// Modify ask order and send to channel
func replaceAsk(ask bitfinex.Order, askChan chan<- bitfinex.Order, theo float64) {
	price := theo + ASKEDGE

	var err error
	if math.Abs(price-ask.Price) >= MINCHANGE {
		count := 0
		for id := 0; id == 0; id = ask.ID {
			fmt.Printf("Attempting to replace ask order %d", ask.ID)
			ask, err = api.ReplaceOrder(ask.ID, SYMBOL, AMOUNT, price, "bitfinex", "sell", "limit")
			checkErr(err)
			count++
			if count >= 5 {
				cancelAll()
				log.Fatalf("Could not replace ask order %d", ask.ID)
			}
		}
	}

	askChan <- ask
}

// Print results
func printResults(book bitfinex.Book, trades bitfinex.Trades, bid, ask bitfinex.Order) {
	clearScreen()

	fmt.Println("----------------------------")
	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	fmt.Println("----------------------------")
	for i := range book.Asks {
		item := book.Asks[len(book.Asks)-1-i]
		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	}
	for _, item := range book.Bids {
		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	}
	fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nCurrent Bid: %6.4f\n", bid.Price)
	fmt.Printf("Current Ask: %6.4f\n", ask.Price)
}

// Exit on any errors
func checkErr(err error) {
	if err != nil {
		cancelAll()
		log.Fatal(err)
	}
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	fmt.Println("\nALL ORDERS HAVE BEEN CANCELLED")
}
<commit_msg>change order id check<commit_after>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"log"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0001   // Minumum change required to update prices
	TRADENUM  = 10       // Number of trades to use in calculations
	AMOUNT    = 0.50     // Size to trade
	BIDEDGE   = 0.02     // Required edge for a buy order
	ASKEDGE   = 0.02     // Required edge for a sell order
)

var (
	api = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
)

func main() {
	fmt.Println("\nConnecting...")

	// Create channels
	bookChan := make(chan bitfinex.Book)
	tradesChan := make(chan bitfinex.Trades)
	bidChan := make(chan bitfinex.Order)
	askChan := make(chan bitfinex.Order)
	inputChan := make(chan rune)

	// Initial orders
	bid, ask := createOrders()

	// Check for input to break loop
	go checkStdin(inputChan)

	var (
		trades bitfinex.Trades
		book   bitfinex.Book
		start  time.Time
		theo   float64
	)

loop:
	for {
		start = time.Now()

		// Get data in separate goroutines
		go processBook(bookChan)
		go processTrades(tradesChan)

		// Modify orders in separate goroutines when trade data returns
		trades = <-tradesChan
		theo = calculateTheo(trades)
		go replaceBid(bid, bidChan, theo)
		go replaceAsk(ask, askChan, theo)

		// Print data and current orders when all communication is finished
		bid = <-bidChan
		ask = <-askChan
		book = <-bookChan
		printResults(book, trades, bid, ask)

		// Print processing time
		fmt.Printf("\n%v processing time...", time.Since(start))

		// Exit if anything entered by user
		select {
		case <-inputChan:
			cancelAll()
			break loop
		default:
		}
	}
}

// Create initial orders
func createOrders() (bitfinex.Order, bitfinex.Order) {
	// Get the current price
	trades, err := api.Trades(SYMBOL, 1)
	checkErr(err)
	price := trades[0].Price

	// Order parameters
	params := []bitfinex.OrderParams{
		{SYMBOL, AMOUNT, price - BIDEDGE, "bitfinex", "buy", "limit"},
		{SYMBOL, AMOUNT, price + ASKEDGE, "bitfinex", "sell", "limit"},
	}

	// Send new order request to the exchange
	orders, err := api.MultipleNewOrders(params)
	checkErr(err)

	return orders.Orders[0], orders.Orders[1]
}

func checkStdin(inputChan chan rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades(tradesChan chan<- bitfinex.Trades) {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	tradesChan <- trades
}

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Modify bid order and send to channel
func replaceBid(bid bitfinex.Order, bidChan chan<- bitfinex.Order, theo float64) {
	price := theo - BIDEDGE

	if math.Abs(price-bid.Price) >= MINCHANGE {
		bid2, err := api.ReplaceOrder(bid.ID, SYMBOL, AMOUNT, price, "bitfinex", "buy", "limit")
		checkErr(err)
		if bid2.ID != 0 {
			bid = bid2
		}
	}

	bidChan <- bid
}

// Modify ask order and send to channel
func replaceAsk(ask bitfinex.Order, askChan chan<- bitfinex.Order, theo float64) {
	price := theo + ASKEDGE

	if math.Abs(price-ask.Price) >= MINCHANGE {
		ask2, err := api.ReplaceOrder(ask.ID, SYMBOL, AMOUNT, price, "bitfinex", "sell", "limit")
		checkErr(err)
		if ask2.ID != 0 {
			ask = ask2
		}
	}

	askChan <- ask
}

// Print results
func printResults(book bitfinex.Book, trades bitfinex.Trades, bid, ask bitfinex.Order) {
	clearScreen()

	fmt.Println("----------------------------")
	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	fmt.Println("----------------------------")
	for i := range book.Asks {
		item := book.Asks[len(book.Asks)-1-i]
		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	}
	for _, item := range book.Bids {
		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	}
	fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nCurrent Bid: %6.4f\n", bid.Price)
	fmt.Printf("Current Ask: %6.4f\n", ask.Price)
}

// Exit on any errors
func checkErr(err error) {
	if err != nil {
		cancelAll()
		log.Fatal(err)
	}
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	fmt.Println("\nALL ORDERS HAVE BEEN CANCELLED")
}
<|endoftext|>


<commit_before>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0001   // Minumum change required to update prices
	TRADENUM  = 20       // Number of trades to use in calculations
	MAXO      = 150      // Max order size
	MINO      = 0.011    // Min order size
	INEDGE    = 0.05     // Required entry edge
	OUTEDGE   = 0.01     // Required exit edge
)

var (
	api        = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
	apiErrors  = false
	liveOrders = false
	orderTheo  = 0.0
)

func main() {
	fmt.Println("\nInitializing...")

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	// Exchange communication channels
	bookChan := make(chan bitfinex.Book)
	tradesChan := make(chan bitfinex.Trades)

	var (
		trades      bitfinex.Trades
		book        bitfinex.Book
		orders      bitfinex.Orders
		start       time.Time
		oldPosition float64
		newPosition float64
		theo        float64
		lastTrade   int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default:
		}

		// Get data in separate goroutines
		go processTrades(tradesChan)
		go processBook(bookChan)

		// Possibly send orders when trades data returns
		trades = <-tradesChan
		if !apiErrors && trades[0].TID != lastTrade { // If new trades
			theo = calculateTheo(trades)
			newPosition = checkPosition()
			if (math.Abs(theo-orderTheo) >= MINCHANGE || math.Abs(oldPosition-
				newPosition) >= MINO || !liveOrders) && !apiErrors {
				orders = sendOrders(theo, newPosition)
			}
		}

		// Print results when book and order data returns
		book = <-bookChan
		if !apiErrors {
			printResults(book, trades, orders, theo, newPosition, start)
			// Reset for next iteration
			oldPosition = newPosition
			lastTrade = trades[0].TID
		}

		// Reset for next iteration
		apiErrors = false
	}
}

// Send orders to the exchange
func sendOrders(theo, position float64) bitfinex.Orders {
	orderTheo = theo

	if liveOrders {
		cancelAll()
	}

	// Send new order request to the exchange
	params := calcOrderParams(position, theo)
	orders, err := api.MultipleNewOrders(params)
	liveOrders = true
	checkErr(err)
	return orders
}

func calcOrderParams(position, theo float64) []bitfinex.OrderParams {
	var params []bitfinex.OrderParams

	if math.Abs(position) < MINO { // No position
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, MAXO, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	} else if position < (-1*MAXO)+MINO { // Max short postion
		params = []bitfinex.OrderParams{
			{SYMBOL, -1 * position, theo - OUTEDGE, "bitfinex", "buy", "limit"},
		}
	} else if position > MAXO-MINO { // Max long postion
		params = []bitfinex.OrderParams{
			{SYMBOL, position, theo + OUTEDGE, "bitfinex", "sell", "limit"},
		}
	} else if (-1*MAXO)+MINO <= position && position <= -1*MINO { // Partial short
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, -1 * position, theo - OUTEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, MAXO + position, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	} else if MINO <= position && position <= MAXO-MINO { // Partial long
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO - position, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, position, theo + OUTEDGE, "bitfinex", "sell", "limit"},
			{SYMBOL, MAXO, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	}

	return params
}

func checkPosition() float64 {
	var position float64
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	return position
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades(tradesChan chan<- bitfinex.Trades) {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	tradesChan <- trades
}

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		cancelAll()
		apiErrors = true
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	liveOrders = false
}

// Print results
func printResults(book bitfinex.Book, trades bitfinex.Trades,
	orders bitfinex.Orders, theo, position float64, start time.Time) {

	clearScreen()

	fmt.Println("----------------------------")
	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	fmt.Println("----------------------------")
	for i := range book.Asks {
		item := book.Asks[len(book.Asks)-1-i]
		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	}
	for _, item := range book.Bids {
		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	}
	fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%8.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<commit_msg>remove viewing book data for running remotely<commit_after>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"math"
	"os"
	"os/exec"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0001   // Minumum change required to update prices
	TRADENUM  = 20       // Number of trades to use in calculations
	MAXO      = 150      // Max order size
	MINO      = 0.011    // Min order size
	INEDGE    = 0.05     // Required entry edge
	OUTEDGE   = 0.01     // Required exit edge
)

var (
	api        = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
	apiErrors  = false
	liveOrders = false
	orderTheo  = 0.0
)

func main() {
	fmt.Println("\nInitializing...")

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	// Exchange communication channels
	// bookChan := make(chan bitfinex.Book)
	// tradesChan := make(chan bitfinex.Trades)

	var (
		trades bitfinex.Trades
		// book        bitfinex.Book
		orders      bitfinex.Orders
		start       time.Time
		oldPosition float64
		newPosition float64
		theo        float64
		lastTrade   int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default:
		}

		// // Get data in separate goroutines
		// go processTrades(tradesChan)
		// // go processBook(bookChan)
		//
		// // Possibly send orders when trades data returns
		// trades = <-tradesChan

		trades = processTrades()
		if !apiErrors && trades[0].TID != lastTrade { // If new trades
			theo = calculateTheo(trades)
			newPosition = checkPosition()
			if (math.Abs(theo-orderTheo) >= MINCHANGE || math.Abs(oldPosition-
				newPosition) >= MINO || !liveOrders) && !apiErrors {
				orders = sendOrders(theo, newPosition)
			}
		}

		// Print results when book and order data returns
		// book = <-bookChan
		if !apiErrors {
			// printResults(book, trades, orders, theo, newPosition, start)
			printResults(trades, orders, theo, newPosition, start)
			// Reset for next iteration
			oldPosition = newPosition
			lastTrade = trades[0].TID
		}

		// Reset for next iteration
		apiErrors = false
	}
}

// Send orders to the exchange
func sendOrders(theo, position float64) bitfinex.Orders {
	orderTheo = theo

	if liveOrders {
		cancelAll()
	}

	// Send new order request to the exchange
	params := calcOrderParams(position, theo)
	orders, err := api.MultipleNewOrders(params)
	liveOrders = true
	checkErr(err)
	return orders
}

func calcOrderParams(position, theo float64) []bitfinex.OrderParams {
	var params []bitfinex.OrderParams

	if math.Abs(position) < MINO { // No position
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, MAXO, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	} else if position < (-1*MAXO)+MINO { // Max short postion
		params = []bitfinex.OrderParams{
			{SYMBOL, -1 * position, theo - OUTEDGE, "bitfinex", "buy", "limit"},
		}
	} else if position > MAXO-MINO { // Max long postion
		params = []bitfinex.OrderParams{
			{SYMBOL, position, theo + OUTEDGE, "bitfinex", "sell", "limit"},
		}
	} else if (-1*MAXO)+MINO <= position && position <= -1*MINO { // Partial short
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, -1 * position, theo - OUTEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, MAXO + position, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	} else if MINO <= position && position <= MAXO-MINO { // Partial long
		params = []bitfinex.OrderParams{
			{SYMBOL, MAXO - position, theo - INEDGE, "bitfinex", "buy", "limit"},
			{SYMBOL, position, theo + OUTEDGE, "bitfinex", "sell", "limit"},
			{SYMBOL, MAXO, theo + INEDGE, "bitfinex", "sell", "limit"},
		}
	}

	return params
}

func checkPosition() float64 {
	var position float64
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	return position
}

// Get book data and send to channel
func processBook(bookChan chan<- bitfinex.Book) {
	book, err := api.Orderbook(SYMBOL, 5, 5)
	checkErr(err)

	bookChan <- book
}

// Get trade data and send to channel
func processTrades() bitfinex.Trades {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	return trades
}

//
// // Get trade data and send to channel
// func processTrades(tradesChan chan<- bitfinex.Trades) {
// 	trades, err := api.Trades(SYMBOL, TRADENUM)
// 	checkErr(err)
//
// 	tradesChan <- trades
// }

// Calculate a volume-weighted moving average of trades
func calculateTheo(trades bitfinex.Trades) float64 {
	var sum1, sum2 float64
	for _, trade := range trades {
		sum1 += trade.Price * trade.Amount
		sum2 += trade.Amount
	}
	return sum1 / sum2
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		cancelAll()
		apiErrors = true
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	liveOrders = false
}

// Print results
func printResults(trades bitfinex.Trades,
	orders bitfinex.Orders, theo, position float64, start time.Time) {

	clearScreen()

	// fmt.Println("----------------------------")
	// fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
	// fmt.Println("----------------------------")
	// for i := range book.Asks {
	// 	item := book.Asks[len(book.Asks)-1-i]
	// 	fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
	// }
	// for _, item := range book.Bids {
	// 	fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
	// }
	// fmt.Println("----------------------------")

	fmt.Println("\nLast Trades:")
	for _, trade := range trades {
		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
	}

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%8.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// // Print results
// func printResults(book bitfinex.Book, trades bitfinex.Trades,
// 	orders bitfinex.Orders, theo, position float64, start time.Time) {
//
// 	clearScreen()
//
// 	fmt.Println("----------------------------")
// 	fmt.Printf("%-10s%-10s%8s\n", " Bid", "  Ask", "Size ")
// 	fmt.Println("----------------------------")
// 	for i := range book.Asks {
// 		item := book.Asks[len(book.Asks)-1-i]
// 		fmt.Printf("%-10s%-10.4f%8.2f\n", "", item.Price, item.Amount)
// 	}
// 	for _, item := range book.Bids {
// 		fmt.Printf("%-10.4f%-10.2s%8.2f\n", item.Price, "", item.Amount)
// 	}
// 	fmt.Println("----------------------------")
//
// 	fmt.Println("\nLast Trades:")
// 	for _, trade := range trades {
// 		fmt.Printf("%-6.4f - size: %6.2f\n", trade.Price, trade.Amount)
// 	}
//
// 	fmt.Printf("\nPosition: %.2f\n", position)
// 	fmt.Printf("Theo:     %.4f\n", theo)
//
// 	fmt.Println("\nActive orders:")
// 	for _, order := range orders.Orders {
// 		fmt.Printf("%8.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
// 	}
//
// 	fmt.Printf("\n%v processing time...", time.Since(start))
// }
//
// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<|endoftext|>


<commit_before>package revok

import (
	"cred-alert/db"
	"cred-alert/revokpb"
	"sort"

	"code.cloudfoundry.org/lager"

	"golang.org/x/net/context"
)

//go:generate bash $GOPATH/scripts/generate_protos.sh

//go:generate go-bindata -o web/bindata.go -ignore bindata -pkg web web/templates/...

//go:generate counterfeiter . Server

type Server interface {
	revokpb.RevokServer
}

type server struct {
	logger lager.Logger
	db     db.RepositoryRepository
}

func NewServer(logger lager.Logger, db db.RepositoryRepository) Server {
	return &server{
		logger: logger,
		db:     db,
	}
}

func (s *server) GetCredentialCounts(
	ctx context.Context,
	in *revokpb.CredentialCountRequest,
) (*revokpb.CredentialCountResponse, error) {
	logger := s.logger.Session("get-organization-credential-counts")

	repositories, err := s.db.All()
	if err != nil {
		logger.Error("failed-getting-repositories-from-db", err)
		return nil, err
	}

	orgCounts := map[string]float64{}
	for i := range repositories {
		for _, branchCountInt := range repositories[i].CredentialCounts {
			if branchCount, ok := branchCountInt.(float64); ok {
				orgCounts[repositories[i].Owner] += branchCount
			}
		}
	}

	orgNames := []string{}
	for name := range orgCounts {
		orgNames = append(orgNames, name)
	}
	sort.Strings(orgNames)

	response := &revokpb.CredentialCountResponse{}
	for _, orgName := range orgNames {
		occ := &revokpb.OrganizationCredentialCount{
			Owner: orgName,
			Count: int64(orgCounts[orgName]),
		}
		response.CredentialCounts = append(response.CredentialCounts, occ)
	}

	return response, nil
}

func (s *server) GetOrganizationCredentialCounts(
	ctx context.Context,
	in *revokpb.OrganizationCredentialCountRequest,
) (*revokpb.OrganizationCredentialCountResponse, error) {
	logger := s.logger.Session("get-repository-credential-counts")

	repositories, err := s.db.AllForOrganization(in.Owner)
	if err != nil {
		logger.Error("failed-getting-repositories-from-db", err)
		return nil, err
	}

	rccs := []*revokpb.RepositoryCredentialCount{}
	for i := range repositories {
		var count int64
		for _, branchCountInt := range repositories[i].CredentialCounts {
			if branchCount, ok := branchCountInt.(float64); ok {
				count += int64(branchCount)
			}
		}

		rccs = append(rccs, &revokpb.RepositoryCredentialCount{
			Owner: repositories[i].Owner,
			Name:  repositories[i].Name,
			Count: count,
		})
	}

	sort.Sort(revokpb.RCCByName(rccs))

	response := &revokpb.OrganizationCredentialCountResponse{
		CredentialCounts: rccs,
	}

	return response, nil
}

func (s *server) GetRepositoryCredentialCounts(
	ctx context.Context,
	in *revokpb.RepositoryCredentialCountRequest,
) (*revokpb.RepositoryCredentialCountResponse, error) {
	logger := s.logger.Session("get-repository-credential-counts")

	repository, err := s.db.Find(in.Owner, in.Name)
	if err != nil {
		logger.Error("failed-getting-repository-from-db", err)
		return nil, err
	}

	bccs := []*revokpb.BranchCredentialCount{}
	for branch, countInt := range repository.CredentialCounts {
		if count, ok := countInt.(float64); ok {
			bccs = append(bccs, &revokpb.BranchCredentialCount{
				Name:  branch,
				Count: int64(count),
			})
		}
	}

	sort.Sort(revokpb.BCCByName(bccs))

	response := &revokpb.RepositoryCredentialCountResponse{
		CredentialCounts: bccs,
	}

	return response, nil
}

func (s *server) Search(*revokpb.SearchQuery, revokpb.Revok_SearchServer) error {
	panic("not implemented")
}
<commit_msg>add fake search results<commit_after>package revok

import (
	"cred-alert/db"
	"cred-alert/revokpb"
	"fmt"
	"sort"
	"time"

	"code.cloudfoundry.org/lager"

	"golang.org/x/net/context"
)

//go:generate bash $GOPATH/scripts/generate_protos.sh

//go:generate go-bindata -o web/bindata.go -ignore bindata -pkg web web/templates/...

//go:generate counterfeiter . Server

type Server interface {
	revokpb.RevokServer
}

type server struct {
	logger lager.Logger
	db     db.RepositoryRepository
}

func NewServer(logger lager.Logger, db db.RepositoryRepository) Server {
	return &server{
		logger: logger,
		db:     db,
	}
}

func (s *server) GetCredentialCounts(
	ctx context.Context,
	in *revokpb.CredentialCountRequest,
) (*revokpb.CredentialCountResponse, error) {
	logger := s.logger.Session("get-organization-credential-counts")

	repositories, err := s.db.All()
	if err != nil {
		logger.Error("failed-getting-repositories-from-db", err)
		return nil, err
	}

	orgCounts := map[string]float64{}
	for i := range repositories {
		for _, branchCountInt := range repositories[i].CredentialCounts {
			if branchCount, ok := branchCountInt.(float64); ok {
				orgCounts[repositories[i].Owner] += branchCount
			}
		}
	}

	orgNames := []string{}
	for name := range orgCounts {
		orgNames = append(orgNames, name)
	}
	sort.Strings(orgNames)

	response := &revokpb.CredentialCountResponse{}
	for _, orgName := range orgNames {
		occ := &revokpb.OrganizationCredentialCount{
			Owner: orgName,
			Count: int64(orgCounts[orgName]),
		}
		response.CredentialCounts = append(response.CredentialCounts, occ)
	}

	return response, nil
}

func (s *server) GetOrganizationCredentialCounts(
	ctx context.Context,
	in *revokpb.OrganizationCredentialCountRequest,
) (*revokpb.OrganizationCredentialCountResponse, error) {
	logger := s.logger.Session("get-repository-credential-counts")

	repositories, err := s.db.AllForOrganization(in.Owner)
	if err != nil {
		logger.Error("failed-getting-repositories-from-db", err)
		return nil, err
	}

	rccs := []*revokpb.RepositoryCredentialCount{}
	for i := range repositories {
		var count int64
		for _, branchCountInt := range repositories[i].CredentialCounts {
			if branchCount, ok := branchCountInt.(float64); ok {
				count += int64(branchCount)
			}
		}

		rccs = append(rccs, &revokpb.RepositoryCredentialCount{
			Owner: repositories[i].Owner,
			Name:  repositories[i].Name,
			Count: count,
		})
	}

	sort.Sort(revokpb.RCCByName(rccs))

	response := &revokpb.OrganizationCredentialCountResponse{
		CredentialCounts: rccs,
	}

	return response, nil
}

func (s *server) GetRepositoryCredentialCounts(
	ctx context.Context,
	in *revokpb.RepositoryCredentialCountRequest,
) (*revokpb.RepositoryCredentialCountResponse, error) {
	logger := s.logger.Session("get-repository-credential-counts")

	repository, err := s.db.Find(in.Owner, in.Name)
	if err != nil {
		logger.Error("failed-getting-repository-from-db", err)
		return nil, err
	}

	bccs := []*revokpb.BranchCredentialCount{}
	for branch, countInt := range repository.CredentialCounts {
		if count, ok := countInt.(float64); ok {
			bccs = append(bccs, &revokpb.BranchCredentialCount{
				Name:  branch,
				Count: int64(count),
			})
		}
	}

	sort.Sort(revokpb.BCCByName(bccs))

	response := &revokpb.RepositoryCredentialCountResponse{
		CredentialCounts: bccs,
	}

	return response, nil
}

func (s *server) Search(query *revokpb.SearchQuery, stream revokpb.Revok_SearchServer) error {
	for i := 0; i < 42; i++ {
		searchResult := &revokpb.SearchResult{
			Location: &revokpb.SourceLocation{
				Repository: &revokpb.Repository{
					Owner: "pivotal-cf",
					Name:  fmt.Sprintf("repository-%d", i),
				},
				Revision:   "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc",
				Path:       "my/special/file.go",
				LineNumber: uint32(i),
				Location:   uint32(i * 2),
				Length:     uint32(i * 3),
			},
			Content: fmt.Sprintf("My Special Content %d", i),
		}

		if err := stream.Send(searchResult); err != nil {
			return err
		}

		time.Sleep(200 * time.Millisecond)
	}

	return nil
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package poll

import (
	"fmt"
	"strings"
	"time"

	"github.com/ava-labs/avalanchego/utils/linkedhashmap"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/utils/metric"
)

type pollHolder interface {
	GetPoll() Poll
	StartTime() time.Time
}

type poll struct {
	Poll
	start time.Time
}

func (p poll) GetPoll() Poll {
	return p
}

func (p poll) StartTime() time.Time {
	return p.start
}

type set struct {
	log      logging.Logger
	numPolls prometheus.Gauge
	durPolls prometheus.Histogram
	factory  Factory
	// maps requestID -> poll
	polls linkedhashmap.LinkedHashmap
}

// NewSet returns a new empty set of polls
func NewSet(
	factory Factory,
	log logging.Logger,
	namespace string,
	registerer prometheus.Registerer,
) Set {
	numPolls := prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: namespace,
		Name:      "polls",
		Help:      "Number of pending network polls",
	})
	if err := registerer.Register(numPolls); err != nil {
		log.Error("failed to register polls statistics due to %s", err)
	}

	durPolls := prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: namespace,
		Name:      "poll_duration",
		Help:      "Length of time the poll existed in milliseconds",
		Buckets:   metric.MillisecondsBuckets,
	})
	if err := registerer.Register(durPolls); err != nil {
		log.Error("failed to register poll_duration statistics due to %s", err)
	}

	return &set{
		log:      log,
		numPolls: numPolls,
		durPolls: durPolls,
		factory:  factory,
		polls:    linkedhashmap.New(),
	}
}

// Add to the current set of polls
// Returns true if the poll was registered correctly and the network sample
//         should be made.
func (s *set) Add(requestID uint32, vdrs ids.ShortBag) bool {
	if _, exists := s.polls.Get(requestID); exists {
		s.log.Debug("dropping poll due to duplicated requestID: %d", requestID)
		return false
	}

	s.log.Verbo("creating poll with requestID %d and validators %s",
		requestID,
		&vdrs)

	s.polls.Put(requestID, poll{
		Poll:  s.factory.New(vdrs), // create the new poll
		start: time.Now(),
	})
	s.numPolls.Inc() // increase the metrics
	return true
}

// Vote registers the connections response to a query for [id]. If there was no
// query, or the response has already be registered, nothing is performed.
func (s *set) Vote(requestID uint32, vdr ids.ShortID, vote ids.ID) []ids.Bag {
	pollHolderIntf, exists := s.polls.Get(requestID)
	if !exists {
		s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d",
			vdr,
			requestID)
		return nil
	}

	holder := pollHolderIntf.(pollHolder)
	p := holder.GetPoll()

	s.log.Verbo("processing vote from %s in the poll with requestID: %d with the vote %s",
		vdr,
		requestID,
		vote)

	p.Vote(vdr, vote)
	if !p.Finished() {
		return nil
	}

	s.log.Verbo("poll with requestID %d finished as %s", requestID, p)
	s.durPolls.Observe(float64(time.Since(holder.StartTime()).Milliseconds()))
	s.numPolls.Dec() // decrease the metrics

	return s.processFinishedPolls(requestID)
}

// processFinishedPolls checks for other finished polls and returns them all if finished
func (s *set) processFinishedPolls(requestID uint32) []ids.Bag {
	var results []ids.Bag
	// If this is not the oldest poll, return as is.
	if oldestRequestID, _, _ := s.polls.Oldest(); oldestRequestID != requestID {
		return nil
	}

	// this is the oldest poll that has just finished
	// iterate from oldest to newest
	iter := s.polls.NewIterator()
	for iter.Next() {
		holder := iter.Value().(pollHolder)
		p := holder.GetPoll()
		if !p.Finished() {
			// since we're iterating from oldest to newest, if the next poll has not finished,
			// we can break and return what we have so far
			break
		}

		results = append(results, p.Result())
		s.polls.Delete(iter.Key())
	}

	// only gets here if the poll has finished
	// results will have values if this and other newer polls have finished
	return results
}

// Drop registers the connections response to a query for [id]. If there was no
// query, or the response has already be registered, nothing is performed.
func (s *set) Drop(requestID uint32, vdr ids.ShortID) []ids.Bag {
	pollHolderIntf, exists := s.polls.Get(requestID)
	if !exists {
		s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d",
			vdr,
			requestID)
		return nil
	}

	s.log.Verbo("processing dropped vote from %s in the poll with requestID: %d",
		vdr,
		requestID)

	pollHolder := pollHolderIntf.(pollHolder)
	poll := pollHolder.GetPoll()

	poll.Drop(vdr)
	if !poll.Finished() {
		return nil
	}

	s.log.Verbo("poll with requestID %d finished as %s", requestID, poll)

	s.durPolls.Observe(float64(time.Since(pollHolder.StartTime()).Milliseconds()))
	s.numPolls.Dec() // decrease the metrics
	return s.processFinishedPolls(requestID)
}

// Len returns the number of outstanding polls
func (s *set) Len() int { return s.polls.Len() }

func (s *set) String() string {
	sb := strings.Builder{}
	sb.WriteString(fmt.Sprintf("current polls: (Size = %d)", s.polls.Len()))
	iter := s.polls.NewIterator()
	for iter.Next() {
		requestID := iter.Key()
		poll := iter.Value().(Poll)
		sb.WriteString(fmt.Sprintf("\n    %d: %s", requestID, poll.PrefixedString("    ")))
	}
	return sb.String()
}
<commit_msg>remove redundant check<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package poll

import (
	"fmt"
	"strings"
	"time"

	"github.com/ava-labs/avalanchego/utils/linkedhashmap"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils/logging"
	"github.com/ava-labs/avalanchego/utils/metric"
)

type pollHolder interface {
	GetPoll() Poll
	StartTime() time.Time
}

type poll struct {
	Poll
	start time.Time
}

func (p poll) GetPoll() Poll {
	return p
}

func (p poll) StartTime() time.Time {
	return p.start
}

type set struct {
	log      logging.Logger
	numPolls prometheus.Gauge
	durPolls prometheus.Histogram
	factory  Factory
	// maps requestID -> poll
	polls linkedhashmap.LinkedHashmap
}

// NewSet returns a new empty set of polls
func NewSet(
	factory Factory,
	log logging.Logger,
	namespace string,
	registerer prometheus.Registerer,
) Set {
	numPolls := prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: namespace,
		Name:      "polls",
		Help:      "Number of pending network polls",
	})
	if err := registerer.Register(numPolls); err != nil {
		log.Error("failed to register polls statistics due to %s", err)
	}

	durPolls := prometheus.NewHistogram(prometheus.HistogramOpts{
		Namespace: namespace,
		Name:      "poll_duration",
		Help:      "Length of time the poll existed in milliseconds",
		Buckets:   metric.MillisecondsBuckets,
	})
	if err := registerer.Register(durPolls); err != nil {
		log.Error("failed to register poll_duration statistics due to %s", err)
	}

	return &set{
		log:      log,
		numPolls: numPolls,
		durPolls: durPolls,
		factory:  factory,
		polls:    linkedhashmap.New(),
	}
}

// Add to the current set of polls
// Returns true if the poll was registered correctly and the network sample
//         should be made.
func (s *set) Add(requestID uint32, vdrs ids.ShortBag) bool {
	if _, exists := s.polls.Get(requestID); exists {
		s.log.Debug("dropping poll due to duplicated requestID: %d", requestID)
		return false
	}

	s.log.Verbo("creating poll with requestID %d and validators %s",
		requestID,
		&vdrs)

	s.polls.Put(requestID, poll{
		Poll:  s.factory.New(vdrs), // create the new poll
		start: time.Now(),
	})
	s.numPolls.Inc() // increase the metrics
	return true
}

// Vote registers the connections response to a query for [id]. If there was no
// query, or the response has already be registered, nothing is performed.
func (s *set) Vote(requestID uint32, vdr ids.ShortID, vote ids.ID) []ids.Bag {
	pollHolderIntf, exists := s.polls.Get(requestID)
	if !exists {
		s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d",
			vdr,
			requestID)
		return nil
	}

	holder := pollHolderIntf.(pollHolder)
	p := holder.GetPoll()

	s.log.Verbo("processing vote from %s in the poll with requestID: %d with the vote %s",
		vdr,
		requestID,
		vote)

	p.Vote(vdr, vote)
	if !p.Finished() {
		return nil
	}

	s.log.Verbo("poll with requestID %d finished as %s", requestID, p)
	s.durPolls.Observe(float64(time.Since(holder.StartTime()).Milliseconds()))
	s.numPolls.Dec() // decrease the metrics

	return s.processFinishedPolls(requestID)
}

// processFinishedPolls checks for other finished polls and returns them all if finished
func (s *set) processFinishedPolls(requestID uint32) []ids.Bag {
	var results []ids.Bag

	// iterate from oldest to newest
	iter := s.polls.NewIterator()
	for iter.Next() {
		holder := iter.Value().(pollHolder)
		p := holder.GetPoll()
		if !p.Finished() {
			// since we're iterating from oldest to newest, if the next poll has not finished,
			// we can break and return what we have so far
			break
		}

		results = append(results, p.Result())
		s.polls.Delete(iter.Key())
	}

	// only gets here if the poll has finished
	// results will have values if this and other newer polls have finished
	return results
}

// Drop registers the connections response to a query for [id]. If there was no
// query, or the response has already be registered, nothing is performed.
func (s *set) Drop(requestID uint32, vdr ids.ShortID) []ids.Bag {
	pollHolderIntf, exists := s.polls.Get(requestID)
	if !exists {
		s.log.Verbo("dropping vote from %s to an unknown poll with requestID: %d",
			vdr,
			requestID)
		return nil
	}

	s.log.Verbo("processing dropped vote from %s in the poll with requestID: %d",
		vdr,
		requestID)

	pollHolder := pollHolderIntf.(pollHolder)
	poll := pollHolder.GetPoll()

	poll.Drop(vdr)
	if !poll.Finished() {
		return nil
	}

	s.log.Verbo("poll with requestID %d finished as %s", requestID, poll)

	s.durPolls.Observe(float64(time.Since(pollHolder.StartTime()).Milliseconds()))
	s.numPolls.Dec() // decrease the metrics
	return s.processFinishedPolls(requestID)
}

// Len returns the number of outstanding polls
func (s *set) Len() int { return s.polls.Len() }

func (s *set) String() string {
	sb := strings.Builder{}
	sb.WriteString(fmt.Sprintf("current polls: (Size = %d)", s.polls.Len()))
	iter := s.polls.NewIterator()
	for iter.Next() {
		requestID := iter.Key()
		poll := iter.Value().(Poll)
		sb.WriteString(fmt.Sprintf("\n    %d: %s", requestID, poll.PrefixedString("    ")))
	}
	return sb.String()
}
<|endoftext|>


<commit_before>package repository

import (
	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/model"
)

var hatenaBookmarkNotFoundError = model.NotFoundError("hatenaBookmark")

func (r *repository) UpdateHatenaBookmark(e *model.Example) error {
	if e.HatenaBookmark == nil {
		return nil
	}

	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id
	if _, err = r.db.Exec(`DELETE FROM hatena_bookmark WHERE example_id = $1;`, id); err != nil {
		return err
	}

	if _, err = r.db.NamedExec(`
INSERT INTO hatena_bookmark
( example_id,  title,  screenshot,  entry_url,  count,  url,  eid)
VALUES
(:example_id, :title, :screenshot, :entry_url, :count, :url, :eid)
;`, e.HatenaBookmark); err != nil {
		return err
	}

	hb := model.HatenaBookmark{}
	if err = r.db.Get(&hb, `SELECT id FROM hatena_bookmark WHERE example_id = $1;`, id); err != nil {
		return err
	}

	for _, b := range e.HatenaBookmark.Bookmarks {
		b.HatenaBookmarkId = hb.Id
		if _, err = r.db.NamedExec(`
INSERT INTO bookmark
(hatena_bookmark_id, "user", comment, timestamp, tags)
VALUES
(:hatena_bookmark_id, :user, :comment, :timestamp, :tags)
;`, b); err != nil {
			return err
		}
	}
	return nil
}

func (r *repository) SearchHatenaBookmarks(examples model.Examples) ([]*model.HatenaBookmark, error) {
	hatenaBookmarks := make([]*model.HatenaBookmark, 0)
	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	query := `SELECT * FROM hatena_bookmark WHERE example_id = ANY($1);`
	err := r.db.Select(&hatenaBookmarks, query, pq.Array(exampleIds))
	if err != nil {
		return hatenaBookmarks, err
	}

	hatenaBookmarkIds := make([]int, 0)
	for _, hb := range hatenaBookmarks {
		hatenaBookmarkIds = append(hatenaBookmarkIds, hb.Id)
	}
	bookmarks := make([]*model.Bookmark, 0)
	query = `SELECT * FROM bookmark WHERE hatena_bookmark_id = ANY($1);`
	err = r.db.Select(&bookmarks, query, pq.Array(hatenaBookmarkIds))
	if err != nil {
		return hatenaBookmarks, err
	}

	bookmarksByHatenaBookmarkId := make(map[int][]*model.Bookmark)
	for _, b := range bookmarks {
		bookmarksByHatenaBookmarkId[b.HatenaBookmarkId] = append(bookmarksByHatenaBookmarkId[b.HatenaBookmarkId], b)
	}

	result := make([]*model.HatenaBookmark, 0)
	for _, hb := range hatenaBookmarks {
		bookmarks := bookmarksByHatenaBookmarkId[hb.Id]
		hb.Bookmarks = bookmarks
		result = append(result, hb)
	}
	return result, nil
}

func (r *repository) FindHatenaBookmark(e *model.Example) (*model.HatenaBookmark, error) {
	hatenaBookmark := &model.HatenaBookmark{}

	query := `SELECT * FROM hatena_bookmark WHERE example_id = $1;`
	err := r.db.Get(hatenaBookmark, query, e.Id)
	if err != nil {
		return hatenaBookmark, err
	}

	hatenaBookmarkId := hatenaBookmark.Id
	bookmarks := make([]*model.Bookmark, 0)
	query = `SELECT * FROM bookmark WHERE hatena_bookmark_id = $1;`
	err = r.db.Select(&bookmarks, query, hatenaBookmarkId)
	if err != nil {
		return hatenaBookmark, err
	}

	hatenaBookmark.Bookmarks = bookmarks
	return hatenaBookmark, nil
}
<commit_msg>idを設定し忘れていた<commit_after>package repository

import (
	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/model"
)

var hatenaBookmarkNotFoundError = model.NotFoundError("hatenaBookmark")

func (r *repository) UpdateHatenaBookmark(e *model.Example) error {
	if e.HatenaBookmark == nil {
		return nil
	}

	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id
	if _, err = r.db.Exec(`DELETE FROM hatena_bookmark WHERE example_id = $1;`, id); err != nil {
		return err
	}

	e.HatenaBookmark.ExampleId = id
	if _, err = r.db.NamedExec(`
INSERT INTO hatena_bookmark
( example_id,  title,  screenshot,  entry_url,  count,  url,  eid)
VALUES
(:example_id, :title, :screenshot, :entry_url, :count, :url, :eid)
;`, e.HatenaBookmark); err != nil {
		return err
	}

	hb := model.HatenaBookmark{}
	if err = r.db.Get(&hb, `SELECT id FROM hatena_bookmark WHERE example_id = $1;`, id); err != nil {
		return err
	}

	for _, b := range e.HatenaBookmark.Bookmarks {
		b.HatenaBookmarkId = hb.Id
		if _, err = r.db.NamedExec(`
INSERT INTO bookmark
(hatena_bookmark_id, "user", comment, timestamp, tags)
VALUES
(:hatena_bookmark_id, :user, :comment, :timestamp, :tags)
;`, b); err != nil {
			return err
		}
	}
	return nil
}

func (r *repository) SearchHatenaBookmarks(examples model.Examples) ([]*model.HatenaBookmark, error) {
	hatenaBookmarks := make([]*model.HatenaBookmark, 0)
	exampleIds := make([]int, 0)
	for _, e := range examples {
		exampleIds = append(exampleIds, e.Id)
	}

	query := `SELECT * FROM hatena_bookmark WHERE example_id = ANY($1);`
	err := r.db.Select(&hatenaBookmarks, query, pq.Array(exampleIds))
	if err != nil {
		return hatenaBookmarks, err
	}

	hatenaBookmarkIds := make([]int, 0)
	for _, hb := range hatenaBookmarks {
		hatenaBookmarkIds = append(hatenaBookmarkIds, hb.Id)
	}
	bookmarks := make([]*model.Bookmark, 0)
	query = `SELECT * FROM bookmark WHERE hatena_bookmark_id = ANY($1);`
	err = r.db.Select(&bookmarks, query, pq.Array(hatenaBookmarkIds))
	if err != nil {
		return hatenaBookmarks, err
	}

	bookmarksByHatenaBookmarkId := make(map[int][]*model.Bookmark)
	for _, b := range bookmarks {
		bookmarksByHatenaBookmarkId[b.HatenaBookmarkId] = append(bookmarksByHatenaBookmarkId[b.HatenaBookmarkId], b)
	}

	result := make([]*model.HatenaBookmark, 0)
	for _, hb := range hatenaBookmarks {
		bookmarks := bookmarksByHatenaBookmarkId[hb.Id]
		hb.Bookmarks = bookmarks
		result = append(result, hb)
	}
	return result, nil
}

func (r *repository) FindHatenaBookmark(e *model.Example) (*model.HatenaBookmark, error) {
	hatenaBookmark := &model.HatenaBookmark{}

	query := `SELECT * FROM hatena_bookmark WHERE example_id = $1;`
	err := r.db.Get(hatenaBookmark, query, e.Id)
	if err != nil {
		return hatenaBookmark, err
	}

	hatenaBookmarkId := hatenaBookmark.Id
	bookmarks := make([]*model.Bookmark, 0)
	query = `SELECT * FROM bookmark WHERE hatena_bookmark_id = $1;`
	err = r.db.Select(&bookmarks, query, hatenaBookmarkId)
	if err != nil {
		return hatenaBookmark, err
	}

	hatenaBookmark.Bookmarks = bookmarks
	return hatenaBookmark, nil
}
<|endoftext|>


<commit_before>package emil

import (
	"errors"
	"fmt"
	"strings"
)

var errNotEmpty = errors.New("not empty")
var errKingsToClose = errors.New("Kings to close")

// Board with an array of field values, representing pieces
type Board struct {
	squares   []int
	whiteKing int
	blackKing int

	str string
}

// NewBoard creates a new board
func NewBoard() *Board {
	return &Board{squares: make([]int, SQUARES)}
}

// Fen2Board creates a new board from a fen string
func Fen2Board(fen string) *Board {
	b := NewBoard()
	b.str = fen

	for fs, part := range strings.Split(fen, "/") {
		fmt.Println(part, ":")
		sq := FirstSquares[fs]
		fmt.Println("Start:", BoardSquares[sq])
		i := 0
		for _, r := range part {
			s := string(r)
			switch s {
			case "1", "8":
				//nothing to do
			case "7":
				i += 6
			case "6":
				i += 5
			case "5":
				i += 4
			case "4":
				i += 3
			case "3":
				i += 2
			case "2":
				i += 1
			default:
				piece, ok := Symbols[s]
				if !ok {
					panic("can't parse " + s + " in " + part)
				} else {
					b.Setup(piece, sq+i)
				}
			}
			i++
		}
		fmt.Println()
		fmt.Println()
		fmt.Println()
	}
	fmt.Println(b.Picture())
	return b
}
func (b *Board) Square(i int) int {
	return b.squares[i]
}
func (b *Board) String() string {
	if len(b.str) > 0 {
		return b.str
	}
	s := ""
	for _, r := range FirstSquares {
		if len(s) > 0 {
			s += "/"
		}
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s", symbol(b.squares[r+f]))
		}
	}
	s = strings.Replace(s, "        ", "8", -1)
	s = strings.Replace(s, "       ", "7", -1)
	s = strings.Replace(s, "      ", "6", -1)
	s = strings.Replace(s, "     ", "5", -1)
	s = strings.Replace(s, "    ", "4", -1)
	s = strings.Replace(s, "   ", "3", -1)
	s = strings.Replace(s, "  ", "2", -1)
	s = strings.Replace(s, " ", "1", -1)
	b.str = s
	return s
}
func (b *Board) Picture() string {
	files := "a b c d e f g h "
	s := "  " + files + " \n"
	for _, r := range FirstSquares {
		s += fmt.Sprintf("%d ", BoardSquares[r].rank)
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s ", symbol(b.squares[r+f]))
		}
		s += fmt.Sprintf("%d\n", BoardSquares[r].rank)
	}
	s += "  " + files + " \n"
	return s
}

//Setup a piece on a square
func (b *Board) Setup(piece, square int) (noError error) {
	if b.squares[square] != Empty {
		return errNotEmpty
	}
	b.squares[square] = piece
	if piece == BlackKing {
		b.blackKing = square
	}
	if piece == WhiteKing {
		b.whiteKing = square
	}
	return noError
}

func (b *Board) kingsToClose() (noError error) {
	if squaresDistances[b.whiteKing][b.blackKing] <= 1 {
		return errKingsToClose
	}
	return noError
}
func (b *Board) DoMove(m *Move) (newBoard *Board) {
	return b.doMove(m)
}

func (b *Board) doMove(m *Move) (newBoard *Board) {
	// if DEBUG {
	// 	fmt.Printf("do move: %s\n", m)
	// }
	newBoard = NewBoard()
	newBoard.whiteKing = b.whiteKing
	newBoard.blackKing = b.blackKing
	copy(newBoard.squares, b.squares)
	newBoard.squares[m.source] = Empty
	newBoard.squares[m.destination] = m.piece
	// if DEBUG {
	// 	fmt.Printf("%s\n", b)
	// }
	return newBoard
}
<commit_msg>remove output<commit_after>package emil

import (
	"errors"
	"fmt"
	"strings"
)

var errNotEmpty = errors.New("not empty")
var errKingsToClose = errors.New("Kings to close")

// Board with an array of field values, representing pieces
type Board struct {
	squares   []int
	whiteKing int
	blackKing int

	str string
}

// NewBoard creates a new board
func NewBoard() *Board {
	return &Board{squares: make([]int, SQUARES)}
}

// Fen2Board creates a new board from a fen string
func Fen2Board(fen string) *Board {
	b := NewBoard()
	b.str = fen

	for fs, part := range strings.Split(fen, "/") {
		sq := FirstSquares[fs]
		i := 0
		for _, r := range part {
			s := string(r)
			switch s {
			case "1", "8":
				//nothing to do
			case "7":
				i += 6
			case "6":
				i += 5
			case "5":
				i += 4
			case "4":
				i += 3
			case "3":
				i += 2
			case "2":
				i += 1
			default:
				piece, ok := Symbols[s]
				if !ok {
					panic("can't parse " + s + " in " + part)
				} else {
					b.Setup(piece, sq+i)
				}
			}
			i++
		}
	}
	return b
}
func (b *Board) Square(i int) int {
	return b.squares[i]
}
func (b *Board) String() string {
	if len(b.str) > 0 {
		return b.str
	}
	s := ""
	for _, r := range FirstSquares {
		if len(s) > 0 {
			s += "/"
		}
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s", symbol(b.squares[r+f]))
		}
	}
	s = strings.Replace(s, "        ", "8", -1)
	s = strings.Replace(s, "       ", "7", -1)
	s = strings.Replace(s, "      ", "6", -1)
	s = strings.Replace(s, "     ", "5", -1)
	s = strings.Replace(s, "    ", "4", -1)
	s = strings.Replace(s, "   ", "3", -1)
	s = strings.Replace(s, "  ", "2", -1)
	s = strings.Replace(s, " ", "1", -1)
	b.str = s
	return s
}
func (b *Board) Picture() string {
	files := "a b c d e f g h "
	s := "  " + files + " \n"
	for _, r := range FirstSquares {
		s += fmt.Sprintf("%d ", BoardSquares[r].rank)
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s ", symbol(b.squares[r+f]))
		}
		s += fmt.Sprintf("%d\n", BoardSquares[r].rank)
	}
	s += "  " + files + " \n"
	return s
}

//Setup a piece on a square
func (b *Board) Setup(piece, square int) (noError error) {
	if b.squares[square] != Empty {
		return errNotEmpty
	}
	b.squares[square] = piece
	if piece == BlackKing {
		b.blackKing = square
	}
	if piece == WhiteKing {
		b.whiteKing = square
	}
	return noError
}

func (b *Board) kingsToClose() (noError error) {
	if squaresDistances[b.whiteKing][b.blackKing] <= 1 {
		return errKingsToClose
	}
	return noError
}
func (b *Board) DoMove(m *Move) (newBoard *Board) {
	return b.doMove(m)
}

func (b *Board) doMove(m *Move) (newBoard *Board) {
	// if DEBUG {
	// 	fmt.Printf("do move: %s\n", m)
	// }
	newBoard = NewBoard()
	newBoard.whiteKing = b.whiteKing
	newBoard.blackKing = b.blackKing
	copy(newBoard.squares, b.squares)
	newBoard.squares[m.source] = Empty
	newBoard.squares[m.destination] = m.piece
	// if DEBUG {
	// 	fmt.Printf("%s\n", b)
	// }
	return newBoard
}
<|endoftext|>


<commit_before>package emil

import (
	"fmt"
)

// Board with an array of field values, representing pieces
type Board struct {
	squares []int
}

// NewBoard creates a new Board
func NewBoard() *Board {
	return &Board{squares: make([]int, SQUARES)}
}
func (b *Board) String() string {
	files := "a b c d e f g h "
	s := "  " + files + " \n"
	for _, r := range FirstSquares {
		s += fmt.Sprintf("%d ", BoardSquares[r].rank)
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s ", symbol(b.squares[r+f]))
		}
		s += fmt.Sprintf("%d\n", BoardSquares[r].rank)
	}
	s += "  " + files + " \n"
	return s
}

func symbol(piece int) string {
	switch piece {
	case WhiteKing:
		return "K"
	case BlackKing:
		return "k"
	case WhiteRock:
		return "R"
	case BlackRock:
		return "r"
	default:
		return " "
	}
}

//Setup a piece on a square
func (b *Board) Setup(piece, square int) {
	b.squares[square] = piece
}

func isOwnPiece(player, capture int) bool {
	return (player == WHITE && capture > 0) ||
		(player == BLACK && capture < 0)
}

func otherPlayer(player int) int {
	if player == WHITE {
		return BLACK
	}
	return WHITE
}

//Moves prints all moves for a piece on square
func (b *Board) Moves(player int, testKingCapture bool) (string, bool) {
	var result, list []*Move
	for src, piece := range b.squares {
		if isOwnPiece(player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := b.squares[dst]
					if isKing(capture) {
						return "", true
					}
					if capture == Empty {
						list = append(list, newSilentMove(player, piece, src, dst))
					} else if !isOwnPiece(player, capture) {
						list = append(list, newCaptureMove(player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := b.squares[dst]
						if isKing(capture) {
							return "", true
						}
						if capture == Empty {
							list = append(list, newSilentMove(player, piece, src, dst))
						} else if !isOwnPiece(player, capture) {
							list = append(list, newCaptureMove(player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}

	if testKingCapture {
		return "no KingCaptured", false
	}

	for _, m := range list {
		println("TEST move", m.String())
		b.doMove(m)
		if !b.isKingCapturedAfter(m) {
			result = append(result, m)
		} else {
			println("KingCaptured")
		}
		b.undoMove(m)
		fmt.Printf("\n\n\n")
	}

	return moveList(result), false
}
func isKing(piece int) bool {
	return abs(piece) == kingValue
}

func (b *Board) isKingCapturedAfter(m *Move) bool {
	_, kingCaptured := b.Moves(otherPlayer(m.player), true)
	return kingCaptured
}

func (b *Board) doMove(m *Move) {
	println("do move", m.String())
	b.squares[m.source] = Empty
	b.squares[m.destination] = m.piece
	fmt.Printf("%s\n", b)

}
func (b *Board) undoMove(m *Move) {
	println("undo move", m.String())
	b.squares[m.source] = m.piece
	b.squares[m.destination] = m.capture
	fmt.Printf("%s\n", b)
}
<commit_msg>refactoring<commit_after>package emil

import (
	"fmt"
)

// Board with an array of field values, representing pieces
type Board struct {
	squares []int
}

// NewBoard creates a new Board
func NewBoard() *Board {
	return &Board{squares: make([]int, SQUARES)}
}
func (b *Board) String() string {
	files := "a b c d e f g h "
	s := "  " + files + " \n"
	for _, r := range FirstSquares {
		s += fmt.Sprintf("%d ", BoardSquares[r].rank)
		for f := 0; f < 8; f++ {
			s += fmt.Sprintf("%s ", symbol(b.squares[r+f]))
		}
		s += fmt.Sprintf("%d\n", BoardSquares[r].rank)
	}
	s += "  " + files + " \n"
	return s
}

func symbol(piece int) string {
	switch piece {
	case WhiteKing:
		return "K"
	case BlackKing:
		return "k"
	case WhiteRock:
		return "R"
	case BlackRock:
		return "r"
	default:
		return " "
	}
}

//Setup a piece on a square
func (b *Board) Setup(piece, square int) {
	b.squares[square] = piece
}

func isOwnPiece(player, capture int) bool {
	return (player == WHITE && capture > 0) ||
		(player == BLACK && capture < 0)
}

func otherPlayer(player int) int {
	if player == WHITE {
		return BLACK
	}
	return WHITE
}

//Moves prints all moves for a piece on square
func (b *Board) Moves(player int, testKingCapture bool) (string, bool) {
	empty := ""
	var result, list []*Move
	for src, piece := range b.squares {
		if isOwnPiece(player, piece) {
			switch abs(piece) {
			case kingValue:
				for _, dst := range kingDestinationsFrom(src) {
					capture := b.squares[dst]
					if isKing(capture) {
						return empty, true
					}
					if capture == Empty {
						list = append(list, newSilentMove(player, piece, src, dst))
					} else if !isOwnPiece(player, capture) {
						list = append(list, newCaptureMove(player, piece, capture, src, dst))
					}
				}
			case rockValue:
				for _, dsts := range rockDestinationsFrom(src) {
					for _, dst := range dsts {
						capture := b.squares[dst]
						if isKing(capture) {
							return empty, true
						}
						if capture == Empty {
							list = append(list, newSilentMove(player, piece, src, dst))
						} else if !isOwnPiece(player, capture) {
							list = append(list, newCaptureMove(player, piece, capture, src, dst))
							break
						} else {
							break // onOwnPiece
						}
					}
				}
			}
		}
	}

	if testKingCapture {
		return empty, false
	}

	for _, m := range list {
		println("TEST move", m.String())
		b.doMove(m)
		if !b.isKingCapturedAfter(m) {
			result = append(result, m)
		} else {
			println("KingCaptured")
		}
		b.undoMove(m)
		fmt.Printf("\n\n\n")
	}

	return moveList(result), false
}
func isKing(piece int) bool {
	return abs(piece) == kingValue
}

func (b *Board) isKingCapturedAfter(m *Move) bool {
	_, kingCaptured := b.Moves(otherPlayer(m.player), true)
	return kingCaptured
}

func (b *Board) doMove(m *Move) {
	println("do move", m.String())
	b.squares[m.source] = Empty
	b.squares[m.destination] = m.piece
	fmt.Printf("%s\n", b)

}
func (b *Board) undoMove(m *Move) {
	println("undo move", m.String())
	b.squares[m.source] = m.piece
	b.squares[m.destination] = m.capture
	fmt.Printf("%s\n", b)
}
<|endoftext|>


<commit_before>package locket

import (
	"net"
	"time"

	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket/models"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

type ClientLocketConfig struct {
	LocketAddress        string `json:"locket_address,omitempty" yaml:"locket_address,omitempty"`
	LocketCACertFile     string `json:"locket_ca_cert_file,omitempty" yaml:"locket_ca_cert_file,omitempty"`
	LocketClientCertFile string `json:"locket_client_cert_file,omitempty" yaml:"locket_client_cert_file,omitempty"`
	LocketClientKeyFile  string `json:"locket_client_key_file,omitempty" yaml:"locket_client_key_file,omitempty"`
}

func NewClientSkipCertVerify(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, true)
}

func NewClient(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, false)
}

func newClientInternal(logger lager.Logger, config ClientLocketConfig, skipCertVerify bool) (models.LocketClient, error) {
	if config.LocketAddress == "" {
		logger.Fatal("invalid-locket-config", nil)
	}

	locketTLSConfig, err := cfhttp.NewTLSConfig(config.LocketClientCertFile, config.LocketClientKeyFile, config.LocketCACertFile)
	if err != nil {
		logger.Error("failed-to-open-tls-config", err, lager.Data{"keypath": config.LocketClientKeyFile, "certpath": config.LocketClientCertFile, "capath": config.LocketCACertFile})
		return nil, err
	}
	locketTLSConfig.InsecureSkipVerify = skipCertVerify

	// TODO: test the following code when the following change is released:
	// 1. https://go-review.googlesource.com/c/go/+/115855
	// 2. https://github.com/golang/go/issues/12503
	//
	// We will need the mentioned change in order to mock the dns resolver to
	// return a list of addresses. We will also need to add a new NewClient
	// method that accepts a dialer in order to mock the ipsec (blocking) issue
	// we ran into in https://www.pivotaltracker.com/story/show/158104990
	conn, err := grpc.Dial(
		config.LocketAddress,
		grpc.WithTransportCredentials(credentials.NewTLS(locketTLSConfig)),
		grpc.WithDialer(func(addr string, _ time.Duration) (net.Conn, error) {
			return net.DialTimeout("tcp", addr, 10*time.Second) // give at least 2 seconds per ip address (assuming there are at most 5)
		}),
		grpc.WithBlock(),
		grpc.WithTimeout(10*time.Second), // ensure that grpc won't keep retrying forever
	)
	if err != nil {
		return nil, err
	}
	return models.NewLocketClient(conn), nil
}
<commit_msg>Sets idle timeout on client TCP connections<commit_after>package locket

import (
	"net"
	"time"

	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket/models"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
)

type ClientLocketConfig struct {
	LocketAddress        string `json:"locket_address,omitempty" yaml:"locket_address,omitempty"`
	LocketCACertFile     string `json:"locket_ca_cert_file,omitempty" yaml:"locket_ca_cert_file,omitempty"`
	LocketClientCertFile string `json:"locket_client_cert_file,omitempty" yaml:"locket_client_cert_file,omitempty"`
	LocketClientKeyFile  string `json:"locket_client_key_file,omitempty" yaml:"locket_client_key_file,omitempty"`
}

func NewClientSkipCertVerify(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, true)
}

func NewClient(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, false)
}

func newClientInternal(logger lager.Logger, config ClientLocketConfig, skipCertVerify bool) (models.LocketClient, error) {
	if config.LocketAddress == "" {
		logger.Fatal("invalid-locket-config", nil)
	}

	locketTLSConfig, err := cfhttp.NewTLSConfig(config.LocketClientCertFile, config.LocketClientKeyFile, config.LocketCACertFile)
	if err != nil {
		logger.Error("failed-to-open-tls-config", err, lager.Data{"keypath": config.LocketClientKeyFile, "certpath": config.LocketClientCertFile, "capath": config.LocketCACertFile})
		return nil, err
	}
	locketTLSConfig.InsecureSkipVerify = skipCertVerify

	// TODO: test the following code when the following change is released:
	// 1. https://go-review.googlesource.com/c/go/+/115855
	// 2. https://github.com/golang/go/issues/12503
	//
	// We will need the mentioned change in order to mock the dns resolver to
	// return a list of addresses. We will also need to add a new NewClient
	// method that accepts a dialer in order to mock the ipsec (blocking) issue
	// we ran into in https://www.pivotaltracker.com/story/show/158104990
	conn, err := grpc.Dial(
		config.LocketAddress,
		grpc.WithTransportCredentials(credentials.NewTLS(locketTLSConfig)),
		grpc.WithDialer(func(addr string, _ time.Duration) (net.Conn, error) {
			return net.DialTimeout("tcp", addr, 10*time.Second) // give at least 2 seconds per ip address (assuming there are at most 5)
		}),
		grpc.WithBlock(),
		grpc.WithTimeout(10*time.Second), // ensure that grpc won't keep retrying forever
		grpc.WithKeepaliveParams(keepalive.ClientParameters{
			Time: 10 * time.Second,
		}),
	)
	if err != nil {
		return nil, err
	}
	return models.NewLocketClient(conn), nil
}
<|endoftext|>


<commit_before>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package logger

import (
	"io"
	"log/syslog"
	"os"
	"os/exec"
	"strings"

	"github.com/adobe-platform/porter/constants"
	"github.com/inconshreveable/log15"
	"github.com/onsi/ginkgo"
)

var cliLog = log15.New("porter_version", constants.Version)
var hostLog = log15.New("porter_version", constants.Version)

func init() {
	out, err := exec.Command("uname").Output()
	if err == nil && strings.TrimSpace(string(out)) == "Linux" {
		initHostLog()
	}

	var logFmt log15.Format
	if os.Getenv(constants.EnvNoLogColor) == "" {
		logFmt = log15.TerminalFormat()
	} else {
		logFmt = log15.LogfmtFormat()
	}

	if os.Getenv("TEST") == "true" {

		handler := log15.StreamHandler(ginkgo.GinkgoWriter, logFmt)
		handler = log15.LvlFilterHandler(log15.LvlError, handler)
		handler = log15.CallerStackHandler("%+v", handler)

		log15.Root().SetHandler(handler)
	} else {
		// the default logging format is best for logging to stdout
		addStackTraceLogging(cliLog, os.Stdout, logFmt)
	}
}

func initHostLog() {

	writer, err := syslog.Dial("udp", "localhost:514", syslog.LOG_DAEMON|syslog.LOG_INFO, constants.ProgramName)
	if err != nil {
		panic(err)
	}

	addStackTraceLogging(hostLog, writer, log15.LogfmtFormat())
}

func CLI(kvps ...interface{}) log15.Logger {
	return cliLog.New(kvps...)
}

func Host(kvps ...interface{}) log15.Logger {
	return hostLog.New(kvps...)
}

func Daemon(kvps ...interface{}) log15.Logger {
	kvps = append(kvps, "service", "porterd")
	return Host(kvps...)
}

func addStackTraceLogging(log log15.Logger, writer io.Writer, logFmt log15.Format) {
	/*
		Log stack traces for LvlCrit, LvlError, and LvlWarn
		to help us debug issues in the wild

		const (
			LvlCrit Lvl = iota
			LvlError
			LvlWarn
			LvlInfo
			LvlDebug
		)
	*/
	stackHandler := log15.StreamHandler(writer, logFmt)
	stackHandler = log15.CallerStackHandler("%+v", stackHandler)
	// put filter last because it will be run first
	stackHandler = log15.FilterHandler(func(r *log15.Record) bool {
		return r.Lvl <= log15.LvlWarn
	}, stackHandler)

	infoHandler := log15.StreamHandler(writer, logFmt)
	// put filter last because it will be run first
	infoHandler = log15.FilterHandler(func(r *log15.Record) bool {
		if os.Getenv(constants.EnvLogDebug) == "" {
			return r.Lvl <= log15.LvlInfo
		} else {
			return r.Lvl <= log15.LvlDebug
		}
	}, infoHandler)

	log.SetHandler(log15.MultiHandler(stackHandler, infoHandler))
}
<commit_msg>only check env var once<commit_after>/*
 *  Copyright 2016 Adobe Systems Incorporated. All rights reserved.
 *  This file is licensed to you under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License. You may obtain a copy
 *  of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software distributed under
 *  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
 *  OF ANY KIND, either express or implied. See the License for the specific language
 *  governing permissions and limitations under the License.
 */
package logger

import (
	"io"
	"log/syslog"
	"os"
	"os/exec"
	"strings"

	"github.com/adobe-platform/porter/constants"
	"github.com/inconshreveable/log15"
	"github.com/onsi/ginkgo"
)

var cliLog = log15.New("porter_version", constants.Version)
var hostLog = log15.New("porter_version", constants.Version)

func init() {
	out, err := exec.Command("uname").Output()
	if err == nil && strings.TrimSpace(string(out)) == "Linux" {
		initHostLog()
	}

	var logFmt log15.Format
	if os.Getenv(constants.EnvNoLogColor) == "" {
		logFmt = log15.TerminalFormat()
	} else {
		logFmt = log15.LogfmtFormat()
	}

	if os.Getenv("TEST") == "true" {

		handler := log15.StreamHandler(ginkgo.GinkgoWriter, logFmt)
		handler = log15.LvlFilterHandler(log15.LvlError, handler)
		handler = log15.CallerStackHandler("%+v", handler)

		log15.Root().SetHandler(handler)
	} else {
		// the default logging format is best for logging to stdout
		addStackTraceLogging(cliLog, os.Stdout, logFmt)
	}
}

func initHostLog() {

	writer, err := syslog.Dial("udp", "localhost:514", syslog.LOG_DAEMON|syslog.LOG_INFO, constants.ProgramName)
	if err != nil {
		panic(err)
	}

	addStackTraceLogging(hostLog, writer, log15.LogfmtFormat())
}

func CLI(kvps ...interface{}) log15.Logger {
	return cliLog.New(kvps...)
}

func Host(kvps ...interface{}) log15.Logger {
	return hostLog.New(kvps...)
}

func Daemon(kvps ...interface{}) log15.Logger {
	kvps = append(kvps, "service", "porterd")
	return Host(kvps...)
}

func addStackTraceLogging(log log15.Logger, writer io.Writer, logFmt log15.Format) {
	/*
		Log stack traces for LvlCrit, LvlError, and LvlWarn
		to help us debug issues in the wild

		const (
			LvlCrit Lvl = iota
			LvlError
			LvlWarn
			LvlInfo
			LvlDebug
		)
	*/
	stackHandler := log15.StreamHandler(writer, logFmt)
	stackHandler = log15.CallerStackHandler("%+v", stackHandler)
	// put filter last because it will be run first
	stackHandler = log15.FilterHandler(func(r *log15.Record) bool {
		return r.Lvl <= log15.LvlWarn
	}, stackHandler)

	infoHandler := log15.StreamHandler(writer, logFmt)
	if os.Getenv(constants.EnvLogDebug) == "" {
		infoHandler = log15.FilterHandler(func(r *log15.Record) bool {
			return r.Lvl <= log15.LvlInfo
		}, infoHandler)
	} else {
		infoHandler = log15.FilterHandler(func(r *log15.Record) bool {
			return r.Lvl <= log15.LvlDebug
		}, infoHandler)
	}

	log.SetHandler(log15.MultiHandler(stackHandler, infoHandler))
}
<|endoftext|>


<commit_before>package logging

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type SyslogLine struct {
	Raw        string
	Time       time.Time
	Host       string
	Tag        string
	Severity   string
	Pid        int
	fields     []string
	parsed     bool
	tags       map[string]interface{}
	tagsParsed bool
}

func (line *SyslogLine) TagString(key string) string {
	return fmt.Sprint(line.Tags()[key])
}

var validKeyRegexp = regexp.MustCompile("(?i)^[a-z]+$")
var callsRegexp = regexp.MustCompile("^([0-9.]+)\\/([0-9]+)$")

func removeQuotes(raw string) string {
	if strings.HasPrefix(raw, `"`) && strings.HasSuffix(raw, `"`) {
		return raw[1 : len(raw)-1]
	}
	return raw
}

func parseTags(raw string) map[string]interface{} {
	fields := strings.Fields(raw)
	inQuotes := false
	currentKey := ""
	valueParts := []string{}
	t := map[string]interface{}{}
	for _, field := range fields {
		if inQuotes {
			valueParts = append(valueParts, field)
			if strings.Contains(field, `"`) {
				inQuotes = false
				v := strings.Join(valueParts, " ")
				t[currentKey] = removeQuotes(v)
			}
		} else {
			kv := strings.SplitN(field, "=", 2)
			if len(kv) == 2 && validKeyRegexp.MatchString(kv[0]) {
				currentKey = kv[0]
				value := kv[1]
				if strings.Contains(value, `"`) && !strings.HasSuffix(value, `"`) {
					valueParts = []string{value}
					inQuotes = true
				} else if value != "-" {
					m := callsRegexp.FindStringSubmatch(value)
					if len(m) == 3 {
						totalTime, e := strconv.ParseFloat(m[1], 64)
						if e == nil {
							calls, e := strconv.ParseInt(m[2], 10, 64)
							if e == nil {
								t[currentKey+"_time"] = totalTime
								t[currentKey+"_calls"] = calls
							}
						}
					} else {
						t[currentKey] = parseTagValue(value)
						currentKey = ""
					}
				}
			}
		}
	}
	return t
}

func (line *SyslogLine) Tags() (t map[string]interface{}) {
	if !line.tagsParsed {
		line.tags = parseTags(line.Raw)
	}
	return line.tags
}

func parseTagValue(raw string) interface{} {
	if i, e := strconv.ParseInt(raw, 10, 64); e == nil {
		return i
	} else if f, e := strconv.ParseFloat(raw, 64); e == nil {
		return f
	}
	return removeQuotes(raw)
}

const (
	timeLayout             = "2006-01-02T15:04:05.000000-07:00"
	timeLayoutWithoutMicro = "2006-01-02T15:04:05-07:00"
)

var TagRegexp = regexp.MustCompile("(.*?)\\[(\\d*)\\]")

func (line *SyslogLine) Parse(raw string) (e error) {
	if line.parsed {
		return nil
	}
	line.Raw = raw
	line.fields = strings.Fields(raw)
	if len(line.fields) >= 3 {
		line.Time, e = time.Parse(timeLayout, line.fields[0])
		if e != nil {
			line.Time, e = time.Parse(timeLayoutWithoutMicro, line.fields[0])
			if e != nil {
				return e
			}
		}
		line.Host = line.fields[1]
		line.Tag, line.Severity, line.Pid = parseTag(line.fields[2])
	}
	line.parsed = true
	return nil
}

func parseTag(raw string) (tag, severity string, pid int) {
	tagAndSeverity, pid := splitTagAndPid(raw)
	tag, severity = splitTagAndSeverity(tagAndSeverity)
	return tag, severity, pid
}

func splitTagAndSeverity(raw string) (tag, severity string) {
	tag = raw
	parts := strings.Split(raw, ".")
	if len(parts) == 2 {
		tag, severity = parts[0], parts[1]
	} else {
		tag = raw
	}
	return tag, severity
}

func splitTagAndPid(raw string) (tag string, pid int) {
	tag = raw
	chunks := TagRegexp.FindStringSubmatch(raw)
	if len(chunks) > 2 {
		tag = chunks[1]
		pid, _ = strconv.Atoi(chunks[2])
	} else {
		if tag[len(tag)-1] == ':' {
			tag = tag[0 : len(tag)-1]
		}
	}
	return tag, pid
}

var UUIDRegexp = regexp.MustCompile("([a-z0-9\\-]{36})")

type UnicornLine struct {
	UUID string
	SyslogLine
}

func (line *UnicornLine) Parse(raw string) error {
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "unicorn" {
		return fmt.Errorf("tag %q not supported", line.Tag)
	}
	if len(line.fields) >= 4 {
		parts := UUIDRegexp.FindStringSubmatch(raw)
		if len(parts) > 1 {
			line.UUID = parts[1]
		}
	}
	return nil
}

type NginxLine struct {
	*SyslogLine
	XForwardedFor []string
	Method        string
	Status        string
	Length        int
	TotalTime     float64
	UnicornTime   float64
	HttpHost      string
	UserAgentName string
	Uri           string
	Referer       string
	Action        string
	Revision      string
	UUID          string
}

var quotesRegexp = regexp.MustCompile(`(ua|uri|ref)="(.*?)"`)

func (line *NginxLine) Parse(raw string) error {
	if line.SyslogLine == nil {
		line.SyslogLine = &SyslogLine{}
	}
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "ssl_endpoint" && line.Tag != "nginx" {
		return fmt.Errorf("tag %q not supported", line.Tag)
	}
	forwarded := false
	for _, field := range line.fields {
		parts := strings.SplitN(field, "=", 2)
		if len(parts) == 2 {
			key := parts[0]
			value := parts[1]
			switch key {
			case "rev":
				line.Revision = value
			case "action":
				line.Action = value
			case "nginx":
				forwarded = true
			case "method":
				line.Method = value
			case "uuid":
				line.UUID = value
			case "status":
				line.Status = value
			case "host":
				line.HttpHost = value
			case "length":
				line.Length, _ = strconv.Atoi(value)
			case "total":
				line.TotalTime, _ = strconv.ParseFloat(value, 64)
			case "unicorn_time":
				line.UnicornTime, _ = strconv.ParseFloat(value, 64)
			}
		} else if field == "nginx:" {
			forwarded = true
		} else if forwarded {
			line.XForwardedFor = append(line.XForwardedFor, field)
		} else if strings.HasPrefix(field, "host=") {
			forwarded = false
		}
	}
	quotes := quotesRegexp.FindAllStringSubmatch(raw, -1)
	for _, quote := range quotes {
		switch quote[1] {
		case "ua":
			line.UserAgentName = quote[2]
		case "uri":
			line.Uri = quote[2]
		case "ref":
			line.Referer = quote[2]
		default:
		}
	}
	return nil
}

type HAProxyLine struct {
	SyslogLine
	Frontend            string
	Backend             string
	BackendHost         string
	BackendImageId      string
	BackendContainerId  string
	Status              string
	Length              int
	ClientRequestTime   int
	ConnectionQueueTime int
	TcpConnectTime      int
	ServerResponseTime  int
	SessionDurationTime int
	ActiveConnections   int
	FrontendConnections int
	BackendConnectons   int
	ServerConnections   int
	Retries             int
	ServerQueue         int
	BackendQueue        int
	Method              string
	Uri                 string
}

func (line *HAProxyLine) Parse(raw string) error {
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "haproxy" {
		return fmt.Errorf("tag was %s", line.Tag)
	}
	if len(line.fields) > 16 {
		line.Frontend = line.fields[5]
		backend := line.fields[6]
		parts := strings.SplitN(backend, "/", 2)
		if len(parts) == 2 {
			line.Backend = parts[0]
			backendContainer := parts[1]
			parts := strings.Split(backendContainer, ":")
			if len(parts) == 3 {
				line.BackendHost = parts[0]
				line.BackendImageId = parts[1]
				line.BackendContainerId = parts[2]
			}
		}
		times := line.fields[7]
		parts = strings.Split(times, "/")
		if len(parts) == 5 {
			line.ClientRequestTime, _ = strconv.Atoi(parts[0])
			line.ConnectionQueueTime, _ = strconv.Atoi(parts[1])
			line.TcpConnectTime, _ = strconv.Atoi(parts[2])
			line.ServerResponseTime, _ = strconv.Atoi(parts[3])
			line.SessionDurationTime, _ = strconv.Atoi(parts[4])
		}
		line.Status = line.fields[8]
		line.Length, _ = strconv.Atoi(line.fields[9])

		connections := line.fields[13]
		parts = strings.Split(connections, "/")
		if len(parts) == 5 {
			line.ActiveConnections, _ = strconv.Atoi(parts[0])
			line.FrontendConnections, _ = strconv.Atoi(parts[1])
			line.BackendConnectons, _ = strconv.Atoi(parts[2])
			line.ServerConnections, _ = strconv.Atoi(parts[3])
			line.Retries, _ = strconv.Atoi(parts[4])
		}

		queues := line.fields[14]
		parts = strings.Split(queues, "/")
		if len(parts) == 2 {
			line.ServerQueue, _ = strconv.Atoi(parts[0])
			line.BackendQueue, _ = strconv.Atoi(parts[1])
		}
		line.Method = line.fields[15][1:]
		line.Uri = line.fields[16]
	}
	return nil
}
<commit_msg>extract etag, fuilter dashes for etag and referer<commit_after>package logging

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type SyslogLine struct {
	Raw        string
	Time       time.Time
	Host       string
	Tag        string
	Severity   string
	Pid        int
	fields     []string
	parsed     bool
	tags       map[string]interface{}
	tagsParsed bool
}

func (line *SyslogLine) TagString(key string) string {
	return fmt.Sprint(line.Tags()[key])
}

var validKeyRegexp = regexp.MustCompile("(?i)^[a-z]+$")
var callsRegexp = regexp.MustCompile("^([0-9.]+)\\/([0-9]+)$")

func removeQuotes(raw string) string {
	if strings.HasPrefix(raw, `"`) && strings.HasSuffix(raw, `"`) {
		return raw[1 : len(raw)-1]
	}
	return raw
}

func parseTags(raw string) map[string]interface{} {
	fields := strings.Fields(raw)
	inQuotes := false
	currentKey := ""
	valueParts := []string{}
	t := map[string]interface{}{}
	for _, field := range fields {
		if inQuotes {
			valueParts = append(valueParts, field)
			if strings.Contains(field, `"`) {
				inQuotes = false
				v := strings.Join(valueParts, " ")
				t[currentKey] = removeQuotes(v)
			}
		} else {
			kv := strings.SplitN(field, "=", 2)
			if len(kv) == 2 && validKeyRegexp.MatchString(kv[0]) {
				currentKey = kv[0]
				value := kv[1]
				if strings.Contains(value, `"`) && !strings.HasSuffix(value, `"`) {
					valueParts = []string{value}
					inQuotes = true
				} else if value != "-" {
					m := callsRegexp.FindStringSubmatch(value)
					if len(m) == 3 {
						totalTime, e := strconv.ParseFloat(m[1], 64)
						if e == nil {
							calls, e := strconv.ParseInt(m[2], 10, 64)
							if e == nil {
								t[currentKey+"_time"] = totalTime
								t[currentKey+"_calls"] = calls
							}
						}
					} else {
						t[currentKey] = parseTagValue(value)
						currentKey = ""
					}
				}
			}
		}
	}
	return t
}

func (line *SyslogLine) Tags() (t map[string]interface{}) {
	if !line.tagsParsed {
		line.tags = parseTags(line.Raw)
	}
	return line.tags
}

func parseTagValue(raw string) interface{} {
	if i, e := strconv.ParseInt(raw, 10, 64); e == nil {
		return i
	} else if f, e := strconv.ParseFloat(raw, 64); e == nil {
		return f
	}
	return removeQuotes(raw)
}

const (
	timeLayout             = "2006-01-02T15:04:05.000000-07:00"
	timeLayoutWithoutMicro = "2006-01-02T15:04:05-07:00"
)

var TagRegexp = regexp.MustCompile("(.*?)\\[(\\d*)\\]")

func (line *SyslogLine) Parse(raw string) (e error) {
	if line.parsed {
		return nil
	}
	line.Raw = raw
	line.fields = strings.Fields(raw)
	if len(line.fields) >= 3 {
		line.Time, e = time.Parse(timeLayout, line.fields[0])
		if e != nil {
			line.Time, e = time.Parse(timeLayoutWithoutMicro, line.fields[0])
			if e != nil {
				return e
			}
		}
		line.Host = line.fields[1]
		line.Tag, line.Severity, line.Pid = parseTag(line.fields[2])
	}
	line.parsed = true
	return nil
}

func parseTag(raw string) (tag, severity string, pid int) {
	tagAndSeverity, pid := splitTagAndPid(raw)
	tag, severity = splitTagAndSeverity(tagAndSeverity)
	return tag, severity, pid
}

func splitTagAndSeverity(raw string) (tag, severity string) {
	tag = raw
	parts := strings.Split(raw, ".")
	if len(parts) == 2 {
		tag, severity = parts[0], parts[1]
	} else {
		tag = raw
	}
	return tag, severity
}

func splitTagAndPid(raw string) (tag string, pid int) {
	tag = raw
	chunks := TagRegexp.FindStringSubmatch(raw)
	if len(chunks) > 2 {
		tag = chunks[1]
		pid, _ = strconv.Atoi(chunks[2])
	} else {
		if tag[len(tag)-1] == ':' {
			tag = tag[0 : len(tag)-1]
		}
	}
	return tag, pid
}

var UUIDRegexp = regexp.MustCompile("([a-z0-9\\-]{36})")

type UnicornLine struct {
	UUID string
	SyslogLine
}

func (line *UnicornLine) Parse(raw string) error {
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "unicorn" {
		return fmt.Errorf("tag %q not supported", line.Tag)
	}
	if len(line.fields) >= 4 {
		parts := UUIDRegexp.FindStringSubmatch(raw)
		if len(parts) > 1 {
			line.UUID = parts[1]
		}
	}
	return nil
}

type NginxLine struct {
	*SyslogLine
	XForwardedFor []string
	Method        string
	Status        string
	Length        int
	TotalTime     float64
	UnicornTime   float64
	HttpHost      string
	UserAgentName string
	Uri           string
	Referer       string
	Action        string
	Revision      string
	UUID          string
	Etag          string
}

var quotesRegexp = regexp.MustCompile(`(ua|uri|ref)="(.*?)"`)

func (line *NginxLine) Parse(raw string) error {
	if line.SyslogLine == nil {
		line.SyslogLine = &SyslogLine{}
	}
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "ssl_endpoint" && line.Tag != "nginx" {
		return fmt.Errorf("tag %q not supported", line.Tag)
	}
	forwarded := false
	for _, field := range line.fields {
		parts := strings.SplitN(field, "=", 2)
		if len(parts) == 2 {
			key := parts[0]
			value := parts[1]
			switch key {
			case "rev":
				line.Revision = value
			case "action":
				line.Action = value
			case "nginx":
				forwarded = true
			case "method":
				line.Method = value
			case "uuid":
				line.UUID = filterDash(value)
			case "etag":
				line.Etag = filterDash(value)
			case "status":
				line.Status = value
			case "host":
				line.HttpHost = value
			case "length":
				line.Length, _ = strconv.Atoi(value)
			case "total":
				line.TotalTime, _ = strconv.ParseFloat(value, 64)
			case "unicorn_time":
				line.UnicornTime, _ = strconv.ParseFloat(value, 64)
			}
		} else if field == "nginx:" {
			forwarded = true
		} else if forwarded {
			line.XForwardedFor = append(line.XForwardedFor, field)
		} else if strings.HasPrefix(field, "host=") {
			forwarded = false
		}
	}
	quotes := quotesRegexp.FindAllStringSubmatch(raw, -1)
	for _, quote := range quotes {
		switch quote[1] {
		case "ua":
			line.UserAgentName = quote[2]
		case "uri":
			line.Uri = quote[2]
		case "ref":
			line.Referer = filterDash(quote[2])
		default:
		}
	}
	return nil
}

func filterDash(raw string) string {
	if raw == "-" {
		return ""
	}
	return raw
}

type HAProxyLine struct {
	SyslogLine
	Frontend            string
	Backend             string
	BackendHost         string
	BackendImageId      string
	BackendContainerId  string
	Status              string
	Length              int
	ClientRequestTime   int
	ConnectionQueueTime int
	TcpConnectTime      int
	ServerResponseTime  int
	SessionDurationTime int
	ActiveConnections   int
	FrontendConnections int
	BackendConnectons   int
	ServerConnections   int
	Retries             int
	ServerQueue         int
	BackendQueue        int
	Method              string
	Uri                 string
}

func (line *HAProxyLine) Parse(raw string) error {
	e := line.SyslogLine.Parse(raw)
	if e != nil {
		return e
	}
	if line.Tag != "haproxy" {
		return fmt.Errorf("tag was %s", line.Tag)
	}
	if len(line.fields) > 16 {
		line.Frontend = line.fields[5]
		backend := line.fields[6]
		parts := strings.SplitN(backend, "/", 2)
		if len(parts) == 2 {
			line.Backend = parts[0]
			backendContainer := parts[1]
			parts := strings.Split(backendContainer, ":")
			if len(parts) == 3 {
				line.BackendHost = parts[0]
				line.BackendImageId = parts[1]
				line.BackendContainerId = parts[2]
			}
		}
		times := line.fields[7]
		parts = strings.Split(times, "/")
		if len(parts) == 5 {
			line.ClientRequestTime, _ = strconv.Atoi(parts[0])
			line.ConnectionQueueTime, _ = strconv.Atoi(parts[1])
			line.TcpConnectTime, _ = strconv.Atoi(parts[2])
			line.ServerResponseTime, _ = strconv.Atoi(parts[3])
			line.SessionDurationTime, _ = strconv.Atoi(parts[4])
		}
		line.Status = line.fields[8]
		line.Length, _ = strconv.Atoi(line.fields[9])

		connections := line.fields[13]
		parts = strings.Split(connections, "/")
		if len(parts) == 5 {
			line.ActiveConnections, _ = strconv.Atoi(parts[0])
			line.FrontendConnections, _ = strconv.Atoi(parts[1])
			line.BackendConnectons, _ = strconv.Atoi(parts[2])
			line.ServerConnections, _ = strconv.Atoi(parts[3])
			line.Retries, _ = strconv.Atoi(parts[4])
		}

		queues := line.fields[14]
		parts = strings.Split(queues, "/")
		if len(parts) == 2 {
			line.ServerQueue, _ = strconv.Atoi(parts[0])
			line.BackendQueue, _ = strconv.Atoi(parts[1])
		}
		line.Method = line.fields[15][1:]
		line.Uri = line.fields[16]
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type logEntry struct {
	Node string
	Time int64
	Msg  string
}

var (
	tsFormat = "2006/01/02 15:04:05"
	addr     string
	dbFile   string
	logChan  chan *logEntry
)

var logSplit = regexp.MustCompile("^([\\w-:]+) (\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2}) (.+)$")
var responseCheck = regexp.MustCompile("response time: (\\d+)us$")

func init() {
	flLogBuffer := flag.Uint("b", 16, "log entries to buffer")
	flDbFile := flag.String("f", "logs.db", "database file")
	port := flag.Uint("p", 5988, "port to listen on")
	flag.Parse()

	addr = fmt.Sprintf(":%d", *port)
	dbFile = *flDbFile
	logChan = make(chan *logEntry, *flLogBuffer)
}

func main() {
	dbSetup()
	go log()
	go listen()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
	<-sigc
	close(logChan)
	<-time.After(100 * time.Millisecond)

	os.Exit(1)
}

func listen() {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to resolve TCP address:", err.Error())
		os.Exit(1)
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to set up TCP listener:", err.Error())
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			fmt.Println("[!] TCP error:", err.Error())
			continue
		}
		go processMessage(conn)
	}

}

func processMessage(conn net.Conn) {
	defer conn.Close()
	r := bufio.NewReader(conn)

	for {
		msg, err := r.ReadString(0x0a)
		if err != nil {
			if err != io.EOF {
				fmt.Println("[!] error reading from client:",
					err.Error())
			}
			return
		}
		msg = strings.Trim(string(msg), "\n \t")
		fmt.Println("-- ", msg)

		nodeID := logSplit.ReplaceAllString(msg, "$1")
		dateString := logSplit.ReplaceAllString(msg, "$2")
		logMsg := logSplit.ReplaceAllString(msg, "$3")
		tm, err := time.Parse(tsFormat, dateString)
		if err != nil {
			fmt.Fprintf(os.Stderr, "[!] error parsing time %s: %s\n",
				dateString, err.Error())
			return
		}
		le := &logEntry{nodeID, tm.UTC().Unix(), logMsg}
		logChan <- le
	}
}

func log() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	for {
		le, ok := <-logChan
		if !ok {
			fmt.Println("[+] shutting down database listener")
			return
		}
		_, err := db.Exec("insert into entries values (?, ?, ?)",
			le.Node, le.Time, le.Msg)
		if err != nil {
			fmt.Println("[!] database error:", err.Error())
			continue
		}

		if responseCheck.MatchString(le.Msg) {
			logResponseTime(db, le)
		}

	}
}

func logResponseTime(db *sql.DB, le *logEntry) {
	respString := responseCheck.ReplaceAllString(le.Msg, "$1")
	rTime, err := strconv.Atoi(respString)
	if err != nil {
		fmt.Println("[!] error reading response time:", err.Error())
		return
	}
	_, err = db.Exec("insert into response_times values (?, ?, ?)",
		le.Node, le.Time, rTime)
	if err != nil {
		fmt.Println("[!] error writing to database:", err.Error())
	}
}

func dbSetup() {
	entryTable()
	respTable()
}

func entryTable() {
	const createSql = `CREATE TABLE entries (node text, timestamp integer, message string)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='entries'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table entries`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}

func respTable() {
	const createSql = `CREATE TABLE response_times (node text, timestamp integer, microsec integer)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='response_times'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table response_times`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}
<commit_msg>Report client connections.<commit_after>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type logEntry struct {
	Node string
	Time int64
	Msg  string
}

var (
	tsFormat = "2006/01/02 15:04:05"
	addr     string
	dbFile   string
	logChan  chan *logEntry
)

var logSplit = regexp.MustCompile("^([\\w-:]+) (\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2}) (.+)$")
var responseCheck = regexp.MustCompile("response time: (\\d+)us$")

func init() {
	flLogBuffer := flag.Uint("b", 16, "log entries to buffer")
	flDbFile := flag.String("f", "logs.db", "database file")
	port := flag.Uint("p", 5988, "port to listen on")
	flag.Parse()

	addr = fmt.Sprintf(":%d", *port)
	dbFile = *flDbFile
	logChan = make(chan *logEntry, *flLogBuffer)
}

func main() {
	dbSetup()
	go log()
	go listen()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
	<-sigc
	close(logChan)
	<-time.After(100 * time.Millisecond)

	os.Exit(1)
}

func listen() {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to resolve TCP address:", err.Error())
		os.Exit(1)
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to set up TCP listener:", err.Error())
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			fmt.Println("[!] TCP error:", err.Error())
			continue
		}
		go processMessage(conn)
	}

}

func processMessage(conn net.Conn) {
	fmt.Println("[+] client connected:", conn.RemoteAddr)
	defer conn.Close()
	r := bufio.NewReader(conn)

	for {
		msg, err := r.ReadString(0x0a)
		if err != nil {
			if err != io.EOF {
				fmt.Println("[!] error reading from client:",
					err.Error())
			}
			return
		}
		msg = strings.Trim(string(msg), "\n \t")
		fmt.Println("-- ", msg)

		nodeID := logSplit.ReplaceAllString(msg, "$1")
		dateString := logSplit.ReplaceAllString(msg, "$2")
		logMsg := logSplit.ReplaceAllString(msg, "$3")
		tm, err := time.Parse(tsFormat, dateString)
		if err != nil {
			fmt.Fprintf(os.Stderr, "[!] error parsing time %s: %s\n",
				dateString, err.Error())
			return
		}
		le := &logEntry{nodeID, tm.UTC().Unix(), logMsg}
		logChan <- le
	}
}

func log() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	for {
		le, ok := <-logChan
		if !ok {
			fmt.Println("[+] shutting down database listener")
			return
		}
		_, err := db.Exec("insert into entries values (?, ?, ?)",
			le.Node, le.Time, le.Msg)
		if err != nil {
			fmt.Println("[!] database error:", err.Error())
			continue
		}

		if responseCheck.MatchString(le.Msg) {
			logResponseTime(db, le)
		}

	}
}

func logResponseTime(db *sql.DB, le *logEntry) {
	respString := responseCheck.ReplaceAllString(le.Msg, "$1")
	rTime, err := strconv.Atoi(respString)
	if err != nil {
		fmt.Println("[!] error reading response time:", err.Error())
		return
	}
	_, err = db.Exec("insert into response_times values (?, ?, ?)",
		le.Node, le.Time, rTime)
	if err != nil {
		fmt.Println("[!] error writing to database:", err.Error())
	}
}

func dbSetup() {
	entryTable()
	respTable()
}

func entryTable() {
	const createSql = `CREATE TABLE entries (node text, timestamp integer, message string)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='entries'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table entries`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}

func respTable() {
	const createSql = `CREATE TABLE response_times (node text, timestamp integer, microsec integer)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='response_times'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table response_times`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}
<|endoftext|>