<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"

	"github.com/google/gxui"
	"github.com/google/gxui/drivers/gl"
	"github.com/google/gxui/math"
	"github.com/google/gxui/themes/dark"

	"image"
	"log"
	"os"

	"fmt"

	"github.com/kyokomi/gomajan/pai"
	"github.com/kyokomi/gomajan/player"
	"github.com/kyokomi/gomajan/taku"
)

type State int

const (
	InitState       State = 0
	PlayerTurnState State = 1
	CPUTurnState    State = 2
)

type MainWindow struct {
	gxui.Window
	theme  gxui.Theme
	driver gxui.Driver
}

func NewMainWindow(driver gxui.Driver) MainWindow {
	var mainWindow MainWindow
	theme := dark.CreateTheme(driver)
	window := theme.CreateWindow(1200, 800, "gxjan")

	mainWindow.Window = window
	mainWindow.theme = theme
	mainWindow.driver = driver

	return mainWindow
}

var _state State
var _taku *taku.Taku
var _yamaImage [4][2][17]*PaiImage

var _playerContainer [4]gxui.LinearLayout

func ChangeState(state State) {
	beforeState := _state
	_state = state

	if beforeState == PlayerTurnState && state == CPUTurnState {
		// playerTurn -> CPUTurn
		EventCPU(2)
		EventCPU(3)
		EventCPU(4)
		ChangeState(PlayerTurnState)
	}
}

func NowState() State {
	return _state
}

func EventCPU(playerID int) {
	// 1枚引いて
	mjPai := nextFunc(playerID)

	// 1枚捨てる
	p := Player(playerID)
	p.PaiInc(mjPai)
	_window.nextPaiImage(_playerContainer[playerID-1], playerID, mjPai)

	yakuCheck := p.NewYakuCheck(mjPai)
	if yakuCheck.Is和了() {
		fmt.Println("アガリ: ", yakuCheck.String())
		return
	}
	fmt.Println("チェック: ", yakuCheck.String())

	for _, noko := range yakuCheck.MentsuCheck().Nokori() {
		if noko.Pai.Type() == pai.NoneType || noko.Val == 0 {
			continue
		}

		fmt.Println("捨てた牌: ", noko.Pai)
		p.PaiDec(noko.Pai)

		for i := 0; i < _playerContainer[playerID-1].ChildCount(); i++ {
			paiImage := _playerContainer[playerID-1].ChildAt(i).(PaiImage)
			if paiImage.Pai == noko.Pai {
				_playerContainer[playerID-1].RemoveChildAt(i)
				_sutehaiLayers[playerID-1].AddChild(paiImage)
				break
			}
		}
		break
	}
}

var _window MainWindow
var _sutehaiLayers [4]gxui.LinearLayout

func appMain(driver gxui.Driver) {
	_window = NewMainWindow(driver)

	ChangeState(InitState)

	_taku = taku.NewTaku()

	// TODO: Splitterにしてみた
	rootLayer := _window.Theme().CreateSplitterLayout()
	//	rootLayer.SetOrientation(gxui.TopToBottom)
	//	rootLayer.SetVerticalAlignment(gxui.AlignTop)

	// 手牌
	{
		playLayer := _window.Theme().CreateSplitterLayout()
		playLayer.SetOrientation(gxui.Horizontal)

		tehaiLayer := _window.Theme().CreateLinearLayout()
		tehaiLayer.SetDirection(gxui.TopToBottom)
		tehaiLayer.SetVerticalAlignment(gxui.AlignTop)
		tehaiLayer.SetMargin(math.CreateSpacing(20))

		sutehaiLayer := _window.Theme().CreateLinearLayout()
		sutehaiLayer.SetDirection(gxui.TopToBottom)
		sutehaiLayer.SetVerticalAlignment(gxui.AlignTop)
		sutehaiLayer.SetMargin(math.CreateSpacing(20))

		for _, player := range _taku.Players {
			container := _window.Theme().CreateLinearLayout()
			container.SetDirection(gxui.LeftToRight)
			container.SetHorizontalAlignment(gxui.AlignCenter)

			for _, hai := range player.Tiles() {
				if hai.Pai.Type() == pai.NoneType || hai.Val <= 0 {
					continue
				}

				for i := 0; i < hai.Val; i++ {
					_window.nextPaiImage(container, player.PlayerID(), hai.Pai)
				}
			}
			container.SetMargin(math.CreateSpacing(10))
			tehaiLayer.AddChild(container)
			_playerContainer[player.PlayerID()-1] = container

			suteContainer := _window.Theme().CreateLinearLayout()
			suteContainer.SetDirection(gxui.LeftToRight)
			suteContainer.SetHorizontalAlignment(gxui.AlignCenter)
			suteContainer.SetMargin(math.CreateSpacing(10))

			_sutehaiLayers[player.PlayerID()-1] = suteContainer
			sutehaiLayer.AddChild(suteContainer)
		}
		playLayer.AddChild(tehaiLayer)
		playLayer.SetChildWeight(tehaiLayer, 0.5)

		playLayer.AddChild(sutehaiLayer)
		playLayer.SetChildWeight(sutehaiLayer, 0.5)

		rootLayer.AddChild(playLayer)
		rootLayer.SetChildWeight(playLayer, 0.4)
	}

	// 山
	{
		yamaLayer := _window.Theme().CreateLinearLayout()
		yamaLayer.SetDirection(gxui.TopToBottom)
		yamaLayer.SetVerticalAlignment(gxui.AlignTop)
		yamaLayer.SetMargin(math.CreateSpacing(20))

		for i, 山2 := range _taku.Yama {
			container := _window.Theme().CreateLinearLayout()
			container.SetDirection(gxui.TopToBottom)
			container.SetVerticalAlignment(gxui.AlignTop)
			for j, 山 := range 山2 {
				container2 := _window.Theme().CreateLinearLayout()
				container2.SetDirection(gxui.LeftToRight)
				container2.SetHorizontalAlignment(gxui.AlignCenter)
				for k, 牌 := range 山 {
					if 牌.Type() == pai.NoneType {
						continue
					}

					imagePai := _window.createPaiImage(牌)
					if _taku.YamaMask[i][j][k] != 0 {
						imagePai.SetVisible(false)
					}
					_yamaImage[i][j][k] = &imagePai
					container2.AddChild(imagePai)
				}
				container2.SetMargin(math.CreateSpacing(1))
				container.AddChild(container2)
			}
			container.SetMargin(math.CreateSpacing(4))
			yamaLayer.AddChild(container)
		}
		rootLayer.AddChild(yamaLayer)
		rootLayer.SetChildWeight(yamaLayer, 0.6)
	}

	//	label := window.Theme().CreateLabel()
	//	label.SetText("hogehogehoge")
	//	label.SetMargin(math.CreateSpacing(200))
	//	container.AddChild(label)
	_window.AddChild(rootLayer)
	_window.OnClose(driver.Terminate)

	ChangeState(PlayerTurnState)

	gxui.EventLoop(driver)
}

func (m MainWindow) Theme() gxui.Theme {
	return m.theme
}

type PaiImage struct {
	gxui.Image
	Pai pai.MJP
}

func (m MainWindow) createPaiImage(p pai.MJP) PaiImage {
	f, err := os.Open("data/pai-images/" + imageMappings[p])
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()
	im, _, err := image.Decode(f)
	if err != nil {
		log.Fatalln(err)
	}

	pict := m.theme.CreateImage()
	texture := m.driver.CreateTexture(im, 96)
	pict.SetTexture(texture)
	pict.SetExplicitSize(math.Size{32, 45})
	//	pict.SetMargin(math.CreateSpacing(4))

	return PaiImage{Image: pict, Pai: p}
}

var ueshita = 1 // ちょんちょん後の下段からなので1

func nextFunc(playerID int) pai.MJP {
	if ueshita == 2 {
		_taku.Add列And山越しFunc(1)
		ueshita = 0
	}

	p := _taku.Next(ueshita, playerID)

	// 引いた牌を非表示
	_yamaImage[_taku.PlayYamaIndex][ueshita][_taku.PlayRetuIndex].SetVisible(false)

	ueshita++
	return p
}

func Player(playerID int) *player.Player {
	for _, p := range _taku.Players {
		if p.PlayerID() == playerID {
			return &p
		}
	}
	return nil
}

func (m MainWindow) nextPaiImage(container gxui.LinearLayout, playerID int, mjPai pai.MJP) {
	paiImage := m.createPaiImage(mjPai)

	fmt.Println("nextPaiImage", playerID)
	// TODO: player1だけ操作可能にする
	if playerID == 1 {
		paiImage.OnClick(func(e gxui.MouseEvent) {
			ChangeState(CPUTurnState)

			fmt.Println("OnClick", playerID)

			Player(playerID).PaiDec(paiImage.Pai)
			container.RemoveChild(paiImage)

			_sutehaiLayers[playerID-1].AddChild(paiImage)

			nextPai := nextFunc(playerID)
			Player(playerID).PaiInc(paiImage.Pai)
			m.nextPaiImage(container, playerID, nextPai)
		})
	}

	for i := 0; i < container.ChildCount(); i++ {
		if container.ChildAt(i).(PaiImage).Pai > paiImage.Pai {
			container.AddChildAt(i, paiImage)
			return
		}
	}
	container.AddChild(paiImage)
}

func main() {
	flag.Parse()
	gl.StartDriver(appMain)
}
<commit_msg>bugfix ツモった牌のミス<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"

	"github.com/google/gxui"
	"github.com/google/gxui/drivers/gl"
	"github.com/google/gxui/math"
	"github.com/google/gxui/themes/dark"

	"image"
	"log"
	"os"

	"fmt"

	"github.com/kyokomi/gomajan/pai"
	"github.com/kyokomi/gomajan/player"
	"github.com/kyokomi/gomajan/taku"
)

type State int

const (
	InitState       State = 0
	PlayerTurnState State = 1
	CPUTurnState    State = 2
)

type MainWindow struct {
	gxui.Window
	theme  gxui.Theme
	driver gxui.Driver
}

func NewMainWindow(driver gxui.Driver) MainWindow {
	var mainWindow MainWindow
	theme := dark.CreateTheme(driver)
	window := theme.CreateWindow(1200, 800, "gxjan")

	mainWindow.Window = window
	mainWindow.theme = theme
	mainWindow.driver = driver

	return mainWindow
}

var _state State
var _taku *taku.Taku
var _yamaImage [4][2][17]*PaiImage

var _playerContainer [4]gxui.LinearLayout

func ChangeState(state State) {
	beforeState := _state
	_state = state

	if beforeState == PlayerTurnState && state == CPUTurnState {
		// playerTurn -> CPUTurn
		EventCPU(2)
		EventCPU(3)
		EventCPU(4)
		ChangeState(PlayerTurnState)
	}
}

func NowState() State {
	return _state
}

func EventCPU(playerID int) {
	// 1枚引いて
	mjPai := nextFunc(playerID)

	// 1枚捨てる
	p := Player(playerID)
	p.PaiInc(mjPai)
	_window.nextPaiImage(_playerContainer[playerID-1], playerID, mjPai)

	yakuCheck := p.NewYakuCheck(mjPai)
	if yakuCheck.Is和了() {
		fmt.Println("アガリ: ", yakuCheck.String())
		return
	}
	fmt.Println("チェック: ", yakuCheck.String())

	for _, noko := range yakuCheck.MentsuCheck().Nokori() {
		if noko.Pai.Type() == pai.NoneType || noko.Val == 0 {
			continue
		}

		fmt.Println("捨てた牌: ", noko.Pai)
		p.PaiDec(noko.Pai)

		for i := 0; i < _playerContainer[playerID-1].ChildCount(); i++ {
			paiImage := _playerContainer[playerID-1].ChildAt(i).(PaiImage)
			if paiImage.Pai == noko.Pai {
				_playerContainer[playerID-1].RemoveChildAt(i)
				_sutehaiLayers[playerID-1].AddChild(paiImage)
				break
			}
		}
		break
	}
}

var _window MainWindow
var _sutehaiLayers [4]gxui.LinearLayout

func appMain(driver gxui.Driver) {
	_window = NewMainWindow(driver)

	ChangeState(InitState)

	_taku = taku.NewTaku()

	// TODO: Splitterにしてみた
	rootLayer := _window.Theme().CreateSplitterLayout()
	//	rootLayer.SetOrientation(gxui.TopToBottom)
	//	rootLayer.SetVerticalAlignment(gxui.AlignTop)

	// 手牌
	{
		playLayer := _window.Theme().CreateSplitterLayout()
		playLayer.SetOrientation(gxui.Horizontal)

		tehaiLayer := _window.Theme().CreateLinearLayout()
		tehaiLayer.SetDirection(gxui.TopToBottom)
		tehaiLayer.SetVerticalAlignment(gxui.AlignTop)
		tehaiLayer.SetMargin(math.CreateSpacing(20))

		sutehaiLayer := _window.Theme().CreateLinearLayout()
		sutehaiLayer.SetDirection(gxui.TopToBottom)
		sutehaiLayer.SetVerticalAlignment(gxui.AlignTop)
		sutehaiLayer.SetMargin(math.CreateSpacing(20))

		for _, player := range _taku.Players {
			container := _window.Theme().CreateLinearLayout()
			container.SetDirection(gxui.LeftToRight)
			container.SetHorizontalAlignment(gxui.AlignCenter)

			for _, hai := range player.Tiles() {
				if hai.Pai.Type() == pai.NoneType || hai.Val <= 0 {
					continue
				}

				for i := 0; i < hai.Val; i++ {
					_window.nextPaiImage(container, player.PlayerID(), hai.Pai)
				}
			}
			container.SetMargin(math.CreateSpacing(10))
			tehaiLayer.AddChild(container)
			_playerContainer[player.PlayerID()-1] = container

			suteContainer := _window.Theme().CreateLinearLayout()
			suteContainer.SetDirection(gxui.LeftToRight)
			suteContainer.SetHorizontalAlignment(gxui.AlignCenter)
			suteContainer.SetMargin(math.CreateSpacing(10))

			_sutehaiLayers[player.PlayerID()-1] = suteContainer
			sutehaiLayer.AddChild(suteContainer)
		}
		playLayer.AddChild(tehaiLayer)
		playLayer.SetChildWeight(tehaiLayer, 0.5)

		playLayer.AddChild(sutehaiLayer)
		playLayer.SetChildWeight(sutehaiLayer, 0.5)

		rootLayer.AddChild(playLayer)
		rootLayer.SetChildWeight(playLayer, 0.4)
	}

	// 山
	{
		yamaLayer := _window.Theme().CreateLinearLayout()
		yamaLayer.SetDirection(gxui.TopToBottom)
		yamaLayer.SetVerticalAlignment(gxui.AlignTop)
		yamaLayer.SetMargin(math.CreateSpacing(20))

		for i, 山2 := range _taku.Yama {
			container := _window.Theme().CreateLinearLayout()
			container.SetDirection(gxui.TopToBottom)
			container.SetVerticalAlignment(gxui.AlignTop)
			for j, 山 := range 山2 {
				container2 := _window.Theme().CreateLinearLayout()
				container2.SetDirection(gxui.LeftToRight)
				container2.SetHorizontalAlignment(gxui.AlignCenter)
				for k, 牌 := range 山 {
					if 牌.Type() == pai.NoneType {
						continue
					}

					imagePai := _window.createPaiImage(牌)
					if _taku.YamaMask[i][j][k] != 0 {
						imagePai.SetVisible(false)
					}
					_yamaImage[i][j][k] = &imagePai
					container2.AddChild(imagePai)
				}
				container2.SetMargin(math.CreateSpacing(1))
				container.AddChild(container2)
			}
			container.SetMargin(math.CreateSpacing(4))
			yamaLayer.AddChild(container)
		}
		rootLayer.AddChild(yamaLayer)
		rootLayer.SetChildWeight(yamaLayer, 0.6)
	}

	//	label := window.Theme().CreateLabel()
	//	label.SetText("hogehogehoge")
	//	label.SetMargin(math.CreateSpacing(200))
	//	container.AddChild(label)
	_window.AddChild(rootLayer)
	_window.OnClose(driver.Terminate)

	ChangeState(PlayerTurnState)

	gxui.EventLoop(driver)
}

func (m MainWindow) Theme() gxui.Theme {
	return m.theme
}

type PaiImage struct {
	gxui.Image
	Pai pai.MJP
}

func (m MainWindow) createPaiImage(p pai.MJP) PaiImage {
	f, err := os.Open("data/pai-images/" + imageMappings[p])
	if err != nil {
		log.Fatalln(err)
	}
	defer f.Close()
	im, _, err := image.Decode(f)
	if err != nil {
		log.Fatalln(err)
	}

	pict := m.theme.CreateImage()
	texture := m.driver.CreateTexture(im, 96)
	pict.SetTexture(texture)
	pict.SetExplicitSize(math.Size{32, 45})
	//	pict.SetMargin(math.CreateSpacing(4))

	return PaiImage{Image: pict, Pai: p}
}

var ueshita = 1 // ちょんちょん後の下段からなので1

func nextFunc(playerID int) pai.MJP {
	if ueshita == 2 {
		_taku.Add列And山越しFunc(1)
		ueshita = 0
	}

	p := _taku.Next(ueshita, playerID)

	// 引いた牌を非表示
	_yamaImage[_taku.PlayYamaIndex][ueshita][_taku.PlayRetuIndex].SetVisible(false)

	ueshita++
	return p
}

func Player(playerID int) *player.Player {
	for _, p := range _taku.Players {
		if p.PlayerID() == playerID {
			return &p
		}
	}
	return nil
}

func (m MainWindow) nextPaiImage(container gxui.LinearLayout, playerID int, mjPai pai.MJP) {
	paiImage := m.createPaiImage(mjPai)

	fmt.Println("nextPaiImage", playerID)
	// TODO: player1だけ操作可能にする
	if playerID == 1 {
		paiImage.OnClick(func(e gxui.MouseEvent) {
			ChangeState(CPUTurnState)

			fmt.Println("OnClick", playerID)

			Player(playerID).PaiDec(paiImage.Pai)
			container.RemoveChild(paiImage)

			_sutehaiLayers[playerID-1].AddChild(paiImage)

			nextPai := nextFunc(playerID)
			Player(playerID).PaiInc(nextPai)
			m.nextPaiImage(container, playerID, nextPai)
		})
	}

	for i := 0; i < container.ChildCount(); i++ {
		if container.ChildAt(i).(PaiImage).Pai > paiImage.Pai {
			container.AddChildAt(i, paiImage)
			return
		}
	}
	container.AddChild(paiImage)
}

func main() {
	flag.Parse()
	gl.StartDriver(appMain)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mpl/basicauth"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass

	childrenMu sync.Mutex
	children   []*os.Process
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, "The endpoints are /run, /kill, and /die.\n")
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window

type limitWriter struct {
	deadline time.Time
	limit    int
	sum      int

	bufMu sync.Mutex
	buf   *bytes.Buffer

	discardingMu sync.RWMutex
	discarding   bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return ioutil.Discard.Write(p)
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	n, err = lw.buf.Write(p)
	lw.bufMu.Unlock()
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discardingMu.Lock()
		lw.discarding = true
		lw.discardingMu.Unlock()
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return 0, io.EOF
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	defer lw.bufMu.Unlock()
	return lw.buf.Read(p)
}

func killChildren() {
	childrenMu.Lock()
	defer childrenMu.Unlock()
	for _, v := range children {
		if err := v.Kill(); err != nil {
			log.Printf("couldn't kill child: %v", err)
		}
	}
}

func handleKillAll(w http.ResponseWriter, r *http.Request) {
	killChildren()
	if _, err := io.Copy(w, strings.NewReader("They have left for a better world.")); err != nil {
		log.Print(err)
	}
}

func handleDie(w http.ResponseWriter, r *http.Request) {
	killChildren()
	sayonara := "The sweet embrace of death, finally."
	if _, err := io.Copy(w, strings.NewReader(sayonara)); err != nil {
		log.Print(err)
	}
	log.Print(sayonara)
	time.Sleep(time.Second)
	os.Exit(0)
}

// TODO(mpl): handler to list all [pid, time started]

// TODO(mpl): do some request suppressing
func handleCommand(w http.ResponseWriter, r *http.Request) {
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	log.Printf("Started %v with pid %v", args[0], cmd.Process.Pid)
	childrenMu.Lock()
	children = append(children, cmd.Process)
	childrenMu.Unlock()
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()

	http.Handle("/run", makeHandler(handleCommand))
	http.Handle("/kill", makeHandler(handleKillAll))
	http.Handle("/die", makeHandler(handleDie))
	if err := http.ListenAndServeTLS(
		*flagHost,
		filepath.Join(os.Getenv("HOME"), "keys", "cert.pem"),
		filepath.Join(os.Getenv("HOME"), "keys", "key.pem"),
		nil); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>added process creation rate limit<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mpl/basicauth"
)

const (
	idstring = "http://golang.org/pkg/http/#ListenAndServe"
)

var (
	flagHost     = flag.String("host", "0.0.0.0:8080", "listening port and hostname")
	flagHelp     = flag.Bool("h", false, "show this help")
	flagUserpass = flag.String("userpass", "", "optional username:password protection")
	flagCommand  = flag.String("command", "", "The command to run")
	flagRate     = flag.Duration("rate", time.Second, "To limit the number of processes created to no more than one per given duration. Set to 0 for no limit.")
)

var (
	rootdir, _ = os.Getwd()
	up         *basicauth.UserPass

	childrenMu sync.Mutex
	children   []*os.Process

	// TODO(mpl): rate limit per source ip instead of for all requests?
	lastRunMu sync.RWMutex
	lastRun   time.Time
)

func usage() {
	fmt.Fprintf(os.Stderr, "\t httprunner \n")
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, "The endpoints are /run, /kill, and /die.\n")
	os.Exit(2)
}

func makeHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				http.Error(w, e.Error(), http.StatusInternalServerError)
				return
			}
		}()
		w.Header().Set("Server", idstring)
		if isAllowed(r) {
			fn(w, r)
		} else {
			basicauth.SendUnauthorized(w, r, "httprunner")
		}
	}
}

func isAllowed(r *http.Request) bool {
	if *flagUserpass == "" {
		return true
	}
	return up.IsAllowed(r)
}

func initUserPass() {
	if *flagUserpass == "" {
		return
	}
	var err error
	up, err = basicauth.New(*flagUserpass)
	if err != nil {
		log.Fatal(err)
	}
}

// TODO(mpl): have a look at https://github.com/cespare/window

type limitWriter struct {
	deadline time.Time
	limit    int
	sum      int

	bufMu sync.Mutex
	buf   *bytes.Buffer

	discardingMu sync.RWMutex
	discarding   bool
}

func (lw limitWriter) Write(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return ioutil.Discard.Write(p)
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	n, err = lw.buf.Write(p)
	lw.bufMu.Unlock()
	lw.sum += n
	if lw.sum > lw.limit {
		lw.discardingMu.Lock()
		lw.discarding = true
		lw.discardingMu.Unlock()
	}
	return
}

func (lw limitWriter) Read(p []byte) (n int, err error) {
	lw.discardingMu.RLock()
	if lw.discarding {
		lw.discardingMu.RUnlock()
		return 0, io.EOF
	}
	lw.discardingMu.RUnlock()
	lw.bufMu.Lock()
	defer lw.bufMu.Unlock()
	return lw.buf.Read(p)
}

func killChildren() {
	childrenMu.Lock()
	defer childrenMu.Unlock()
	for _, v := range children {
		if err := v.Kill(); err != nil {
			log.Printf("couldn't kill child: %v", err)
		}
	}
}

func handleKillAll(w http.ResponseWriter, r *http.Request) {
	killChildren()
	if _, err := io.Copy(w, strings.NewReader("They have left for a better world.")); err != nil {
		log.Print(err)
	}
}

func handleDie(w http.ResponseWriter, r *http.Request) {
	killChildren()
	sayonara := "The sweet embrace of death, finally."
	if _, err := io.Copy(w, strings.NewReader(sayonara)); err != nil {
		log.Print(err)
	}
	log.Print(sayonara)
	time.Sleep(time.Second)
	os.Exit(0)
}

// TODO(mpl): handler to list all [pid, time started]

// TODO(mpl): do some request suppressing
func handleCommand(w http.ResponseWriter, r *http.Request) {
	if *flagRate != 0 {
		lastRunMu.RLock()
		if time.Now().Before(lastRun.Add(*flagRate)) {
			http.Error(w, "Command process creation is rate limited", http.StatusTooManyRequests)
			lastRunMu.RUnlock()
			return
		}
		lastRunMu.RUnlock()
	}
	// TODO(mpl): be less lazy about the doubled spaces, and probably other things.
	args := strings.Fields(*flagCommand)
	cmd := exec.Command(args[0], args[1:]...)
	var buf, berr bytes.Buffer
	lw := limitWriter{
		limit: 1 << 20,
		buf:   &buf,
	}
	stdout := io.MultiWriter(os.Stdout, lw)
	cmd.Stdout = stdout
	cmd.Stderr = &berr
	if err := cmd.Start(); err != nil {
		log.Printf("%v failed to start: %v, %v", args[0], err, berr.String())
		return
	}
	log.Printf("Started %v with pid %v", args[0], cmd.Process.Pid)
	childrenMu.Lock()
	children = append(children, cmd.Process)
	childrenMu.Unlock()
	lastRunMu.Lock()
	lastRun = time.Now()
	lastRunMu.Unlock()
	go func() {
		if err := cmd.Wait(); err != nil {
			log.Printf("%v failed: %v, %v", args[0], err, berr.String())
		}
	}()
	var bufout bytes.Buffer
	sendResponse := func(b *bytes.Buffer) {
		var response io.Reader
		if b.Len() > 0 {
			response = b
		} else {
			response = strings.NewReader("Command started but no output yet.")
		}
		if _, err := io.Copy(w, response); err != nil {
			log.Printf("response copy error: %v", err)
		}
	}
	var seenData bool
	// TODO(mpl): test if we could relax both these times now that we're sending the header asap.
	maxIdle := 200 * time.Millisecond
	t := time.After(1 * time.Second)
	lastDataTime := time.Now()
	for {
		select {
		case <-t:
			sendResponse(&bufout)
			return
		default:
		}
		n, err := io.Copy(&bufout, lw)
		if err != nil {
			log.Printf("output copy error: %v", err)
			break
		}
		if n > 0 {
			if !seenData {
				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
				w.WriteHeader(http.StatusOK)
				seenData = true
			}
			lastDataTime = time.Now()
		} else {
			if lastDataTime.Add(maxIdle).Before(time.Now()) {
				log.Printf("no output for more than %v, wrapping up.", maxIdle)
				break
			}
		}
	}
	sendResponse(&bufout)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *flagHelp {
		usage()
	}
	nargs := flag.NArg()
	if nargs > 0 {
		usage()
	}
	if *flagCommand == "" {
		fmt.Printf("No command to run")
		usage()
	}

	initUserPass()

	http.Handle("/run", makeHandler(handleCommand))
	http.Handle("/kill", makeHandler(handleKillAll))
	http.Handle("/die", makeHandler(handleDie))
	if err := http.ListenAndServeTLS(
		*flagHost,
		filepath.Join(os.Getenv("HOME"), "keys", "cert.pem"),
		filepath.Join(os.Getenv("HOME"), "keys", "key.pem"),
		nil); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"time"

	irukaLogger "github.com/bottlenose-inc/go-iruka/logger" // go-iruka bunyan-style logger package
	"github.com/bottlenose-inc/go-iruka/metrics"            // go-iruka Prometheus metrics package
	"github.com/codegangsta/cli"                            // CLI helper
	"github.com/prometheus/client_golang/prometheus"        // Prometheus client library
)

var (
	scrapeInterval       int
	nsqdUrl              string
	allTopics            []string
	logger               *irukaLogger.Logger
	depthGaugeVec        *prometheus.GaugeVec
	inFlightGaugeVec     *prometheus.GaugeVec
	backendDepthGaugeVec *prometheus.GaugeVec
	timeoutGaugeVec      *prometheus.GaugeVec
	requeueGaugeVec      *prometheus.GaugeVec
	deferredGaugeVec     *prometheus.GaugeVec
	messageCountGaugeVec *prometheus.GaugeVec
	clientCountGaugeVec  *prometheus.GaugeVec
	channelCountGaugeVec *prometheus.GaugeVec
)

func main() {
	app := cli.NewApp()
	app.Version = "0.1.0"
	app.Name = "nsqd-prometheus-exporter"
	app.Usage = "Scrapes nsqd stats and serves them up as Prometheus metrics"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "nsqdUrl, n",
			Value:  "http://localhost:4151",
			Usage:  "URL of nsqd to export stats from",
			EnvVar: "NSQD_URL",
		},
		cli.StringFlag{
			Name:   "listenPort, lp",
			Value:  "30000",
			Usage:  "Port on which prometheus will expose metrics",
			EnvVar: "LISTEN_PORT",
		},
		cli.StringFlag{
			Name:   "scrapeInterval, s",
			Value:  "30",
			Usage:  "How often (in seconds) nsqd stats should be scraped",
			EnvVar: "SCRAPE_INTERVAL",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "run",
			Usage: "./nsqd-prometheus-exporter [options] run",
			Action: func(c *cli.Context) {
				// Initialize logger
				var err error
				logger, err = irukaLogger.NewLogger(app.Name + " v" + app.Version)
				if err != nil {
					log.Fatal("Unable to initialize go-iruka logger, exiting: " + err.Error())
				}

				// Set and validate configuration
				nsqdUrl = c.GlobalString("nsqdUrl")
				if nsqdUrl == "" {
					logger.Warning("Invalid nsqd URL set, continuing with default (http://localhost:4151)")
					nsqdUrl = "http://localhost:4151"
				}
				scrapeInterval = c.GlobalInt("scrapeInterval")
				if scrapeInterval < 1 {
					logger.Warning("Invalid scrape interval set, continuing with default (30s)")
					scrapeInterval = 30
				}

				// Initialize Prometheus metrics
				var emptyMap map[string]string
				labels := []string{"type", "topic", "paused", "channel"}
				// # HELP nsq_depth Queue depth
				// # TYPE nsq_depth gauge
				depthGaugeVec, _ = metrics.CreateGaugeVector("nsq_depth", "", "", "Queue depth", emptyMap, labels)
				// # HELP nsq_backend_depth Queue backend depth
				// # TYPE nsq_backend_depth gauge
				backendDepthGaugeVec, _ = metrics.CreateGaugeVector("nsq_backend_depth", "", "", "Queue backend depth", emptyMap, labels)
				// # HELP nsq_in_flight_count In flight count
				// # TYPE nsq_in_flight_count gauge
				inFlightGaugeVec, _ = metrics.CreateGaugeVector("nsq_in_flight_count", "", "", "In flight count", emptyMap, labels)
				// # HELP nsq_timeout_count Timeout count
				// # TYPE nsq_timeout_count gauge
				timeoutGaugeVec, _ = metrics.CreateGaugeVector("nsq_timeout_count", "", "", "Timeout count", emptyMap, labels)
				// # HELP nsq_requeue_count Requeue Count
				// # TYPE nsq_requeue_count gauge
				requeueGaugeVec, _ = metrics.CreateGaugeVector("nsq_requeue_count", "", "", "Requeue Count", emptyMap, labels)
				// # HELP nsq_deferred_count Deferred count
				// # TYPE nsq_deferred_count gauge
				deferredGaugeVec, _ = metrics.CreateGaugeVector("nsq_deferred_count", "", "", "Deferred count", emptyMap, labels)
				// # HELP nsq_message_count Queue message count
				// # TYPE nsq_message_count gauge
				messageCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_message_count", "", "", "Queue message count", emptyMap, labels)
				// # HELP nsq_client_count Number of clients
				// # TYPE nsq_client_count gauge
				clientCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_client_count", "", "", "Number of clients", emptyMap, labels)
				// # HELP nsq_channel_count Number of channels
				// # TYPE nsq_channel_count gauge
				channelCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_channel_count", "", "", "Number of channels", emptyMap, []string{"type", "topic", "paused"})

				go fetchAndSetStats()

				err = metrics.StartPrometheusMetricsServer(app.Name, logger, c.GlobalInt("listenPort"))
				if err != nil {
					os.Exit(1)
				}
			},
		},
	}

	app.Run(os.Args)
}

// fetchAndSetStats scrapes stats from nsqd and updates all the Prometheus metrics on the provided interval.
func fetchAndSetStats() {
	for {
		// Fetch stats
		stats, err := getNsqdStats(nsqdUrl)
		if err != nil {
			logger.Fatal("Error scraping stats from nsqd: " + err.Error())
			os.Exit(1)
		}

		// Exit if any "dead" topics are detected
		for _, topicName := range allTopics {
			found := false
			for _, topic := range stats.Topics {
				if topicName == topic.Name {
					found = true
					break
				}
			}
			if !found {
				logger.Fatal("At least one old topic no longer included in nsqd stats - exiting")
				os.Exit(0)
			}
		}

		// Loop through topics and set metrics
		allTopics = nil // Rebuild list of all topics
		for _, topic := range stats.Topics {
			allTopics = append(allTopics, topic.Name)
			paused := "false"
			if topic.Paused {
				paused = "true"
			}
			depthGaugeVec.WithLabelValues("topic", topic.Name, paused, "").Set(float64(topic.Depth))
			backendDepthGaugeVec.WithLabelValues("topic", topic.Name, paused, "").Set(float64(topic.BackendDepth))
			channelCountGaugeVec.WithLabelValues("topic", topic.Name, paused).Set(float64(len(topic.Channels)))

			// Loop through a topic's channels and set metrics
			for _, channel := range topic.Channels {
				paused = "false"
				if channel.Paused {
					paused = "true"
				}
				depthGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.Depth))
				backendDepthGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.BackendDepth))
				inFlightGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.InFlightCount))
				timeoutGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.TimeoutCount))
				requeueGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.RequeueCount))
				deferredGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.DeferredCount))
				messageCountGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.MessageCount))
				clientCountGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(len(channel.Clients)))
			}
		}

		// Scrape every scrapeInterval
		time.Sleep(time.Duration(scrapeInterval) * time.Second)
	}
}
<commit_msg>add build info metric<commit_after>package main

import (
	"log"
	"os"
	"time"

	irukaLogger "github.com/bottlenose-inc/go-iruka/logger" // go-iruka bunyan-style logger package
	"github.com/bottlenose-inc/go-iruka/metrics"            // go-iruka Prometheus metrics package
	"github.com/codegangsta/cli"                            // CLI helper
	"github.com/prometheus/client_golang/prometheus"        // Prometheus client library
)

var (
	scrapeInterval       int
	nsqdUrl              string
	allTopics            []string
	logger               *irukaLogger.Logger
	depthGaugeVec        *prometheus.GaugeVec
	inFlightGaugeVec     *prometheus.GaugeVec
	backendDepthGaugeVec *prometheus.GaugeVec
	timeoutGaugeVec      *prometheus.GaugeVec
	requeueGaugeVec      *prometheus.GaugeVec
	deferredGaugeVec     *prometheus.GaugeVec
	messageCountGaugeVec *prometheus.GaugeVec
	clientCountGaugeVec  *prometheus.GaugeVec
	channelCountGaugeVec *prometheus.GaugeVec
	buildInfoMetric      *prometheus.GaugeVec
)

func main() {
	app := cli.NewApp()
	app.Version = "0.1.0"
	app.Name = "nsqd-prometheus-exporter"
	app.Usage = "Scrapes nsqd stats and serves them up as Prometheus metrics"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "nsqdUrl, n",
			Value:  "http://localhost:4151",
			Usage:  "URL of nsqd to export stats from",
			EnvVar: "NSQD_URL",
		},
		cli.StringFlag{
			Name:   "listenPort, lp",
			Value:  "30000",
			Usage:  "Port on which prometheus will expose metrics",
			EnvVar: "LISTEN_PORT",
		},
		cli.StringFlag{
			Name:   "scrapeInterval, s",
			Value:  "30",
			Usage:  "How often (in seconds) nsqd stats should be scraped",
			EnvVar: "SCRAPE_INTERVAL",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "run",
			Usage: "./nsqd-prometheus-exporter [options] run",
			Action: func(c *cli.Context) {
				// Initialize logger
				var err error
				logger, err = irukaLogger.NewLogger(app.Name + " v" + app.Version)
				if err != nil {
					log.Fatal("Unable to initialize go-iruka logger, exiting: " + err.Error())
				}

				// Set and validate configuration
				nsqdUrl = c.GlobalString("nsqdUrl")
				if nsqdUrl == "" {
					logger.Warning("Invalid nsqd URL set, continuing with default (http://localhost:4151)")
					nsqdUrl = "http://localhost:4151"
				}
				scrapeInterval = c.GlobalInt("scrapeInterval")
				if scrapeInterval < 1 {
					logger.Warning("Invalid scrape interval set, continuing with default (30s)")
					scrapeInterval = 30
				}

				// Initialize Prometheus metrics
				var emptyMap map[string]string
				labels := []string{"type", "topic", "paused", "channel"}
				buildInfoMetric, _ = metrics.CreateGaugeVector("nsqd_prometheus_exporter_build_info", "", "",
					"nsqd-prometheus-exporter build info", emptyMap, []string{"version"})
				buildInfoMetric.WithLabelValues(app.Version).Set(1)
				// # HELP nsq_depth Queue depth
				// # TYPE nsq_depth gauge
				depthGaugeVec, _ = metrics.CreateGaugeVector("nsq_depth", "", "", "Queue depth", emptyMap, labels)
				// # HELP nsq_backend_depth Queue backend depth
				// # TYPE nsq_backend_depth gauge
				backendDepthGaugeVec, _ = metrics.CreateGaugeVector("nsq_backend_depth", "", "", "Queue backend depth", emptyMap, labels)
				// # HELP nsq_in_flight_count In flight count
				// # TYPE nsq_in_flight_count gauge
				inFlightGaugeVec, _ = metrics.CreateGaugeVector("nsq_in_flight_count", "", "", "In flight count", emptyMap, labels)
				// # HELP nsq_timeout_count Timeout count
				// # TYPE nsq_timeout_count gauge
				timeoutGaugeVec, _ = metrics.CreateGaugeVector("nsq_timeout_count", "", "", "Timeout count", emptyMap, labels)
				// # HELP nsq_requeue_count Requeue Count
				// # TYPE nsq_requeue_count gauge
				requeueGaugeVec, _ = metrics.CreateGaugeVector("nsq_requeue_count", "", "", "Requeue Count", emptyMap, labels)
				// # HELP nsq_deferred_count Deferred count
				// # TYPE nsq_deferred_count gauge
				deferredGaugeVec, _ = metrics.CreateGaugeVector("nsq_deferred_count", "", "", "Deferred count", emptyMap, labels)
				// # HELP nsq_message_count Queue message count
				// # TYPE nsq_message_count gauge
				messageCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_message_count", "", "", "Queue message count", emptyMap, labels)
				// # HELP nsq_client_count Number of clients
				// # TYPE nsq_client_count gauge
				clientCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_client_count", "", "", "Number of clients", emptyMap, labels)
				// # HELP nsq_channel_count Number of channels
				// # TYPE nsq_channel_count gauge
				channelCountGaugeVec, _ = metrics.CreateGaugeVector("nsq_channel_count", "", "", "Number of channels", emptyMap, []string{"type", "topic", "paused"})

				go fetchAndSetStats()

				err = metrics.StartPrometheusMetricsServer(app.Name, logger, c.GlobalInt("listenPort"))
				if err != nil {
					os.Exit(1)
				}
			},
		},
	}

	app.Run(os.Args)
}

// fetchAndSetStats scrapes stats from nsqd and updates all the Prometheus metrics on the provided interval.
func fetchAndSetStats() {
	for {
		// Fetch stats
		stats, err := getNsqdStats(nsqdUrl)
		if err != nil {
			logger.Fatal("Error scraping stats from nsqd: " + err.Error())
			os.Exit(1)
		}

		// Exit if any "dead" topics are detected
		for _, topicName := range allTopics {
			found := false
			for _, topic := range stats.Topics {
				if topicName == topic.Name {
					found = true
					break
				}
			}
			if !found {
				logger.Fatal("At least one old topic no longer included in nsqd stats - exiting")
				os.Exit(0)
			}
		}

		// Loop through topics and set metrics
		allTopics = nil // Rebuild list of all topics
		for _, topic := range stats.Topics {
			allTopics = append(allTopics, topic.Name)
			paused := "false"
			if topic.Paused {
				paused = "true"
			}
			depthGaugeVec.WithLabelValues("topic", topic.Name, paused, "").Set(float64(topic.Depth))
			backendDepthGaugeVec.WithLabelValues("topic", topic.Name, paused, "").Set(float64(topic.BackendDepth))
			channelCountGaugeVec.WithLabelValues("topic", topic.Name, paused).Set(float64(len(topic.Channels)))

			// Loop through a topic's channels and set metrics
			for _, channel := range topic.Channels {
				paused = "false"
				if channel.Paused {
					paused = "true"
				}
				depthGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.Depth))
				backendDepthGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.BackendDepth))
				inFlightGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.InFlightCount))
				timeoutGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.TimeoutCount))
				requeueGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.RequeueCount))
				deferredGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.DeferredCount))
				messageCountGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(channel.MessageCount))
				clientCountGaugeVec.WithLabelValues("channel", channel.Name, paused, channel.Name).Set(float64(len(channel.Clients)))
			}
		}

		// Scrape every scrapeInterval
		time.Sleep(time.Duration(scrapeInterval) * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/ainoya/sewer/drainer"
	"github.com/ainoya/sewer/flusher"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()

	app.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "drain",
			Value: &cli.StringSlice{},
			Usage: "Destination where you want to notify piped messages",
		},
		cli.StringFlag{
			Name:  "template",
			Value: "{{ .Message }}",
			Usage: "Template format (STDIN is expanded as {{ .Message }} in the template)",
		},
	}

	app.Action = func(c *cli.Context) error {
		info, _ := os.Stdin.Stat()

		if (info.Mode() & os.ModeCharDevice) == os.ModeCharDevice {
			fmt.Println("The command is intended to work with pipes.")
			fmt.Println("Usage:")
			fmt.Println("cat yourfile.txt | sewer --drain=github")
			return nil
		}

		reader := bufio.NewReader(os.Stdin)
		tmpl := c.String("template")

		drains := c.StringSlice("drain")
		fmt.Printf("got drains %+v", drains)
		drainers, err := newDrainers(drains)

		if err != nil {
			fmt.Println(err)
			return nil
		}

		f := flusher.NewFlusher(drainers, reader, tmpl)
		err = f.Flush()

		if err != nil {
			fmt.Println(err)
		}

		return nil
	}

	app.Run(os.Args)
}

func newDrainers(drains []string) ([]drainer.Drainer, error) {
	var drainers []drainer.Drainer
	for _, drain := range drains {
		var dnr drainer.Drainer
		var err error
		fmt.Printf("drain setting %s\n", drain)
		switch drain {
		case "github":
			dnr, err = drainer.NewGitHubDrainer()
		case "slack":
			dnr, err = drainer.NewSlackDrainer()
		default:
			err = fmt.Errorf("drain type %s is not defined.", drain)
		}
		if err != nil {
			return nil, err
		}

		drainers = append(drainers, dnr)
	}

	fmt.Printf("got drainers %+v\n", drainers)
	return drainers, nil
}
<commit_msg>remove debug print<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/ainoya/sewer/drainer"
	"github.com/ainoya/sewer/flusher"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()

	app.Flags = []cli.Flag{
		cli.StringSliceFlag{
			Name:  "drain",
			Value: &cli.StringSlice{},
			Usage: "Destination where you want to notify piped messages",
		},
		cli.StringFlag{
			Name:  "template",
			Value: "{{ .Message }}",
			Usage: "Template format (STDIN is expanded as {{ .Message }} in the template)",
		},
	}

	app.Action = func(c *cli.Context) error {
		info, _ := os.Stdin.Stat()

		if (info.Mode() & os.ModeCharDevice) == os.ModeCharDevice {
			fmt.Println("The command is intended to work with pipes.")
			fmt.Println("Usage:")
			fmt.Println("cat yourfile.txt | sewer --drain=github")
			return nil
		}

		reader := bufio.NewReader(os.Stdin)
		tmpl := c.String("template")

		drains := c.StringSlice("drain")
		drainers, err := newDrainers(drains)

		if err != nil {
			fmt.Println(err)
			return nil
		}

		f := flusher.NewFlusher(drainers, reader, tmpl)
		err = f.Flush()

		if err != nil {
			fmt.Println(err)
		}

		return nil
	}

	app.Run(os.Args)
}

func newDrainers(drains []string) ([]drainer.Drainer, error) {
	var drainers []drainer.Drainer
	for _, drain := range drains {
		var dnr drainer.Drainer
		var err error
		switch drain {
		case "github":
			dnr, err = drainer.NewGitHubDrainer()
		case "slack":
			dnr, err = drainer.NewSlackDrainer()
		default:
			err = fmt.Errorf("drain type %s is not defined.", drain)
		}
		if err != nil {
			return nil, err
		}

		drainers = append(drainers, dnr)
	}

	return drainers, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/gracecat/cmdreport/execute"
	"github.com/gracecat/cmdreport/report"
	"io/ioutil"
	"os"
	"strings"
)

var (
	version   = "0.0.0"
	author    = os.Getenv("CMDREPORT_AUTHOR")
	serverURL = os.Getenv("CMDREPORT_SERVER")
)

func main() {
	// create and parse flags
	versionFlag := flag.Bool("v", false, "print the version and exit")
	envFlag := flag.Bool("e", false, "print the env vars and exit")
	flag.Usage = func() {
		fmt.Printf("\ncmdreport is a tool to report stdout/stderr messages to a server.\n\n")
		fmt.Printf("Usage: cmdreport [cmd] | <flag>\n\n")
		fmt.Printf("cmd:\n")
		fmt.Printf("  run a terminal command\n\n")
		fmt.Printf("flag:\n")
		flag.PrintDefaults()
		fmt.Println("")
	}
	flag.Parse()
	if *versionFlag {
		fmt.Println(version)
		os.Exit(0)
	}
	if *envFlag {
		fmt.Println("CMDREPORT_AUTHOR = " + author)
		fmt.Println("CMDREPORT_SERVER = " + serverURL)
		os.Exit(0)
	}

	if len(os.Args) == 1 {
		fmt.Println("missing command...")
		os.Exit(1)
	}
	cmd := os.Args[1]
	cmdArgs := os.Args[2:]
	cmdArgsStr := strings.Join(cmdArgs, " ")
	fmt.Printf("try to run '%s %s'\n", cmd, cmdArgsStr)

	stdout, stderr, err := execute.Execute(cmd, cmdArgs...)
	if err != nil {
		fmt.Println("execution error", err)
		os.Exit(127)
	}

	if len(stdout) != 0 {
		fmt.Println("\nstdout:")
		printStringList(stdout)
	}
	if len(stderr) != 0 {
		fmt.Println("\nstderr:")
		printStringList(stderr)
	}

	// post the report...
	resp, err := report.Post(serverURL, author, cmd, cmdArgs, stdout, stderr)
	if err != nil {
		fmt.Println(err)
		os.Exit(2)
	}
	fmt.Printf("server sesponse with status %s\n", resp.Status)
	if resp.StatusCode == 200 {
		fmt.Printf("successful send report (%s)\n", resp.Status)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println(string(body))
	}
}

// pretty print a string array
func printStringList(s []string) {
	for i, d := range s {
		fmt.Printf("%d\t%s\n", i, d)
	}
	fmt.Println("")
}
<commit_msg>Bump version<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/gracecat/cmdreport/execute"
	"github.com/gracecat/cmdreport/report"
	"io/ioutil"
	"os"
	"strings"
)

var (
	version   = "0.1.0"
	author    = os.Getenv("CMDREPORT_AUTHOR")
	serverURL = os.Getenv("CMDREPORT_SERVER")
)

func main() {
	// create and parse flags
	versionFlag := flag.Bool("v", false, "print the version and exit")
	envFlag := flag.Bool("e", false, "print the env vars and exit")
	flag.Usage = func() {
		fmt.Printf("\ncmdreport is a tool to report stdout/stderr messages to a server.\n\n")
		fmt.Printf("Usage: cmdreport [cmd] | <flag>\n\n")
		fmt.Printf("cmd:\n")
		fmt.Printf("  run a terminal command\n\n")
		fmt.Printf("flag:\n")
		flag.PrintDefaults()
		fmt.Println("")
	}
	flag.Parse()
	if *versionFlag {
		fmt.Println(version)
		os.Exit(0)
	}
	if *envFlag {
		fmt.Println("CMDREPORT_AUTHOR = " + author)
		fmt.Println("CMDREPORT_SERVER = " + serverURL)
		os.Exit(0)
	}

	if len(os.Args) == 1 {
		fmt.Println("missing command...")
		os.Exit(1)
	}
	cmd := os.Args[1]
	cmdArgs := os.Args[2:]
	cmdArgsStr := strings.Join(cmdArgs, " ")
	fmt.Printf("try to run '%s %s'\n", cmd, cmdArgsStr)

	stdout, stderr, err := execute.Execute(cmd, cmdArgs...)
	if err != nil {
		fmt.Println("execution error", err)
		os.Exit(127)
	}

	if len(stdout) != 0 {
		fmt.Println("\nstdout:")
		printStringList(stdout)
	}
	if len(stderr) != 0 {
		fmt.Println("\nstderr:")
		printStringList(stderr)
	}

	// post the report...
	resp, err := report.Post(serverURL, author, cmd, cmdArgs, stdout, stderr)
	if err != nil {
		fmt.Println(err)
		os.Exit(2)
	}
	fmt.Printf("server sesponse with status %s\n", resp.Status)
	if resp.StatusCode == 200 {
		fmt.Printf("successful send report (%s)\n", resp.Status)
		body, _ := ioutil.ReadAll(resp.Body)
		fmt.Println(string(body))
	}
}

// pretty print a string array
func printStringList(s []string) {
	for i, d := range s {
		fmt.Printf("%d\t%s\n", i, d)
	}
	fmt.Println("")
}
<|endoftext|>


<commit_before>package main

// http://tsdb.kronos.d:4242/#start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95&o=&yrange=%5B0:%5D&wxh=1420x608&style=linespoint
// http://tsdb.kronos.d:4242/api/query/?start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95

// http://tsdb.kronos.d:4242/#start=2016/02/03-09:23:00&end=2016/02/10-09:23:58&m=mimmax:gauges.do.sphinx.lag&o=&yrange=%5B0:%5D&wxh=1380x636&style=linespoint
import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"time"
)

const tsdbHost = "http://tsdb.kronos.d:4242"

const htmlHeader = `<!DOCTYPE html>

		<title>chart</title>

`

type tsdbResponse struct {
	Metric        string             `json:"metric"`
	Tags          map[string]string  `json:"Tags"`
	AggregateTags []string           `json:"aggregateTags"`
	DataPoints    map[string]float64 `json:"dps"`
}

type DataPoint struct {
	Time  time.Time
	Value float64
}

type DataPoints []DataPoint

func (dps DataPoints) Len() int           { return len(dps) }
func (dps DataPoints) Swap(i, j int)      { dps[i], dps[j] = dps[j], dps[i] }
func (dps DataPoints) Less(i, j int) bool { return dps[i].Time.Before(dps[j].Time) }

type chartData struct {
	Date    string  `json:"date"`
	Value   float64 `json:"val"`
	Average float64 `json:"avg"`
	Sigma   float64 `json:"sigma"`
}

func main() {
	port := "80"
	if os.Getenv("PORT") != "" {
		port = os.Getenv("PORT")
	}

	// Простой хендлер, который показывает хост и текущее время
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()

		start := r.URL.Query().Get("start")
		end := r.URL.Query().Get("end")
		m := r.URL.Query().Get("m")
		periodRaw := r.URL.Query().Get("period")

		if m == "" {
			fmt.Fprintln(w, "Берём урл от tsdb, заменяем хост и порт на текущий, вместо '/#start' делаем '/?start'")
			return
		}

		if periodRaw == "" {
			periodRaw = "120"
		}
		period, err := strconv.Atoi(periodRaw)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to atoi: %v", err), http.StatusInternalServerError)
		}

		result := bytes.NewBufferString(htmlHeader)
		chart, err := detectAnomalyes(start, end, m, period)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to detect: %v", err), http.StatusInternalServerError)
		}

		if err := json.NewEncoder(result).Encode(chart); err != nil {
			http.Error(w, fmt.Sprintf("Failed to encode: %v", err), http.StatusInternalServerError)
		}
		result.WriteString(htmlFooter)

		/*
			{
				"date": "2016-02-09 09:20:00",
				"id": "anomaly-01",
				"lineAlpha": 0.5,
				"lineColor": "#FF0000",
				"lineThickness": 5
			}
		*/

		result.WriteTo(w)

		// Ну и access.log для наглядности
		log.Printf("%s %q %v\n", r.Method, r.URL.String(), time.Since(t))
	})
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func detectAnomalyes(start, end, m string, period int) ([]chartData, error) {
	path := url.Values{}
	// path.Add("start", "2016/02/09-08:00:00")
	// path.Add("end", "2016/02/09-11:00:00")
	// path.Add("m", "mimmax:php.timers.mongo.p95")
	path.Add("start", start)
	path.Add("end", end)
	path.Add("m", m)

	resp, err := http.Get(fmt.Sprintf("%s/api/query/?%s", tsdbHost, path.Encode()))
	if err != nil {
		return nil, fmt.Errorf("failed to get data from TSDB: %v", err)
	}

	var data []tsdbResponse
	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
		return nil, fmt.Errorf("failed to decode TSDB response: %v", err)
	}

	dps := make(DataPoints, 0)
	for ts, val := range data[0].DataPoints {
		timestamp, err := strconv.Atoi(ts)
		if err != nil {
			continue
		}
		dp := DataPoint{
			Time:  time.Unix(int64(timestamp), 0),
			Value: val,
		}
		dps = append(dps, dp)
	}

	sort.Sort(dps)

	var movingAverage float64
	var sigma float64
	vals := make([]float64, period)

	chart := make([]chartData, 0)
	for idx, cdp := range dps {
		if idx <= period {
			continue
		}
		points := dps[idx-period : idx]
		for i, dp := range points {
			vals[i] = dp.Value
		}

		movingAverage = average(vals)
		sigma = div3Sigma(vals)
		// if sigma > 1 || sigma < -1 {
		// 	fmt.Printf("Panic! %v", cdp)
		// }
		//fmt.Printf("%v,%v,%v,%v\n", cdp.Time.Format("2006-01-02 15:04:05"), cdp.Value, movingAverage, sigma)

		chart = append(chart, chartData{
			cdp.Time.Format("2006-01-02 15:04:05"),
			cdp.Value,
			movingAverage,
			sigma,
		})
	}

	return chart, nil
}

// Get the average of float64 values.
func average(vals []float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		sum += vals[i]
	}
	return sum / float64(len(vals))
}

// Get the standard deviation of float64 values, with
// an input average.
func stdDev(vals []float64, avg float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		dis := vals[i] - avg
		sum += dis * dis
	}
	return math.Sqrt(sum / float64(len(vals)))
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func div3Sigma(vals []float64) float64 {
	if len(vals) == 0 {
		// Values empty.
		return 0
	}

	// Values average and standard deviation.
	avg := average(vals)
	std := stdDev(vals, avg)

	// // Set metric score
	// if len(vals) <= int(d.cfg.Detector.LeastCount) {
	// 	// Values not enough.
	// 	m.Score = 0
	// 	return
	// }

	last := vals[len(vals)-1]
	if std == 0 {
		switch {
		case last == avg:
			return 0
		case last > avg:
			return 1
		case last < avg:
			return -1
		}
		return 0
	}
	// 3-sigma
	return (last - avg) / (3 * std)
}
<commit_msg>detect anomalyes<commit_after>package main

// http://tsdb.kronos.d:4242/#start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95&o=&yrange=%5B0:%5D&wxh=1420x608&style=linespoint
// http://tsdb.kronos.d:4242/api/query/?start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95

// http://tsdb.kronos.d:4242/#start=2016/02/03-09:23:00&end=2016/02/10-09:23:58&m=mimmax:gauges.do.sphinx.lag&o=&yrange=%5B0:%5D&wxh=1380x636&style=linespoint
import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"time"
)

const tsdbHost = "http://tsdb.kronos.d:4242"

const htmlHeader = `<!DOCTYPE html>

		<title>chart</title>

`

type tsdbResponse struct {
	Metric        string             `json:"metric"`
	Tags          map[string]string  `json:"Tags"`
	AggregateTags []string           `json:"aggregateTags"`
	DataPoints    map[string]float64 `json:"dps"`
}

type DataPoint struct {
	Time  time.Time
	Value float64
}

type DataPoints []DataPoint

func (dps DataPoints) Len() int           { return len(dps) }
func (dps DataPoints) Swap(i, j int)      { dps[i], dps[j] = dps[j], dps[i] }
func (dps DataPoints) Less(i, j int) bool { return dps[i].Time.Before(dps[j].Time) }

type chartData struct {
	Date    string  `json:"date"`
	Value   float64 `json:"val"`
	Average float64 `json:"avg"`
	Sigma   float64 `json:"sigma"`
}

/*
	{
		"date": "2016-02-09 09:20:00",
		"id": "anomaly-01",
		"lineAlpha": 0.5,
		"lineColor": "#FF0000",
		"lineThickness": 5
	}
*/
type anomalyData struct {
	ID        string  `json:"id"`
	Date      string  `json:"date",omitempty`
	Alpha     float64 `json:"lineAlpha",omitempty`
	Color     string  `json:"lineColor"`
	Thickness int64   `json:"lineThickness"`
	Value     int64   `json:"value",omitempty`
	Axis      string  `json:"valueAxis",omitempty`
}

func main() {
	port := "80"
	if os.Getenv("PORT") != "" {
		port = os.Getenv("PORT")
	}

	// Простой хендлер, который показывает хост и текущее время
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()

		start := r.URL.Query().Get("start")
		end := r.URL.Query().Get("end")
		m := r.URL.Query().Get("m")
		periodRaw := r.URL.Query().Get("period")

		if m == "" {
			fmt.Fprintln(w, "Берём урл от tsdb, заменяем хост и порт на текущий, вместо '/#start' делаем '/?start'")
			return
		}

		if periodRaw == "" {
			periodRaw = "120"
		}
		period, err := strconv.Atoi(periodRaw)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to atoi: %v", err), http.StatusInternalServerError)
		}

		result := bytes.NewBufferString(htmlHeader)
		chart, _, err := detectAnomalyes(start, end, m, period)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to detect: %v", err), http.StatusInternalServerError)
		}

		if err := json.NewEncoder(result).Encode(chart); err != nil {
			http.Error(w, fmt.Sprintf("Failed to encode: %v", err), http.StatusInternalServerError)
		}
		result.WriteString(htmlFooter)

		/*
			{
				"date": "2016-02-09 09:20:00",
				"id": "anomaly-01",
				"lineAlpha": 0.5,
				"lineColor": "#FF0000",
				"lineThickness": 5
			}
		*/

		result.WriteTo(w)

		// Ну и access.log для наглядности
		log.Printf("%s %q %v\n", r.Method, r.URL.String(), time.Since(t))
	})
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func detectAnomalyes(start, end, m string, period int) ([]chartData, []anomalyData, error) {
	path := url.Values{}
	// path.Add("start", "2016/02/09-08:00:00")
	// path.Add("end", "2016/02/09-11:00:00")
	// path.Add("m", "mimmax:php.timers.mongo.p95")
	path.Add("start", start)
	path.Add("end", end)
	path.Add("m", m)

	resp, err := http.Get(fmt.Sprintf("%s/api/query/?%s", tsdbHost, path.Encode()))
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get data from TSDB: %v", err)
	}

	var data []tsdbResponse
	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
		return nil, nil, fmt.Errorf("failed to decode TSDB response: %v", err)
	}

	dps := make(DataPoints, 0)
	for ts, val := range data[0].DataPoints {
		timestamp, err := strconv.Atoi(ts)
		if err != nil {
			continue
		}
		dp := DataPoint{
			Time:  time.Unix(int64(timestamp), 0),
			Value: val,
		}
		dps = append(dps, dp)
	}

	sort.Sort(dps)

	var movingAverage float64
	var sigma float64
	vals := make([]float64, period)

	chart := make([]chartData, 0)
	anomalyes := make([]anomalyData, 0)

	loc, _ := time.LoadLocation("Asia/Novosibirsk")
	for idx, cdp := range dps {
		if idx <= period {
			continue
		}
		points := dps[idx-period : idx]
		for i, dp := range points {
			vals[i] = dp.Value
		}

		movingAverage = average(vals)
		sigma = div3Sigma(vals)
		anomalyIdx := 0
		if sigma > 1.2 || sigma < -1.2 {
			anomalyIdx++
			anomalyes = append(anomalyes, anomalyData{
				ID:        fmt.Sprintf("anomaly-%d", anomalyIdx),
				Date:      cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
				Alpha:     0.5,
				Color:     "#FF0000",
				Thickness: 5,
			})
		}
		//fmt.Printf("%v,%v,%v,%v\n", cdp.Time.Format("2006-01-02 15:04:05"), cdp.Value, movingAverage, sigma)

		chart = append(chart, chartData{
			cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
			cdp.Value,
			movingAverage,
			sigma,
		})
	}

	return chart, anomalyes, nil
}

// Get the average of float64 values.
func average(vals []float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		sum += vals[i]
	}
	return sum / float64(len(vals))
}

// Get the standard deviation of float64 values, with
// an input average.
func stdDev(vals []float64, avg float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		dis := vals[i] - avg
		sum += dis * dis
	}
	return math.Sqrt(sum / float64(len(vals)))
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func div3Sigma(vals []float64) float64 {
	if len(vals) == 0 {
		// Values empty.
		return 0
	}

	// Values average and standard deviation.
	avg := average(vals)
	std := stdDev(vals, avg)

	// // Set metric score
	// if len(vals) <= int(d.cfg.Detector.LeastCount) {
	// 	// Values not enough.
	// 	m.Score = 0
	// 	return
	// }

	last := vals[len(vals)-1]
	if std == 0 {
		switch {
		case last == avg:
			return 0
		case last > avg:
			return 1
		case last < avg:
			return -1
		}
		return 0
	}
	// 3-sigma
	return (last - avg) / (3 * std)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"time"
)

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func OutputPath() string {
	return "horoscopes"
}

func periodically(fn func(time.Time)) *time.Ticker {
	ticker := time.NewTicker(time.Hour * 24 * 7)
	go func() {
		for t := range ticker.C {
			fn(t)
		}
	}()
	return ticker
}

func main() {
	seedPtr := flag.Bool("seed", false, "perform the initial seed")
	serverPtr := flag.Bool("server", false, "start the web server")
	portPtr := flag.String("port", ":8000", "port to run the web server")
	releasePtr := flag.Bool("release", false, "set the mode to release")
	debugPtr := flag.Bool("debug", false, "enable debugging mode")

	flag.Parse()

	seed := *seedPtr
	server := *serverPtr
	port := *portPtr
	release := *releasePtr
	debug := *debugPtr

	if seed {
		fmt.Println("Fetching all horoscopes...")
		SeedHoroscopes()
	} else {
		fmt.Println("Fetching current horoscopes...")
		FetchHoroscopes()
	}

	if server {
		fmt.Print("Setting up store... ")
		s := Store()
		s.BuildIndexes()
		fmt.Println("done!")

		if debug {
			data, _ := json.MarshalIndent(s.weekIndex, "", "  ")
			fmt.Println(string(data), "", "  ")
		}

		fmt.Print("Scheduling periodic updates... ")
		periodically(func(t time.Time) {
			FetchHoroscopes()
			s.BuildIndexes()
		})
		fmt.Println("done!")

		fmt.Println("Bootstrapping server... done!")
		if debug {
			Server(s, false).Run(port)
		} else {
			Server(s, release).Run(port)
		}
	}
}
<commit_msg>Catch error<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"time"
)

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func OutputPath() string {
	return "horoscopes"
}

func periodically(fn func(time.Time)) *time.Ticker {
	ticker := time.NewTicker(time.Hour * 24 * 7)
	go func() {
		for t := range ticker.C {
			fn(t)
		}
	}()
	return ticker
}

func main() {
	seedPtr := flag.Bool("seed", false, "perform the initial seed")
	serverPtr := flag.Bool("server", false, "start the web server")
	portPtr := flag.String("port", ":8000", "port to run the web server")
	releasePtr := flag.Bool("release", false, "set the mode to release")
	debugPtr := flag.Bool("debug", false, "enable debugging mode")

	flag.Parse()

	seed := *seedPtr
	server := *serverPtr
	port := *portPtr
	release := *releasePtr
	debug := *debugPtr

	if seed {
		fmt.Println("Fetching all horoscopes...")
		SeedHoroscopes()
	} else {
		fmt.Println("Fetching current horoscopes...")
		FetchHoroscopes()
	}

	if server {
		fmt.Print("Setting up store... ")
		s := Store()
		s.BuildIndexes()
		fmt.Println("done!")

		if debug {
			data, err := json.MarshalIndent(s.weekIndex, "", "  ")
			fmt.Println(string(data), "", "  ")
		}

		fmt.Print("Scheduling periodic updates... ")
		periodically(func(t time.Time) {
			FetchHoroscopes()
			s.BuildIndexes()
		})
		fmt.Println("done!")

		fmt.Println("Bootstrapping server... done!")
		if debug {
			Server(s, false).Run(port)
		} else {
			Server(s, release).Run(port)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"net/http"
	"strconv"
	"strings"
	"time"

	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	gocache "github.com/patrickmn/go-cache"
	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const (
	baseURL = "http://www.auboutdufil.com/index.php?"
)

var (
	cache = gocache.New(1*time.Hour, 1*time.Minute)
)

type audio struct {
	Title       string    `json:"title"`
	Artist      string    `json:"artist"`
	TrackURL    string    `json:"track_url"`
	Genres      []string  `json:"genres"`
	CoverArtURL string    `json:"cover_art_url"`
	DownloadURL string    `json:"download_url"`
	License     string    `json:"license"`
	Downloads   int       `json:"downloads"`
	Plays       int       `json:"play_count"`
	Rating      float32   `json:"rating"`
	Date        time.Time `json:"published_date"`
	ID          uint8     `json:"-"`
}

type requestOptions struct {
	alike        uint
	attribution  uint
	commercial   uint
	derivative   uint
	license      string
	minDownloads float32
	minPlays     float32
	minRating    float32
}

type request struct {
	URL     string
	options *requestOptions
	query   string
	page    int
	hash    string
}

func (r *request) getHash() string {
	if r.hash == "" {
		r.hash += r.URL
		r.hash += strconv.Itoa(int(r.options.alike))
		r.hash += strconv.Itoa(int(r.options.attribution))
		r.hash += strconv.Itoa(int(r.options.commercial))
		r.hash += strconv.Itoa(int(r.options.derivative))
		r.hash += r.options.license
		r.hash += strconv.Itoa(int(r.page))
	}
	return r.hash
}

func parseInfos(parentNode *html.Node, track audio) (audio, error) {

	infosParentDiv := scrape.FindAllNested(parentNode, scrape.ByClass("pure-u-2-3"))
	if len(infosParentDiv) == 0 {
		log.Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	notPure23Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-2-3"
	}

	divs := scrape.FindAll(infosParentDiv[0], notPure23Matcher)
	if len(divs) != 10 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "10",
		}).Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	// Parse title infos
	titleTag, ok := scrape.Find(divs[3], scrape.ByTag(atom.B))
	if !ok {
		log.Warn("Incorrect html data while searching for title, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.Title = scrape.Text(titleTag)

	// Parse artist name and url
	artistTagParent, ok := scrape.Find(divs[4], scrape.ByTag(atom.Strong))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return track, errors.New("Malformed html")
	}
	artistTag, ok := scrape.Find(artistTagParent, scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.Artist = scrape.Text(artistTag)
	track.TrackURL = scrape.Attr(artistTag, "href")

	// Parse genres
	genreTags := scrape.FindAll(divs[6], scrape.ByTag(atom.Span))
	for _, genreTag := range genreTags {
		track.Genres = append(track.Genres, scrape.Text(genreTag))
	}

	return track, nil
}

func parseAudioData(node *html.Node) (track audio, err error) {

	track, err = parseInfos(node, track)
	if err != nil {
		return track, err
	}

	// look for cover image
	coverParentDiv := scrape.FindAllNested(node, scrape.ByClass("pure-u-1-3"))
	if len(coverParentDiv) == 0 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}

	notPure13Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-1-3"
	}

	divs := scrape.FindAll(coverParentDiv[0], notPure13Matcher)
	if len(divs) != 6 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "6",
		}).Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}

	coverTag := scrape.FindAllNested(divs[5], scrape.ByTag(atom.Img))
	if len(coverTag) != 1 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.CoverArtURL = scrape.Attr(coverTag[0], "src")

	// download url
	mp3PlayerDiv, ok := scrape.Find(node.Parent, scrape.ByClass("mp3player"))
	if !ok {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	downloadURLParent := scrape.FindAllNested(mp3PlayerDiv, scrape.ByClass("sm2-playlist-bd"))
	if len(downloadURLParent) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	downloadURLTag := scrape.FindAllNested(downloadURLParent[0], scrape.ByTag(atom.A))
	if len(downloadURLTag) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.DownloadURL = scrape.Attr(downloadURLTag[0], "href")

	// additional infos
	additionalInfosParent, ok := scrape.Find(node.Parent, scrape.ByClass("legenddata"))
	if !ok {
		log.Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	additionalInfosSpans := scrape.FindAll(additionalInfosParent, scrape.ByTag(atom.Span))
	if len(additionalInfosSpans) != 5 {
		log.Warn("Incorrect html data while searching for additional infos, layout may have changed")
		return track, errors.New("Malformed html")
	}

	licenseTag, ok := scrape.Find(additionalInfosSpans[4], scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for license infos, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.License = strings.Split(scrape.Attr(licenseTag, "href"), "license=")[1]

	return track, nil
}

func scrapePage(url string) (tracks []audio) {
	resp, err := http.Get(url)

	if err != nil {
		log.WithFields(log.Fields{
			"url": url,
			"err": err,
		}).Error("Failed to get page")
		return
	}

	body := resp.Body
	defer body.Close()

	root, err := html.Parse(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"url": url,
		}).Error("Unable to parse this web page")
		return
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Div && n.Parent != nil {
			return strings.Contains(scrape.Attr(n, "class"), "audio-wrapper")
		}
		return false
	}

	audioWrappers := scrape.FindAllNested(root, matcher)
	for _, wrapper := range audioWrappers {
		track, err := parseAudioData(wrapper)
		if err != nil {
			continue
		}

		tracks = append(tracks, track)
	}

	return tracks
}

func getSwitchableOpt(opt string) uint {
	if opt == "1" || opt == "true" {
		return 1
	} else if opt == "0" || opt == "false" {
		return 2
	} else {
		return 0
	}
}

func getRequestOptions(query url.Values, ps httprouter.Params) (opts *requestOptions) {
	opts = &requestOptions{}

	license := ps.ByName("license")
	if license == "" {
		license = query.Get("license")
	}
	opts.license = license

	log.WithField("license", opts.license).Info("blbl")

	// ignore license terms if a license value is set
	if opts.license != "" {
		return
	}

	alike := query.Get("alike")
	if alike == "" {
		alike = query.Get("share-alike")
		if alike == "" {
			alike = query.Get("sa")
		}
	}
	opts.alike = getSwitchableOpt(alike)

	attribution := query.Get("attribution")
	if attribution == "" {
		attribution = query.Get("by")
	}
	opts.attribution = getSwitchableOpt(attribution)

	commercial := query.Get("commercial")
	if commercial == "" {
		commercial = query.Get("nc")
	}
	opts.attribution = getSwitchableOpt(commercial)

	derivative := query.Get("derivative")
	if derivative == "" {
		derivative = query.Get("nd")
	}
	opts.attribution = getSwitchableOpt(derivative)

	return
}

func getRequest(r *http.Request, ps httprouter.Params) (req *request) {
	req = &request{}
	req.URL = r.URL.Path
	queryParams := r.URL.Query()
	req.options = getRequestOptions(queryParams, ps)

	page := queryParams.Get("page")
	if page != "" {
		req.page, _ = strconv.Atoi(page)
	} else {
		req.page = 1
	}
	return
}

func scrapData(r *request) (musics []audio) {
	var urls []string

	if r.options.alike == 0 && r.options.attribution == 0 && r.options.commercial == 0 &&
		r.options.derivative == 0 && r.options.license == "" {
		urls = []string{""}
	}

	if r.options.license != "" {
		var licenseURL string
		switch r.options.license {
		case "art-libre":
			licenseURL = "ART-LIBRE"
			break
		case "cc":
			break
		case "cc0":
			licenseURL = "CC0"
			break
		case "cc-by":
			licenseURL = "CC-BY"
			break
		case "cc-bync":
			licenseURL = "CC-BYNC"
			break
		case "cc-byncnd":
			licenseURL = "CC-BYNCND"
			break
		case "cc-byncsa":
			licenseURL = "CC-BYNCSA"
			break
		case "cc-bynd":
			licenseURL = "CC-BYND"
			break
		case "cc-bysa":
			licenseURL = "CC-BYSA"
			break
		}

		if licenseURL != "" {
			urls = []string{"license=" + licenseURL}
		}
	}

	for _, url := range urls {
		u := baseURL + url
		if url != "" {
			// add an "&" if there already is query params
			u += "&"
		}
		u += "page=" + strconv.Itoa(r.page)

		log.WithField("url", u).Info("Scraping page...")
		musics = scrapePage(u)
	}

	return
}

func handleLatest(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	req := getRequest(r, ps)

	tracks, found := cache.Get(req.getHash())
	if !found {
		log.Info("Cache expired, scraping data...")
		scrapeTracks := scrapData(req)
		cache.Set(req.getHash(), scrapeTracks, 0)
		tracks = scrapeTracks
	}

	body, err := json.Marshal(tracks)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(body)
}

func server(port string) {
	server := httprouter.New()
	server.GET("/latest/license/:license", handleLatest)
	server.GET("/latest", handleLatest)

	log.WithFields(log.Fields{
		"port": port,
	}).Info("Starting HTTP Server")

	http.ListenAndServe(":"+port, server)

}

func main() {
	var (
		port = flag.String("p", "14000", "Port used for server")
	)
	flag.Parse()

	server(*port)
}
<commit_msg>Implemented sorting by ratings, downloads and web plays<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"net/http"
	"strconv"
	"strings"
	"time"

	"net/url"

	log "github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	gocache "github.com/patrickmn/go-cache"
	"github.com/yhat/scrape"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

const (
	baseURL = "http://www.auboutdufil.com/index.php?"
)

var (
	cache = gocache.New(1*time.Hour, 1*time.Minute)
)

type audio struct {
	Title       string    `json:"title"`
	Artist      string    `json:"artist"`
	TrackURL    string    `json:"track_url"`
	Genres      []string  `json:"genres"`
	CoverArtURL string    `json:"cover_art_url"`
	DownloadURL string    `json:"download_url"`
	License     string    `json:"license"`
	Downloads   int       `json:"downloads"`
	Plays       int       `json:"play_count"`
	Rating      float32   `json:"rating"`
	Date        time.Time `json:"published_date"`
	ID          uint8     `json:"-"`
}

type requestOptions struct {
	alike        uint
	attribution  uint
	commercial   uint
	derivative   uint
	license      string
	minDownloads float32
	minPlays     float32
	minRating    float32
	sorting      string
}

type request struct {
	URL     string
	options *requestOptions
	query   string
	page    int
	hash    string
}

func (r *request) getHash() string {
	if r.hash == "" {
		r.hash += r.URL
		r.hash += strconv.Itoa(int(r.options.alike))
		r.hash += strconv.Itoa(int(r.options.attribution))
		r.hash += strconv.Itoa(int(r.options.commercial))
		r.hash += strconv.Itoa(int(r.options.derivative))
		r.hash += r.options.license
		r.hash += r.options.sorting
		r.hash += strconv.Itoa(int(r.page))
	}
	return r.hash
}

func parseInfos(parentNode *html.Node, track audio) (audio, error) {

	infosParentDiv := scrape.FindAllNested(parentNode, scrape.ByClass("pure-u-2-3"))
	if len(infosParentDiv) == 0 {
		log.Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	notPure23Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-2-3"
	}

	divs := scrape.FindAll(infosParentDiv[0], notPure23Matcher)
	if len(divs) != 10 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "10",
		}).Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	// Parse title infos
	titleTag, ok := scrape.Find(divs[3], scrape.ByTag(atom.B))
	if !ok {
		log.Warn("Incorrect html data while searching for title, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.Title = scrape.Text(titleTag)

	// Parse artist name and url
	artistTagParent, ok := scrape.Find(divs[4], scrape.ByTag(atom.Strong))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return track, errors.New("Malformed html")
	}
	artistTag, ok := scrape.Find(artistTagParent, scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for artist, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.Artist = scrape.Text(artistTag)
	track.TrackURL = scrape.Attr(artistTag, "href")

	// Parse genres
	genreTags := scrape.FindAll(divs[6], scrape.ByTag(atom.Span))
	for _, genreTag := range genreTags {
		track.Genres = append(track.Genres, scrape.Text(genreTag))
	}

	return track, nil
}

func parseAudioData(node *html.Node) (track audio, err error) {

	track, err = parseInfos(node, track)
	if err != nil {
		return track, err
	}

	// look for cover image
	coverParentDiv := scrape.FindAllNested(node, scrape.ByClass("pure-u-1-3"))
	if len(coverParentDiv) == 0 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}

	notPure13Matcher := func(n *html.Node) bool {
		return n.DataAtom == atom.Div && scrape.Attr(n, "class") != "pure-u-1-3"
	}

	divs := scrape.FindAll(coverParentDiv[0], notPure13Matcher)
	if len(divs) != 6 {
		log.WithFields(log.Fields{
			"divsNumber": len(divs),
			"expected":   "6",
		}).Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}

	coverTag := scrape.FindAllNested(divs[5], scrape.ByTag(atom.Img))
	if len(coverTag) != 1 {
		log.Warn("Incorrect html data while searching for cover url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.CoverArtURL = scrape.Attr(coverTag[0], "src")

	// download url
	mp3PlayerDiv, ok := scrape.Find(node.Parent, scrape.ByClass("mp3player"))
	if !ok {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	downloadURLParent := scrape.FindAllNested(mp3PlayerDiv, scrape.ByClass("sm2-playlist-bd"))
	if len(downloadURLParent) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	downloadURLTag := scrape.FindAllNested(downloadURLParent[0], scrape.ByTag(atom.A))
	if len(downloadURLTag) != 1 {
		log.Warn("Incorrect html data while searching for download url, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.DownloadURL = scrape.Attr(downloadURLTag[0], "href")

	// additional infos
	additionalInfosParent, ok := scrape.Find(node.Parent, scrape.ByClass("legenddata"))
	if !ok {
		log.Warn("Incorrect html data, layout may have changed")
		return track, errors.New("Malformed html")
	}

	additionalInfosSpans := scrape.FindAll(additionalInfosParent, scrape.ByTag(atom.Span))
	if len(additionalInfosSpans) != 5 {
		log.Warn("Incorrect html data while searching for additional infos, layout may have changed")
		return track, errors.New("Malformed html")
	}

	licenseTag, ok := scrape.Find(additionalInfosSpans[4], scrape.ByTag(atom.A))
	if !ok {
		log.Warn("Incorrect html data while searching for license infos, layout may have changed")
		return track, errors.New("Malformed html")
	}
	track.License = strings.Split(scrape.Attr(licenseTag, "href"), "license=")[1]

	return track, nil
}

func scrapePage(url string) (tracks []audio) {
	resp, err := http.Get(url)

	if err != nil {
		log.WithFields(log.Fields{
			"url": url,
			"err": err,
		}).Error("Failed to get page")
		return
	}

	body := resp.Body
	defer body.Close()

	root, err := html.Parse(resp.Body)
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
			"url": url,
		}).Error("Unable to parse this web page")
		return
	}

	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Div && n.Parent != nil {
			return strings.Contains(scrape.Attr(n, "class"), "audio-wrapper")
		}
		return false
	}

	audioWrappers := scrape.FindAllNested(root, matcher)
	for _, wrapper := range audioWrappers {
		track, err := parseAudioData(wrapper)
		if err != nil {
			continue
		}

		tracks = append(tracks, track)
	}

	return tracks
}

func getSwitchableOpt(opt string) uint {
	if opt == "1" || opt == "true" {
		return 1
	} else if opt == "0" || opt == "false" {
		return 2
	} else {
		return 0
	}
}

func getRequestOptions(query url.Values, ps httprouter.Params) (opts *requestOptions) {
	opts = &requestOptions{}

	license := ps.ByName("license")
	if license == "" {
		license = query.Get("license")
	}
	opts.license = license

	// ignore license terms if a license value is set
	if opts.license != "" {
		return
	}

	alike := query.Get("alike")
	if alike == "" {
		alike = query.Get("share-alike")
		if alike == "" {
			alike = query.Get("sa")
		}
	}
	opts.alike = getSwitchableOpt(alike)

	attribution := query.Get("attribution")
	if attribution == "" {
		attribution = query.Get("by")
	}
	opts.attribution = getSwitchableOpt(attribution)

	commercial := query.Get("commercial")
	if commercial == "" {
		commercial = query.Get("nc")
	}
	opts.attribution = getSwitchableOpt(commercial)

	derivative := query.Get("derivative")
	if derivative == "" {
		derivative = query.Get("nd")
	}
	opts.attribution = getSwitchableOpt(derivative)

	return
}

func getRequest(r *http.Request, ps httprouter.Params) (req *request) {
	req = &request{}
	req.URL = r.URL.Path
	queryParams := r.URL.Query()
	req.options = getRequestOptions(queryParams, ps)

	page := queryParams.Get("page")
	if page != "" {
		req.page, _ = strconv.Atoi(page)
	} else {
		req.page = 1
	}

	if strings.HasPrefix(req.URL, "/latest") {
		req.options.sorting = "posted"
	} else if strings.HasPrefix(req.URL, "/best") {
		req.options.sorting = "note"
	} else if strings.HasPrefix(req.URL, "/downloads") {
		req.options.sorting = "countweb"
	} else if strings.HasPrefix(req.URL, "/plays") {
		req.options.sorting = "countfla"
	}
	return
}

func scrapData(r *request) (musics []audio) {
	var licenseURL string

	if r.options.license != "" {
		switch r.options.license {
		case "art-libre":
			licenseURL = "ART-LIBRE"
			break
		case "cc":
			break
		case "cc0":
			licenseURL = "CC0"
			break
		case "cc-by":
			licenseURL = "CC-BY"
			break
		case "cc-bync":
			licenseURL = "CC-BYNC"
			break
		case "cc-byncnd":
			licenseURL = "CC-BYNCND"
			break
		case "cc-byncsa":
			licenseURL = "CC-BYNCSA"
			break
		case "cc-bynd":
			licenseURL = "CC-BYND"
			break
		case "cc-bysa":
			licenseURL = "CC-BYSA"
			break
		}
	}

	u := baseURL + "sort=" + r.options.sorting
	if licenseURL != "" {
		u += "&license=" + licenseURL
	}
	u += "&page=" + strconv.Itoa(r.page)

	log.WithField("url", u).Info("Scraping page...")
	musics = scrapePage(u)

	return
}

func handleRequest(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	req := getRequest(r, ps)

	tracks, found := cache.Get(req.getHash())
	if !found {
		log.Info("Cache expired, scraping data...")
		scrapeTracks := scrapData(req)
		cache.Set(req.getHash(), scrapeTracks, 0)
		tracks = scrapeTracks
	}

	body, err := json.Marshal(tracks)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(body)
}

func server(port string) {
	server := httprouter.New()
	server.GET("/latest/license/:license", handleRequest)
	server.GET("/latest", handleRequest)
	server.GET("/best/license/:license", handleRequest)
	server.GET("/best", handleRequest)
	server.GET("/downloads/license/:license", handleRequest)
	server.GET("/downloads", handleRequest)
	server.GET("/plays/license/:license", handleRequest)
	server.GET("/plays", handleRequest)

	log.WithFields(log.Fields{
		"port": port,
	}).Info("Starting HTTP Server")

	http.ListenAndServe(":"+port, server)

}

func main() {
	var (
		port = flag.String("p", "14000", "Port used for server")
	)
	flag.Parse()

	server(*port)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/smtp"
	"os"
	"strings"
	"sync"
	"time"

	ini "gopkg.in/ini.v1"
)

var (
	errExpiringSoon error = errors.New("expiring soon")
	errExpired            = errors.New("expired")
	errTimeout            = errors.New("timeout connecting to host")
)

func main() {
	urlFile := flag.String("urls", "/etc/certwatcher/urls.csv", "path to CSV containing list of URLs to monitor")
	iniFile := flag.String("config", "/etc/certwatcher/config.ini", "path to config.ini")
	days := flag.Int("days", 30, "number of days before triggering alert")
	verbose := flag.Bool("v", false, "verbose output")

	flag.Parse()

	// read config
	cfg, err := ini.Load(*iniFile)
	if err != nil {
		log.Fatalf("could not open config file: %s", err)
	}

	// load list of hosts to watch
	f, err := os.Open(*urlFile)
	if err != nil {
		log.Fatalf("could not open URL file: %s", err)
	}

	rdr := csv.NewReader(f)
	records, err := rdr.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	for _, r := range records {
		host, desc := r[0], r[1]

		wg.Add(1)

		go func() {
			defer wg.Done()
			if err := check(host, "443", *days, *verbose); err != nil {
				switch err {
				case errExpiringSoon, errExpired:
					notify(host, desc, cfg, *days, err, *verbose)
					log.Printf("main: sent notification for host %s - %s", host, err)
				default:
					log.Printf("main: ERROR: unexpected error checking host %s - %s", host, err)
				}
			}
		}()
	}

	wg.Wait()
}

func check(host, port string, days int, verbose bool) error {
	var conn *tls.Conn

	errc := make(chan error, 1)
	go func() {
		var err error
		conn, err = tls.Dial("tcp", host+":"+port, &tls.Config{
			InsecureSkipVerify: true,
		})
		if err != nil {
			errc <- err
		}

		errc <- nil
	}()

	select {
	case err := <-errc:
		if err != nil {
			return err
		}
	case <-time.Tick(5 * time.Second):
		return errTimeout
	}

	defer conn.Close()
	if err := conn.Handshake(); err != nil {
		return err
	}

	for i, cert := range conn.ConnectionState().PeerCertificates {
		if cert.IsCA {
			continue
		}

		if verbose {
			log.Printf("check: %s certificate %d: expires after %s (%s)", host, i, cert.NotAfter, time.Until(cert.NotAfter))
			log.Printf("check: %s certificate %d: issuer: %s", host, i, cert.Issuer.Names)
			log.Printf("check: %s certificate %d: names: %s", host, i, cert.Subject.Names)
			log.Printf("check: %s certificate %d: DNSNames: %s", host, i, cert.DNSNames)
		}

		if time.Now().After(cert.NotAfter) {
			return errExpired
		}

		if time.Until(cert.NotAfter) < time.Duration(days)*time.Hour*24 {
			return errExpiringSoon
		}
	}

	log.Printf("check: %s - certificate is ok", host)

	return nil
}

func notify(host, desc string, cfg *ini.File, days int, err error, verbose bool) {
	section := cfg.Section("certwatcher")

	if !section.Key("sendmail").MustBool() {
		log.Println("notify: refusing to send email due to config.")
		return
	}

	port := "587"
	if section.Key("port").String() != "" {
		port = section.Key("port").String()
	}
	mailhost := section.Key("host").String()

	auth := smtp.PlainAuth("",
		section.Key("username").String(),
		section.Key("password").String(),
		mailhost,
	)

	to := []string{section.Key("rcpt").String()}
	var subject, body string
	if err == errExpiringSoon {
		subject = fmt.Sprintf("Subject: %s certificate expiring soon: %s", section.Key("subjectprefix").String(), desc)
		body = fmt.Sprintf("The SSL certificate for the host %s (%s) is expiring in less than %d days.", host, desc, days)
	} else if err == errExpired {
		subject = fmt.Sprintf("Subject: %s certificate has expired! %s", section.Key("subjectprefix").String(), desc)
		body = fmt.Sprintf("The SSL certificate for the host %s (%s) has expired!", host, desc)
	}
	msg := []byte(strings.Join([]string{subject,
		fmt.Sprintf("To: %s", strings.Join(to, ", ")),
		fmt.Sprintf("From: %s", section.Key("from").String()),
		"",
		body,
		"",
		"Please take appropriate action!",
	},
		"\r\n",
	))

	if verbose {
		log.Printf("notify: sending host %s expiration notification to %s", host, section.Key("rcpt").String())
	}

	if err := smtp.SendMail(mailhost+":"+port, auth, section.Key("from").String(), to, msg); err != nil {
		log.Fatalf("could not send email: %s", err)
	}
}
<commit_msg>add a timeout err if cannot reach mail server<commit_after>package main

import (
	"crypto/tls"
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/smtp"
	"os"
	"strings"
	"sync"
	"time"

	ini "gopkg.in/ini.v1"
)

var (
	errExpiringSoon error = errors.New("expiring soon")
	errExpired            = errors.New("expired")
	errTimeout            = errors.New("timeout connecting to host")
)

func main() {
	urlFile := flag.String("urls", "/etc/certwatcher/urls.csv", "path to CSV containing list of URLs to monitor")
	iniFile := flag.String("config", "/etc/certwatcher/config.ini", "path to config.ini")
	days := flag.Int("days", 30, "number of days before triggering alert")
	verbose := flag.Bool("v", false, "verbose output")

	flag.Parse()

	// read config
	cfg, err := ini.Load(*iniFile)
	if err != nil {
		log.Fatalf("could not open config file: %s", err)
	}

	// load list of hosts to watch
	f, err := os.Open(*urlFile)
	if err != nil {
		log.Fatalf("could not open URL file: %s", err)
	}

	rdr := csv.NewReader(f)
	records, err := rdr.ReadAll()
	if err != nil {
		log.Fatal(err)
	}

	var wg sync.WaitGroup
	for _, r := range records {
		host, desc := r[0], r[1]

		wg.Add(1)

		go func() {
			defer wg.Done()
			if err := check(host, "443", *days, *verbose); err != nil {
				switch err {
				case errExpiringSoon, errExpired:
					notify(host, desc, cfg, *days, err, *verbose)
					log.Printf("main: sent notification for host %s - %s", host, err)
				default:
					log.Printf("main: ERROR: unexpected error checking host %s - %s", host, err)
				}
			}
		}()
	}

	wg.Wait()
}

func check(host, port string, days int, verbose bool) error {
	var conn *tls.Conn

	errc := make(chan error, 1)
	go func() {
		var err error
		conn, err = tls.Dial("tcp", host+":"+port, &tls.Config{
			InsecureSkipVerify: true,
		})
		if err != nil {
			errc <- err
		}

		errc <- nil
	}()

	select {
	case err := <-errc:
		if err != nil {
			return err
		}
	case <-time.Tick(5 * time.Second):
		return errTimeout
	}

	defer conn.Close()
	if err := conn.Handshake(); err != nil {
		return err
	}

	for i, cert := range conn.ConnectionState().PeerCertificates {
		if cert.IsCA {
			continue
		}

		if verbose {
			log.Printf("check: %s certificate %d: expires after %s (%s)", host, i, cert.NotAfter, time.Until(cert.NotAfter))
			log.Printf("check: %s certificate %d: issuer: %s", host, i, cert.Issuer.Names)
			log.Printf("check: %s certificate %d: names: %s", host, i, cert.Subject.Names)
			log.Printf("check: %s certificate %d: DNSNames: %s", host, i, cert.DNSNames)
		}

		if time.Now().After(cert.NotAfter) {
			return errExpired
		}

		if time.Until(cert.NotAfter) < time.Duration(days)*time.Hour*24 {
			return errExpiringSoon
		}
	}

	log.Printf("check: %s - certificate is ok", host)

	return nil
}

func notify(host, desc string, cfg *ini.File, days int, err error, verbose bool) {
	section := cfg.Section("certwatcher")

	if !section.Key("sendmail").MustBool() {
		log.Println("notify: refusing to send email due to config.")
		return
	}

	port := "587"
	if section.Key("port").String() != "" {
		port = section.Key("port").String()
	}
	mailhost := section.Key("host").String()

	auth := smtp.PlainAuth("",
		section.Key("username").String(),
		section.Key("password").String(),
		mailhost,
	)

	to := []string{section.Key("rcpt").String()}
	var subject, body string
	if err == errExpiringSoon {
		subject = fmt.Sprintf("Subject: %s certificate expiring soon: %s", section.Key("subjectprefix").String(), desc)
		body = fmt.Sprintf("The SSL certificate for the host %s (%s) is expiring in less than %d days.", host, desc, days)
	} else if err == errExpired {
		subject = fmt.Sprintf("Subject: %s certificate has expired! %s", section.Key("subjectprefix").String(), desc)
		body = fmt.Sprintf("The SSL certificate for the host %s (%s) has expired!", host, desc)
	}
	msg := []byte(strings.Join([]string{subject,
		fmt.Sprintf("To: %s", strings.Join(to, ", ")),
		fmt.Sprintf("From: %s", section.Key("from").String()),
		"",
		body,
		"",
		"Please take appropriate action!",
	},
		"\r\n",
	))

	if verbose {
		log.Printf("notify: sending host %s expiration notification to %s", host, section.Key("rcpt").String())
	}

	errc := make(chan error, 1)
	go func() {
		errc <- smtp.SendMail(mailhost+":"+port, auth, section.Key("from").String(), to, msg)
	}()
	select {
	case err := <-errc:
		log.Fatalf("could not send email: %s", err)
	case <-time.Tick(30 * time.Second):
		log.Fatalf("Timeout reaching mail server")
	}
}
<|endoftext|>