<commit_before>package quic

type FrameType uint8

const (
	PaddingFrameType FrameType = iota
	RstStreamFrameType
	ConnectionCloseFrameType
	GoawayFrameType
	WindowUpdateFrameType
	BlockedFrameType
	StopWaitingFrameType
	PingFrameType
	StreamFrameType             = 0x80
	AckFrameType                = 0x40
	CongestionFeedbackFrameType = 0x20
)

// Frame Header
/*
+--------+--------+--------+--------+--------+---    ---+
 Public |    Connection ID (0, 8, 32, or 64)    ...    | ->
|Flags(8)|      (variable length)                       |
+--------+--------+--------+--------+--------+---    ---+

    9       10       11        12
+--------+--------+--------+--------+
|      Quic Version (32)            | ->
|         (optional)                |
+--------+--------+--------+--------+

    13      14       15        16        17       18       19       20
+--------+--------+--------+--------+--------+--------+--------+--------+
|         Sequence Number (8, 16, 32, or 48)          |Private | FEC (8)|
|                         (variable length)           |Flags(8)|  (opt) |
+--------+--------+--------+--------+--------+--------+--------+--------+
*/

type FrameHeader struct {
	PublicFlags    byte
	ConnectionID   uint64
	Version        uint32
	SequenceNumber uint64
	PrivateFlags   byte
	FEC            byte
}

func NewFrameHeader(publicFlags byte, connectionID uint64, version uint32, sequenceNumber uint64, privateFlags, fec byte) *FrameHeader {
	fh := &FrameHeader{
		publicFlags,
		connectionID,
		version,
		sequenceNumber,
		privateFlags,
		fec,
	}
	return fh
}

func (self *FrameHeader) Parse(data []byte) (err error) {
	index := 0
	self.PublicFlags = data[index]

	if self.PublicFlags&0x0c > 0 {
		self.ConnectionID = uint64(data[1]<<56 | data[2]<<48 | data[3]<<40 | data[4]<<32 | data[5]<<24 | data[6]<<16 | data[7]<<8 | data[8])
		index = 9
	} else if self.PublicFlags&0x08 > 0 {
		self.ConnectionID = uint64(data[1]<<24 | data[2]<<16 | data[3]<<8 | data[4])
		index = 5
	} else if self.PublicFlags&0x04 > 0 {
		self.ConnectionID = uint64(data[1])
		index = 2
	} else {
		self.ConnectionID = 0 // omitted
		index = 1
	}

	if self.PublicFlags&0x01 > 0 {
		self.Version = uint32(data[index]<<24 | data[index+1]<<16 | data[index+2]<<8 | data[index+3])
		index += 4
	}

	// TODO: parse sequence number

	self.PrivateFlags = data[index]

	// TODO: parse FEC

	return err
}
<commit_msg>write public and private flags<commit_after>package quic

type FrameType uint8

const (
	PaddingFrameType FrameType = iota
	RstStreamFrameType
	ConnectionCloseFrameType
	GoawayFrameType
	WindowUpdateFrameType
	BlockedFrameType
	StopWaitingFrameType
	PingFrameType
	StreamFrameType             = 0x80
	AckFrameType                = 0x40
	CongestionFeedbackFrameType = 0x20
)

type PublicFlags uint8

const (
	QUIC_VERSION              PublicFlags = 0x01
	PUBLIC_RESET                          = 0x02
	CONTAIN_CONNECTION_ID_8               = 0x0c
	CONTAIN_CONNECTION_ID_4               = 0x08
	CONTAIN_CONNECTION_ID_1               = 0x04
	OMIT_CONNECTION_ID                    = 0x00
	CONTAIN_SEQUENCE_NUMBER_6             = 0x30
	CONTAIN_SEQUENCE_NUMBER_4             = 0x20
	CONTAIN_SEQUENCE_NUMBER_2             = 0x10
	CONTAIN_SEQUENCE_NUMBER_1             = 0x00
	RESERVED                              = 0xc0
)

type PrivateFlags uint8

const (
	FLAG_ENTROPY   PrivateFlags = 0x01
	FLAG_FEC_GROUP              = 0x02
	FLAG_FEC                    = 0x03
)

// Frame Header
/*
+--------+--------+--------+--------+--------+---    ---+
 Public |    Connection ID (0, 8, 32, or 64)    ...    | ->
|Flags(8)|      (variable length)                       |
+--------+--------+--------+--------+--------+---    ---+

    9       10       11        12
+--------+--------+--------+--------+
|      Quic Version (32)            | ->
|         (optional)                |
+--------+--------+--------+--------+

    13      14       15        16        17       18       19       20
+--------+--------+--------+--------+--------+--------+--------+--------+
|         Sequence Number (8, 16, 32, or 48)          |Private | FEC (8)|
|                         (variable length)           |Flags(8)|  (opt) |
+--------+--------+--------+--------+--------+--------+--------+--------+
*/

type FrameHeader struct {
	PublicFlags    byte
	ConnectionID   uint64
	Version        uint32
	SequenceNumber uint64
	PrivateFlags   byte
	FEC            byte
}

func NewFrameHeader(publicFlags byte, connectionID uint64, version uint32, sequenceNumber uint64, privateFlags, fec byte) *FrameHeader {
	fh := &FrameHeader{
		publicFlags,
		connectionID,
		version,
		sequenceNumber,
		privateFlags,
		fec,
	}
	return fh
}

func (self *FrameHeader) Parse(data []byte) (err error) {
	index := 0
	self.PublicFlags = data[index]

	if self.PublicFlags&0x0c > 0 {
		self.ConnectionID = uint64(data[1]<<56 | data[2]<<48 | data[3]<<40 | data[4]<<32 | data[5]<<24 | data[6]<<16 | data[7]<<8 | data[8])
		index = 9
	} else if self.PublicFlags&0x08 > 0 {
		self.ConnectionID = uint64(data[1]<<24 | data[2]<<16 | data[3]<<8 | data[4])
		index = 5
	} else if self.PublicFlags&0x04 > 0 {
		self.ConnectionID = uint64(data[1])
		index = 2
	} else {
		self.ConnectionID = 0 // omitted
		index = 1
	}

	if self.PublicFlags&0x01 > 0 {
		self.Version = uint32(data[index]<<24 | data[index+1]<<16 | data[index+2]<<8 | data[index+3])
		index += 4
	}

	// TODO: parse sequence number

	self.PrivateFlags = data[index]

	// TODO: parse FEC

	return err
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/database/sql"
	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getSnapshotTimeout = 10 * time.Second
const heightPollingPeriod = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
// After each attempt to fetch and apply a block, it calls health
// to report either an error or nil to indicate success.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client, health func(error)) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// Fetch the generator height periodically.
	go pollGeneratorHeight(ctx, peer)

	var ntimeouts uint // for backoff
	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height, timeoutBackoffDur(ntimeouts))
			if err != nil {
				health(err)
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			if len(blocks) == 0 {
				// Request time out. There might not have been any blocks published,
				// or there was a network error or it just took too long to process the
				// request.
				ntimeouts++
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				health(err)
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			health(nil)
			nfailures, ntimeouts = 0, 0
		}
	}
}

func pollGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	updateGeneratorHeight(ctx, peer)

	ticker := time.NewTicker(heightPollingPeriod)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, fetchGeneratorHeight exiting")
			ticker.Stop()
			return
		case <-ticker.C:
			updateGeneratorHeight(ctx, peer)
		}
	}
}

func updateGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	gh, err := getHeight(ctx, peer)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	generatorLock.Lock()
	defer generatorLock.Unlock()
	generatorHeight = gh
	generatorHeightFetchedAt = time.Now()
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			// TODO(kr): this is a validation failure.
			// It's either a serious bug or an attack.
			// Do something better than just log the error
			// (in the caller above). Alert a human,
			// the security team, the legal team, the A-team,
			// somebody.
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

func timeoutBackoffDur(n uint) time.Duration {
	const baseTimeout = 3 * time.Second
	if n > 4 {
		n = 4 // cap to extra 16s
	}
	d := rand.Int63n(int64(time.Second) * (1 << n))
	return baseTimeout + time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64, timeout time.Duration) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if ctx.Err() == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}

// Snapshot fetches the latest snapshot from the generator and applies it to this
// core's snapshot set. It should only be called on freshly configured cores--
// cores that have been operating should replay all transactions so that they can
// index them properly.
func Snapshot(ctx context.Context, peer *rpc.Client, s protocol.Store, db *sql.DB) error {
	ctx, cancel := context.WithTimeout(ctx, getSnapshotTimeout)
	defer cancel()

	var snapResp struct {
		Data   []byte
		Height uint64
	}
	err := peer.Call(ctx, "/rpc/get-snapshot", nil, &snapResp)
	if err != nil {
		return err
	}

	const snapQ = `
		INSERT INTO snapshots (height, data) VALUES ($1, $2)
		ON CONFLICT DO NOTHING
	`
	_, err = db.Exec(ctx, snapQ, snapResp.Height, snapResp.Data)
	if err != nil {
		return err
	}

	// Next, get the genesis block.
	blocks, err := getBlocks(ctx, peer, 0, getSnapshotTimeout)
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is afoot.
		return errors.New("could not get initial block from generator")
	}

	err = s.SaveBlock(ctx, blocks[0])

	// Also get the corresponding block.
	blocks, err = getBlocks(ctx, peer, snapResp.Height-1, getSnapshotTimeout) // because we get the NEXT block
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is still afoot.
		return errors.New("generator provided snapshot but could not provide block")
	}

	return s.SaveBlock(ctx, blocks[0])
}
<commit_msg>core/fetch: remove done TODO<commit_after>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/database/sql"
	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getSnapshotTimeout = 10 * time.Second
const heightPollingPeriod = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
// After each attempt to fetch and apply a block, it calls health
// to report either an error or nil to indicate success.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client, health func(error)) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	// Fetch the generator height periodically.
	go pollGeneratorHeight(ctx, peer)

	var ntimeouts uint // for backoff
	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height, timeoutBackoffDur(ntimeouts))
			if err != nil {
				health(err)
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			if len(blocks) == 0 {
				// Request time out. There might not have been any blocks published,
				// or there was a network error or it just took too long to process the
				// request.
				ntimeouts++
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				health(err)
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			health(nil)
			nfailures, ntimeouts = 0, 0
		}
	}
}

func pollGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	updateGeneratorHeight(ctx, peer)

	ticker := time.NewTicker(heightPollingPeriod)
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, fetchGeneratorHeight exiting")
			ticker.Stop()
			return
		case <-ticker.C:
			updateGeneratorHeight(ctx, peer)
		}
	}
}

func updateGeneratorHeight(ctx context.Context, peer *rpc.Client) {
	gh, err := getHeight(ctx, peer)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	generatorLock.Lock()
	defer generatorLock.Unlock()
	generatorHeight = gh
	generatorHeightFetchedAt = time.Now()
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

func timeoutBackoffDur(n uint) time.Duration {
	const baseTimeout = 3 * time.Second
	if n > 4 {
		n = 4 // cap to extra 16s
	}
	d := rand.Int63n(int64(time.Second) * (1 << n))
	return baseTimeout + time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64, timeout time.Duration) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if ctx.Err() == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}

// Snapshot fetches the latest snapshot from the generator and applies it to this
// core's snapshot set. It should only be called on freshly configured cores--
// cores that have been operating should replay all transactions so that they can
// index them properly.
func Snapshot(ctx context.Context, peer *rpc.Client, s protocol.Store, db *sql.DB) error {
	ctx, cancel := context.WithTimeout(ctx, getSnapshotTimeout)
	defer cancel()

	var snapResp struct {
		Data   []byte
		Height uint64
	}
	err := peer.Call(ctx, "/rpc/get-snapshot", nil, &snapResp)
	if err != nil {
		return err
	}

	const snapQ = `
		INSERT INTO snapshots (height, data) VALUES ($1, $2)
		ON CONFLICT DO NOTHING
	`
	_, err = db.Exec(ctx, snapQ, snapResp.Height, snapResp.Data)
	if err != nil {
		return err
	}

	// Next, get the genesis block.
	blocks, err := getBlocks(ctx, peer, 0, getSnapshotTimeout)
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is afoot.
		return errors.New("could not get initial block from generator")
	}

	err = s.SaveBlock(ctx, blocks[0])

	// Also get the corresponding block.
	blocks, err = getBlocks(ctx, peer, snapResp.Height-1, getSnapshotTimeout) // because we get the NEXT block
	if err != nil {
		return err
	}

	if len(blocks) < 1 {
		// Something seriously funny is still afoot.
		return errors.New("generator provided snapshot but could not provide block")
	}

	return s.SaveBlock(ctx, blocks[0])
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getBlocksTimeout = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height)
			if err != nil {
				log.Error(ctx, err)
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			nfailures = 0

			gh, err := getHeight(ctx, peer)
			if err != nil {
				log.Error(ctx, err)
			} else {
				generatorLock.Lock()
				generatorHeight = gh
				generatorHeightFetchedAt = time.Now()
				generatorLock.Unlock()
			}
		}
	}
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			// TODO(kr): this is a validation failure.
			// It's either a serious bug or an attack.
			// Do something better than just log the error
			// (in the caller above). Alert a human,
			// the security team, the legal team, the A-team,
			// somebody.
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, getBlocksTimeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if err == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}
<commit_msg>core/fetch: backoff on network failures<commit_after>package fetch

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"chain/errors"
	"chain/log"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
	"chain/protocol/state"
)

const getBlocksTimeout = 3 * time.Second

var (
	generatorHeight          uint64
	generatorHeightFetchedAt time.Time
	generatorLock            sync.Mutex
)

func GeneratorHeight() (uint64, time.Time) {
	generatorLock.Lock()
	h := generatorHeight
	t := generatorHeightFetchedAt
	generatorLock.Unlock()
	return h, t
}

// Fetch runs in a loop, fetching blocks from the configured
// peer (e.g. the generator) and applying them to the local
// Chain.
//
// It returns when its context is canceled.
func Fetch(ctx context.Context, c *protocol.Chain, peer *rpc.Client) {
	// This process just became leader, so it's responsible
	// for recovering after the previous leader's exit.
	prevBlock, prevSnapshot, err := c.Recover(ctx)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}

	var nfailures uint // for backoff
	for {
		select {
		case <-ctx.Done():
			log.Messagef(ctx, "Deposed, Fetch exiting")
			return
		default:
			var height uint64
			if prevBlock != nil {
				height = prevBlock.Height
			}

			blocks, err := getBlocks(ctx, peer, height)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}

			prevSnapshot, prevBlock, err = applyBlocks(ctx, c, prevSnapshot, prevBlock, blocks)
			if err != nil {
				log.Error(ctx, err)
				nfailures++
				time.Sleep(backoffDur(nfailures))
				continue
			}
			nfailures = 0

			gh, err := getHeight(ctx, peer)
			if err != nil {
				log.Error(ctx, err)
			} else {
				generatorLock.Lock()
				generatorHeight = gh
				generatorHeightFetchedAt = time.Now()
				generatorLock.Unlock()
			}
		}
	}
}

func applyBlocks(ctx context.Context, c *protocol.Chain, snap *state.Snapshot, block *bc.Block, blocks []*bc.Block) (*state.Snapshot, *bc.Block, error) {
	for _, b := range blocks {
		ss, err := c.ValidateBlock(ctx, snap, block, b)
		if err != nil {
			// TODO(kr): this is a validation failure.
			// It's either a serious bug or an attack.
			// Do something better than just log the error
			// (in the caller above). Alert a human,
			// the security team, the legal team, the A-team,
			// somebody.
			return snap, block, err
		}

		err = c.CommitBlock(ctx, b, ss)
		if err != nil {
			return snap, block, err
		}

		snap, block = ss, b
	}
	return snap, block, nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}

// getBlocks sends a get-blocks RPC request to another Core
// for all blocks since the highest-known one.
func getBlocks(ctx context.Context, peer *rpc.Client, height uint64) ([]*bc.Block, error) {
	ctx, cancel := context.WithTimeout(ctx, getBlocksTimeout)
	defer cancel()

	var blocks []*bc.Block
	err := peer.Call(ctx, "/rpc/get-blocks", height, &blocks)
	if err == context.DeadlineExceeded {
		return nil, nil
	}
	return blocks, errors.Wrap(err, "get blocks rpc")
}

// getHeight sends a get-height RPC request to another Core for
// the latest height that that peer knows about.
func getHeight(ctx context.Context, peer *rpc.Client) (uint64, error) {
	var resp map[string]uint64
	err := peer.Call(ctx, "/rpc/block-height", nil, &resp)
	if err != nil {
		return 0, errors.Wrap(err, "could not get remote block height")
	}
	h, ok := resp["block_height"]
	if !ok {
		return 0, errors.New("unexpected response from generator")
	}

	return h, nil
}
<|endoftext|>


<commit_before>// Package pages provides some helpers for serving web pages.
//
// Example usage:
//   var page = Add("/uri", handler, "tmpl/base.tmpl", "tmpl/page.tmpl")
//
//   func handler(w http.ResponseWriter, r *http.Request) pages.Result {
//     return pages.OK("some data to page.tmpl")
//   }
//
//   pages.SetLogger(func(r *http.Request) pages.Logger {
//     return appengine.NewContext(r)
//   })
//   http.Handle(page.URI, page)
package pages

import (
	"html/template"
	"net/http"

	"log"
)

var (
	logger              LoggerFunc
	BaseTemplate        = "base" // name of top-level template to invoke for each page
	StatusBadRequest    = Result{responseCode: http.StatusBadRequest}
	StatusNotFound      = Result{responseCode: http.StatusNotFound}
	StatusInternalError = Result{responseCode: http.StatusInternalServerError}
)

// SetLogger registers a logger function.
func SetLogger(l LoggerFunc) {
	logger = l
}

// Renderer is a function to render a page result.
type Renderer func(w http.ResponseWriter, r *http.Request) Result

// A Page to be rendered.
type Page struct {
	URI    string             // URI path
	Render Renderer           // func to render the page
	tmpl   *template.Template // backing template
}

// Add creates a new page.
//
// Add panics if the page templates cannot be parsed.
func Add(uri string, render Renderer, tmpls ...string) Page {
	t := template.Must(template.ParseFiles(tmpls...))
	return Page{
		URI:    uri,
		tmpl:   t,
		Render: render,
	}
}

// Result is the result of rendering a page.
type Result struct {
	data         interface{} // Data to render the page.
	responseCode int         // HTTP response code.
	err          error       // Error, or nil.
}

// StatusOK returns http.StatusOK with given data passed to the template.
func StatusOK(data interface{}) Result {
	return Result{
		responseCode: http.StatusOK,
		data:         data,
	}
}

// BadRequestWith returns a Result indicating a bad request.
func BadRequestWith(err error) Result {
	return Result{
		responseCode: http.StatusBadRequest,
		err:          err,
	}
}

// ServeHTTP serves HTTP for the page.
//
// ServeHTTP panics if no logger has been registered with SetLogger.
func (p Page) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if logger == nil {
		log.Fatalf("no logger specified; call SetLogger\n")
	}
	l := logger(r)
	l.Infof("Page %+v will ServeHTTP for URL: %v", p, r.URL)

	// Render the page, retrieving any data for the template.
	pr := p.Render(w, r)
	if pr.err != nil || pr.responseCode != http.StatusOK {
		if pr.err != nil {
			l.Errorf("Error while rendering %v: %v\n", r.URL, pr.err)
		}
		if pr.responseCode == http.StatusNotFound {
			http.NotFound(w, r)
		} else if pr.responseCode == http.StatusBadRequest {
			http.Error(w, "Bad request", http.StatusBadRequest)
		} else {
			http.Error(w, "Internal server error.", pr.responseCode)
		}
		return
	}

	err := p.tmpl.ExecuteTemplate(w, BaseTemplate, pr.data)
	if err != nil {
		// TODO: If this happens, partial template data is still written
		// to w by ExecuteTemplate, which isn't ideal; we'd like the 500
		// to be the only thing returned to viewing user.

		// Error rendering the template is a programming bug.
		l.Errorf("Failed to render template: %v", err)
		http.Error(w, "Internal server error.", http.StatusInternalServerError)
	}
}

// LoggerFunc returns a logger from a http request.
type LoggerFunc func(*http.Request) Logger

// Logger specifies logging functions.
type Logger interface {
	// Debugf formats its arguments according to the format, analogous to fmt.Printf,
	// and records the text as a log message at Debug level.
	Debugf(format string, args ...interface{})

	// Infof is like Debugf, but at Info level.
	Infof(format string, args ...interface{})

	// Warningf is like Debugf, but at Warning level.
	Warningf(format string, args ...interface{})

	// Errorf is like Debugf, but at Error level.
	Errorf(format string, args ...interface{})

	// Criticalf is like Debugf, but at Critical level.
	Criticalf(format string, args ...interface{})
}
<commit_msg>adds ShowError utility to redirect to /?error='Bad request.' while still logging the underlying error<commit_after>// Package pages provides some helpers for serving web pages.
//
// Example usage:
//   var page = Add("/uri", handler, "tmpl/base.tmpl", "tmpl/page.tmpl")
//
//   func handler(w http.ResponseWriter, r *http.Request) pages.Result {
//     return pages.OK("some data to page.tmpl")
//   }
//
//   pages.SetLogger(func(r *http.Request) pages.Logger {
//     return appengine.NewContext(r)
//   })
//   http.Handle(page.URI, page)
package pages

import (
	"fmt"
	"html/template"
	"net/http"
	"net/url"

	"log"
)

var (
	logger              LoggerFunc
	BaseTemplate        = "base"         // Name of top-level template to invoke for each page.
	BadRequestMsg       = "Bad request." // Message to display if ShowError is called.
	StatusBadRequest    = Result{responseCode: http.StatusBadRequest}
	StatusNotFound      = Result{responseCode: http.StatusNotFound}
	StatusInternalError = Result{responseCode: http.StatusInternalServerError}
)

// SetLogger registers a logger function.
func SetLogger(l LoggerFunc) {
	logger = l
}

// Renderer is a function to render a page result.
type Renderer func(w http.ResponseWriter, r *http.Request) Result

// A Page to be rendered.
type Page struct {
	URI    string             // URI path
	Render Renderer           // func to render the page
	tmpl   *template.Template // backing template
}

// Add creates a new page.
//
// Add panics if the page templates cannot be parsed.
func Add(uri string, render Renderer, tmpls ...string) Page {
	t := template.Must(template.ParseFiles(tmpls...))
	return Page{
		URI:    uri,
		tmpl:   t,
		Render: render,
	}
}

// Result is the result of rendering a page.
type Result struct {
	data         interface{} // Data to render the page.
	responseCode int         // HTTP response code.
	err          error       // Error, or nil.
}

// StatusOK returns http.StatusOK with given data passed to the template.
func StatusOK(data interface{}) Result {
	return Result{
		responseCode: http.StatusOK,
		data:         data,
	}
}

// BadRequestWith returns a Result indicating a bad request.
func BadRequestWith(err error) Result {
	return Result{
		responseCode: http.StatusBadRequest,
		err:          err,
	}
}

// ShowError redirects to the index page with the "error" param set to
// a static error message.
//
// Provided error is logged, but not displayed to the user.
func ShowError(w http.ResponseWriter, r *http.Request, err error) {
	if logger == nil {
		log.Fatalf("no logger specified; call SetLogger\n")
	}
	l := logger(r)
	q := url.Values{
		"error": []string{BadRequestMsg},
	}
	nextUrl := fmt.Sprintf("/?%s", q.Encode())
	l.Errorf("returning StatusBadRequest and redirecting to %q: %v\n", nextUrl, err)
	http.Redirect(w, r, nextUrl, http.StatusBadRequest)
}

// ServeHTTP serves HTTP for the page.
//
// ServeHTTP panics if no logger has been registered with SetLogger.
func (p Page) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if logger == nil {
		log.Fatalf("no logger specified; call SetLogger\n")
	}
	l := logger(r)
	l.Infof("Page %+v will ServeHTTP for URL: %v", p, r.URL)

	// Render the page, retrieving any data for the template.
	pr := p.Render(w, r)
	if pr.err != nil || pr.responseCode != http.StatusOK {
		if pr.err != nil {
			l.Errorf("Error while rendering %v: %v\n", r.URL, pr.err)
		}
		if pr.responseCode == http.StatusNotFound {
			http.NotFound(w, r)
		} else if pr.responseCode == http.StatusBadRequest {
			http.Error(w, "Bad request", http.StatusBadRequest)
		} else {
			http.Error(w, "Internal server error.", pr.responseCode)
		}
		return
	}

	err := p.tmpl.ExecuteTemplate(w, BaseTemplate, pr.data)
	if err != nil {
		// TODO: If this happens, partial template data is still written
		// to w by ExecuteTemplate, which isn't ideal; we'd like the 500
		// to be the only thing returned to viewing user.

		// Error rendering the template is a programming bug.
		l.Errorf("Failed to render template: %v", err)
		http.Error(w, "Internal server error.", http.StatusInternalServerError)
	}
}

// LoggerFunc returns a logger from a http request.
type LoggerFunc func(*http.Request) Logger

// Logger specifies logging functions.
type Logger interface {
	// Debugf formats its arguments according to the format, analogous to fmt.Printf,
	// and records the text as a log message at Debug level.
	Debugf(format string, args ...interface{})

	// Infof is like Debugf, but at Info level.
	Infof(format string, args ...interface{})

	// Warningf is like Debugf, but at Warning level.
	Warningf(format string, args ...interface{})

	// Errorf is like Debugf, but at Error level.
	Errorf(format string, args ...interface{})

	// Criticalf is like Debugf, but at Critical level.
	Criticalf(format string, args ...interface{})
}
<|endoftext|>


<commit_before>package utils
import (
  "io/ioutil"
  "os"
)

const (
	packer_filename = "/packer.json"
	packer_content = `{
"builders": [
  {
    "type": "docker",
    "image": "docker-registry.priv.blablacar.net/blablajessie",
    "export_path": "rootfs.tar"
  }
  ],
  "provisioners": [
    {
      "type": "shell",
      "script": "cnt-provision.sh"
    },
    {
      "type": "shell",
      "script": "../install.sh"
    },
    {
      "type": "shell",
      "script": "cnt-cleanup.sh"
    }
  ]
}
`
	pre_filename = "/cnt-provision.sh"
	pre_content = `#!/bin/bash

wget -qO- http://debmirror.priv.blablacar.net/debian/key.gpg | apt-key add - && \
echo 'deb http://debmirror.priv.blablacar.net/debian/ jessie main contrib non-free' > /etc/apt/sources.list

# package pinning
cat > /etc/apt/preferences.d/blablacar.pref <<EEOF
Package: *
Pin: release o=BlaBlaCar
Pin-Priority: 900
EEOF

apt-get update
`
	post_filename = "/cnt-cleanup.sh"
	post_content = `#!/bin/bash

apt-get purge curl chef -y
apt-get autoremove --purge -y
rm -rf /tmp/*
rm -rf /var/lib/apt/lists/*
`

)

func WritePackerFiles(target string) {
	// TODO check error
	ioutil.WriteFile(target + packer_filename, []byte(packer_content), 0644)
	ioutil.WriteFile(target + pre_filename, []byte(pre_content), 0644)
	ioutil.WriteFile(target + post_filename, []byte(post_content), 0644)
}

func RemovePackerFiles(target string) {
  os.Remove(target + packer_filename)
  os.Remove(target + pre_filename)
  os.Remove(target + post_filename)
}
<commit_msg>Remove Curl from cleanup to be able to use php5-curl or curl in container that needs it<commit_after>package utils
import (
  "io/ioutil"
  "os"
)

const (
	packer_filename = "/packer.json"
	packer_content = `{
"builders": [
  {
    "type": "docker",
    "image": "docker-registry.priv.blablacar.net/blablajessie",
    "export_path": "rootfs.tar"
  }
  ],
  "provisioners": [
    {
      "type": "shell",
      "script": "cnt-provision.sh"
    },
    {
      "type": "shell",
      "script": "../install.sh"
    },
    {
      "type": "shell",
      "script": "cnt-cleanup.sh"
    }
  ]
}
`
	pre_filename = "/cnt-provision.sh"
	pre_content = `#!/bin/bash

wget -qO- http://debmirror.priv.blablacar.net/debian/key.gpg | apt-key add - && \
echo 'deb http://debmirror.priv.blablacar.net/debian/ jessie main contrib non-free' > /etc/apt/sources.list

# package pinning
cat > /etc/apt/preferences.d/blablacar.pref <<EEOF
Package: *
Pin: release o=BlaBlaCar
Pin-Priority: 900
EEOF

apt-get update
`
	post_filename = "/cnt-cleanup.sh"
	post_content = `#!/bin/bash

apt-get purge chef -y
apt-get autoremove --purge -y
rm -rf /tmp/*
rm -rf /var/lib/apt/lists/*
`

)

func WritePackerFiles(target string) {
	// TODO check error
	ioutil.WriteFile(target + packer_filename, []byte(packer_content), 0644)
	ioutil.WriteFile(target + pre_filename, []byte(pre_content), 0644)
	ioutil.WriteFile(target + post_filename, []byte(post_content), 0644)
}

func RemovePackerFiles(target string) {
  os.Remove(target + packer_filename)
  os.Remove(target + pre_filename)
  os.Remove(target + post_filename)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

const buffersize = 1024

type ParseError struct {
	String string
}

func (err ParseError) Error() string {
	return fmt.Sprintf("could not parse line: %s", err.String)
}

type ConfigValue struct {
	Name  string
	Value string
}

type ConfigReader interface {
	StringVal(key, dflt string) string
	StringSlice(key string, dflt []string) []string
	IntVal(key string, dflt int) int
	BoolVal(key string, dflt bool) bool
	GetConfigValues() map[string]ConfigValue
}

type EnvironConfigReader struct {
	prefix string
	configValues map[string]ConfigValue
}

func NewEnvironConfigReader(filename, prefix string) (*EnvironConfigReader, error) {
	r := &EnvironConfigReader{prefix, map[string]ConfigValue{}}

	file, err := os.Open(filename)
	if err != nil {
		return r, err
	}
	defer file.Close()
	if err := r.parse(file); err != nil {
		return r, err
	}
	return r, nil
}

// NewEnvironOnlyConfigReader creates an EnvironConfigReader without parsing
// a file first.
func NewEnvironOnlyConfigReader(prefix string) *EnvironConfigReader {
	return &EnvironConfigReader{prefix, map[string]ConfigValue{}}
}

// parse is a really dumb reader parser.  It maps only key values in the form
// of key=value and strips whitespaces surrounding either field.  If the format
// does not match, then and error will return.
func (p *EnvironConfigReader) parse(reader io.Reader) error {
	var (
		line string
		err  error
	)

	bufReader := bufio.NewReader(reader)
	for err != io.EOF {
		line, err = bufReader.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}

		line = strings.TrimSpace(strings.Split(line, "#")[0])
		if err := keyvalue([]byte(line)); err != nil {
			return err
		}
	}
	return nil
}

func (p *EnvironConfigReader) StringVal(name string, defaultval string) string {
	configValue := ConfigValue{
		Name:  p.getFullValueName(name),
		Value: defaultval,
	}

	if val := os.Getenv(configValue.Name); val != "" {
		configValue.Value = val
	}
	p.configValues[name] = configValue
	return configValue.Value
}

func (p *EnvironConfigReader) StringSlice(name string, defaultval []string) []string {
	strval := p.StringVal(name, "")
	if strval != "" {
		return strings.Split(strval, ",")
	}
	entry, _ := p.configValues[name]
	entry.Value = strings.Join(defaultval,",")
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) IntVal(name string, defaultval int) int {
	strval := p.StringVal(name, "")
	if strval != "" {
		if val, err := strconv.Atoi(strval); err == nil {
			return val
		}
	}
	entry, _ := p.configValues[name]
	entry.Value = fmt.Sprintf("%d", defaultval)
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) BoolVal(name string, defaultval bool) bool {
	strval := p.StringVal(name, "")
	if strval != "" {
		val := strings.ToLower(strval)

		trues := []string{"1", "true", "t", "yes"}
		for _, t := range trues {
			if val == t {
				return true
			}
		}

		falses := []string{"0", "false", "f", "no"}
		for _, f := range falses {
			if val == f {
				return false
			}
		}
	}
	entry, _ := p.configValues[name]
	entry.Value = fmt.Sprintf("%v", defaultval)
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) GetConfigValues() map[string]ConfigValue {
	return p.configValues
}

func keyvalue(line []byte) error {
	pair := string(line)
	if idx := strings.Index(pair, "="); idx >= 0 {
		key, value := strings.TrimSpace(pair[:idx]), translate(strings.TrimSpace(pair[idx+1:]))
		if err := os.Setenv(key, value); err != nil {
			return err
		}
	} else if pair != "" {
		return ParseError{pair}
	}
	return nil
}

func translate(value string) string {
	result, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("echo -n \"%s\"", value)).Output()
	if err != nil {
		return ""
	}
	return string(result)
}

func (p *EnvironConfigReader) getFullValueName(name string) string {
	return p.prefix + name
}
<commit_msg>Include values parsed from the config file<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

const buffersize = 1024

type ParseError struct {
	String string
}

func (err ParseError) Error() string {
	return fmt.Sprintf("could not parse line: %s", err.String)
}

type ConfigValue struct {
	Name  string
	Value string
}

type ConfigReader interface {
	StringVal(key, dflt string) string
	StringSlice(key string, dflt []string) []string
	IntVal(key string, dflt int) int
	BoolVal(key string, dflt bool) bool
	GetConfigValues() map[string]ConfigValue
}

type EnvironConfigReader struct {
	prefix string
	configValues map[string]ConfigValue
}

func NewEnvironConfigReader(filename, prefix string) (*EnvironConfigReader, error) {
	r := &EnvironConfigReader{prefix, map[string]ConfigValue{}}

	file, err := os.Open(filename)
	if err != nil {
		return r, err
	}
	defer file.Close()
	if err := r.parse(file); err != nil {
		return r, err
	}
	return r, nil
}

// NewEnvironOnlyConfigReader creates an EnvironConfigReader without parsing
// a file first.
func NewEnvironOnlyConfigReader(prefix string) *EnvironConfigReader {
	return &EnvironConfigReader{prefix, map[string]ConfigValue{}}
}

// parse is a really dumb reader parser.  It maps only key values in the form
// of key=value and strips whitespaces surrounding either field.  If the format
// does not match, then and error will return.
func (p *EnvironConfigReader) parse(reader io.Reader) error {
	var (
		line string
		err  error
	)

	bufReader := bufio.NewReader(reader)
	for err != io.EOF {
		line, err = bufReader.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}

		line = strings.TrimSpace(strings.Split(line, "#")[0])
		if err := p.keyvalue([]byte(line)); err != nil {
			return err
		}
	}
	return nil
}

func (p *EnvironConfigReader) StringVal(name string, defaultval string) string {
	configValue := ConfigValue{
		Name:  p.getFullValueName(name),
		Value: defaultval,
	}

	if val := os.Getenv(configValue.Name); val != "" {
		configValue.Value = val
	}
	p.configValues[name] = configValue
	return configValue.Value
}

func (p *EnvironConfigReader) StringSlice(name string, defaultval []string) []string {
	strval := p.StringVal(name, "")
	if strval != "" {
		return strings.Split(strval, ",")
	}
	entry, _ := p.configValues[name]
	entry.Value = strings.Join(defaultval,",")
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) IntVal(name string, defaultval int) int {
	strval := p.StringVal(name, "")
	if strval != "" {
		if val, err := strconv.Atoi(strval); err == nil {
			return val
		}
	}
	entry, _ := p.configValues[name]
	entry.Value = fmt.Sprintf("%d", defaultval)
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) BoolVal(name string, defaultval bool) bool {
	strval := p.StringVal(name, "")
	if strval != "" {
		val := strings.ToLower(strval)

		trues := []string{"1", "true", "t", "yes"}
		for _, t := range trues {
			if val == t {
				return true
			}
		}

		falses := []string{"0", "false", "f", "no"}
		for _, f := range falses {
			if val == f {
				return false
			}
		}
	}
	entry, _ := p.configValues[name]
	entry.Value = fmt.Sprintf("%v", defaultval)
	p.configValues[name] = entry
	return defaultval
}

func (p *EnvironConfigReader) GetConfigValues() map[string]ConfigValue {
	return p.configValues
}

func (p *EnvironConfigReader) keyvalue(line []byte) error {
	pair := string(line)
	if idx := strings.Index(pair, "="); idx >= 0 {
		key, value := strings.TrimSpace(pair[:idx]), translate(strings.TrimSpace(pair[idx+1:]))
		if err := os.Setenv(key, value); err != nil {
			return err
		}
		configValue := ConfigValue{
			Name: key,
			Value: value,
		}
		if strings.HasPrefix(key, p.prefix) {
			key = strings.TrimPrefix(key, p.prefix)
		}
		p.configValues[key] = configValue
	} else if pair != "" {
		return ParseError{pair}
	}
	return nil
}

func translate(value string) string {
	result, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("echo -n \"%s\"", value)).Output()
	if err != nil {
		return ""
	}
	return string(result)
}

func (p *EnvironConfigReader) getFullValueName(name string) string {
	return p.prefix + name
}
<|endoftext|>


<commit_before>package tmysql

import (
	"fmt"
	"math/rand"
	"net/url"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
)

var (
	engines        map[string]gorm.DB
	connContainer  map[string]string
	connMySqlWrite string
	connMySqlRead  []string
)

func init() {
	engines = make(map[string]gorm.DB)
	connContainer = make(map[string]string)
}

func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = getMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = getMySqlConnString(container)
		}
	}
}
func getMySqlConnString(container string) (connStr string) {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}
	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	connStr = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
	connContainer[connStr] = container
	return
}

func getDefaultConn(write bool) string {
	if write {
		initMaster()
		return connMySqlWrite
	} else {
		initSlaves()
		if connMySqlRead == nil || len(connMySqlRead) == 0 {
			initMaster()
			return connMySqlWrite
		}
		return connMySqlRead[rand.Intn(len(connMySqlRead))]
	}
}

//newConn will create new database connection and initialize all database setting
func newConn(connStr string) (dbConn gorm.DB, err error) {
	if dbConn, err = gorm.Open("mysql", connStr); err == nil {
		engines[connStr] = dbConn
		container := connContainer[connStr]
		connMaxLifetime := conf.Int(container+".maxlifetime", 10)
		fmt.Println(connMaxLifetime)
		maxIdleConns := conf.Int(container+".maxidleconn", 10)
		maxOpenConns := conf.Int(container+".maxopenconn", 200)
		dbConn.DB().SetConnMaxLifetime(time.Second * time.Duration(connMaxLifetime))
		dbConn.DB().SetMaxIdleConns(maxIdleConns)
		dbConn.DB().SetMaxOpenConns(maxOpenConns)
	}
	return
}

//Get MySql connection
func GetMySqlConn(writable bool) (dbConn gorm.DB, err error) {
	var connExists bool
	connStr := getDefaultConn(writable)
	if dbConn, connExists = engines[connStr]; connExists == true {
		err = dbConn.DB().Ping()
	}
	if connExists == false || err != nil {
		dbConn, err = newConn(connStr)
	}
	return
}
<commit_msg>PRA-TMYSQL-POOL: debug fmt removed<commit_after>package tmysql

import (
	"fmt"
	"math/rand"
	"net/url"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
)

var (
	engines        map[string]gorm.DB
	connContainer  map[string]string
	connMySqlWrite string
	connMySqlRead  []string
)

func init() {
	engines = make(map[string]gorm.DB)
	connContainer = make(map[string]string)
}

func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = getMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = getMySqlConnString(container)
		}
	}
}
func getMySqlConnString(container string) (connStr string) {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}
	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	connStr = fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
	connContainer[connStr] = container
	return
}

func getDefaultConn(write bool) string {
	if write {
		initMaster()
		return connMySqlWrite
	} else {
		initSlaves()
		if connMySqlRead == nil || len(connMySqlRead) == 0 {
			initMaster()
			return connMySqlWrite
		}
		return connMySqlRead[rand.Intn(len(connMySqlRead))]
	}
}

//newConn will create new database connection and initialize all database setting
func newConn(connStr string) (dbConn gorm.DB, err error) {
	if dbConn, err = gorm.Open("mysql", connStr); err == nil {
		engines[connStr] = dbConn
		container := connContainer[connStr]
		connMaxLifetime := conf.Int(container+".maxlifetime", 10)
		maxIdleConns := conf.Int(container+".maxidleconn", 10)
		maxOpenConns := conf.Int(container+".maxopenconn", 200)
		dbConn.DB().SetConnMaxLifetime(time.Second * time.Duration(connMaxLifetime))
		dbConn.DB().SetMaxIdleConns(maxIdleConns)
		dbConn.DB().SetMaxOpenConns(maxOpenConns)
	}
	return
}

//Get MySql connection
func GetMySqlConn(writable bool) (dbConn gorm.DB, err error) {
	var connExists bool
	connStr := getDefaultConn(writable)
	if dbConn, connExists = engines[connStr]; connExists == true {
		err = dbConn.DB().Ping()
	}
	if connExists == false || err != nil {
		dbConn, err = newConn(connStr)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
  "os"
  "fmt"
  "io"
  "bufio"
  "strings"
  "path/filepath"
  "errors"
  "go/token"
  "go/parser"
  "go/printer"
)

const verbose bool = false

var output ByteBuffer = ByteBuffer{ &[]byte{} }

type ByteBuffer struct {
  bytes *[]byte
}
func (buffer ByteBuffer) Write(bytes []byte) (n int, err error) {
  *buffer.bytes = append(*buffer.bytes, bytes...)
  return len(bytes), nil
}
func (buffer ByteBuffer) Bytes() []byte {
  return *buffer.bytes
}

//--- Linear pattern matcher

type Pattern struct {
  Text []rune
  Length, Pos int
}

func newPattern(s string) Pattern {
  runes := []rune(s)
  return Pattern{ Text: runes, Length: len(runes) }
}

// Next returns true if Pos advances past the last character of Text.
func (pattern *Pattern) Next(ch rune) bool {
  // If Pos is past the end of Text, reset it to the beginning.
  if pattern.Pos == pattern.Length {
    pattern.Pos = 0
  }
  // Try to match the current rune in Text.
  if ch == pattern.Text[pattern.Pos] {
    pattern.Pos++
  }
  // Check for a complete match.
  return pattern.Pos == pattern.Length
}

//--- Template parsing and output generation

type TemplateEntry struct {
  SitePath, HardPath string
  FileInfo os.FileInfo
  InsertionLine int
}

func (entry TemplateEntry) String() string {
  if entry.InsertionLine == 0 {
    return entry.SitePath
  }
  return fmt.Sprintf("-> line %d: %s", entry.InsertionLine, entry.SitePath)
}

func makeTemplateEntry(siteRoot, startDir, sitePath string,
    insertionLine int) (*TemplateEntry, error) {
  hardPath := makeHardPath(siteRoot, startDir, sitePath)
  fileInfo, error := os.Stat(hardPath)
  if error != nil {
    return nil, error
  }
  entry := TemplateEntry{
      SitePath: sitePath,
      HardPath: hardPath,
      FileInfo: fileInfo,
      InsertionLine: insertionLine,
    }
  return &entry, nil
}

func makeHardPath(siteRoot, startDir, sitePath string) string {
  // A hard path names a location in the physical file system rather than
  //  in the website's directory structure. It is either an absolute path
  //  or a relative path with respect to the directory containing the
  //  top-level template that is being parsed.
  var dir string
  if filepath.IsAbs(sitePath) {
    dir = siteRoot
  } else {
    dir = startDir
  }
  // Note that filepath.Join automatically performs filepath.Clean, thus
  //  returning a lexically unique form of the path. However, the path
  //  does not uniquely identify a file if it includes a symbolic link.
  //  Therefore, we cannot rely on string comparison to prevent cycles.
  hardPath := filepath.Join(dir, sitePath)
  return hardPath
}

func parse(templatePath string) error {

  // We resolve absolute paths with the website root.
  siteRoot := "/var/www/dd1"  // Stub. We'll get the real value from Apache.
  // We resolve relative paths using the starting directory.
  startDir := filepath.Dir(templatePath)

  // Make an insertion stack with a top-level entry.
  entry, error := makeTemplateEntry(siteRoot, startDir, templatePath, 0)
  if error != nil {
    return error
  }
  stack := []*TemplateEntry{ entry }
  return doParse(siteRoot, startDir, stack)
}

func doParse(siteRoot, startDir string, stack []*TemplateEntry) error {
  current := stack[len(stack)-1]
  if verbose {
    fmt.Fprintf(output, "// start \"%s\"\n", current.SitePath)
  }
  var topLevel bool
  if len(stack) == 1 {
    topLevel = true
  }

  // Check for an insertion cycle.
  for i := len(stack)-2; i >= 0; i-- {
    ancestor := stack[i]
    if os.SameFile(ancestor.FileInfo, current.FileInfo) {
      lines := []string{ "doParse: insertion cycle" }
      for j := i; j < len(stack); j++ {
        lines = append(lines, stack[j].String())
      }
      message := fmt.Sprintf(strings.Join(lines, "\n  "))
      return errors.New(message)
    }
  }

  var error error
  var file *os.File

  file, error = os.Open(current.HardPath)
  if error != nil {
    return error
  }

  reader := bufio.NewReader(file)
  writer := bufio.NewWriter(os.Stdout)
  defer writer.Flush()

  codePattern := newPattern("<?code")
  insertPattern := newPattern("<?insert")
  openPatterns := []*Pattern{ &codePattern, &insertPattern }
  var open *Pattern
  close := newPattern("?>")

  var buffer []rune
  var ch rune
  var size int
  countBytes, countRunes := 0, 0
  lineIndex := 1
  prefix := true

  for {
    ch, size, error = reader.ReadRune()
    if error == nil {
      buffer = append(buffer, ch)
      countBytes += size
      countRunes += 1
      if ch == '\n' {
        lineIndex += 1
      }
    } else {
      content := string(buffer)
      if topLevel {
        content = strings.TrimSpace(content)
      }
      emitStatic(content)
      break
    }

    if open == nil {
      for _, pattern := range openPatterns {
        if pattern.Next(ch) {
          open = pattern
          content := string(buffer[0:len(buffer)-open.Length])
          if prefix {
            if topLevel {
              content = strings.TrimSpace(content)
            }
            prefix = false
          }
          emitStatic(content)
          buffer = []rune{}
        }
      }
    } else {
      if close.Next(ch) {
        content := buffer[0:len(buffer)-close.Length]
        if open == &codePattern {
          emitCode(string(content))
        } else if open == &insertPattern {
          childPath := strings.TrimSpace(string(content))
          entry, error := makeTemplateEntry(siteRoot, startDir, childPath,
              lineIndex)
          if error != nil {
            return error
          }
          stack = append(stack, entry)
          error = doParse(siteRoot, startDir, stack)
          if error != nil {
            return error
          }
          stack = stack[0:len(stack)-1]
        }
        open = nil
        buffer = []rune{}
      }
    }
  }
  if verbose {
    fmt.Fprintf(output, "// finish \"%s\"\n", current.SitePath)
    fmt.Fprintf(output, "// read %d bytes, %d runes\n", countBytes, countRunes)
    fmt.Fprintf(output, "// finished on line %d\n", lineIndex)
  }
  if error == io.EOF {
    return nil
  }
  return error
}

func emitCode(content string) {
  fmt.Fprint(output, content)
}

func emitStatic(content string) {
  if len(content) == 0 {
    return
  }
  from := 0
  for pos, ch := range content {
    if ch == '`' {
      if pos != from {
        raw := fmt.Sprintf("`%s`", content[from:pos])
        emitRaw(raw)
      }
      emitRaw("'`'")
      from = pos+1
    }
  }
  if from != len(content) {
    raw := fmt.Sprintf("`%s`", content[from:len(content)])
    emitRaw(raw)
  }
}
func emitRaw(s string) {
  fmt.Fprintf(output, "fmt.Print(%s)\n", s)
}

func main() {
  writer := bufio.NewWriter(os.Stdout)
  defer writer.Flush()

  numFiles := len(os.Args)-1
  if numFiles == 0 {
    writer.WriteString("No files specified.\n")
    return
  }
  for argIx := 1; argIx <= numFiles; argIx++ {
    path := os.Args[argIx]
    error := parse(path)
    if error == nil {
      fileSet := token.NewFileSet()
      source, error := parser.ParseFile(fileSet, "output", output.Bytes(), 0)
      if error == nil {
        printer.Fprint(writer, fileSet, source)
      } else {
        writer.Write(output.Bytes())
        writer.WriteString("\n----------\n")
        writer.WriteString(fmt.Sprintf("Go parsing error: %s\n", error))
      }
    } else {
      writer.WriteString(fmt.Sprintf("template parsing error: %s\n", error))
    }
  }
}
<commit_msg>Replace ByteBuffer with bytes.Buffer<commit_after>package main

import (
  "os"
  "fmt"
  "io"
  "bufio"
  "strings"
  "path/filepath"
  "errors"
  "bytes"
  "go/token"
  "go/parser"
  "go/printer"
)

const verbose bool = false

var output bytes.Buffer

//--- Linear pattern matcher

type Pattern struct {
  Text []rune
  Length, Pos int
}

func newPattern(s string) Pattern {
  runes := []rune(s)
  return Pattern{ Text: runes, Length: len(runes) }
}

// Next returns true if Pos advances past the last character of Text.
func (pattern *Pattern) Next(ch rune) bool {
  // If Pos is past the end of Text, reset it to the beginning.
  if pattern.Pos == pattern.Length {
    pattern.Pos = 0
  }
  // Try to match the current rune in Text.
  if ch == pattern.Text[pattern.Pos] {
    pattern.Pos++
  }
  // Check for a complete match.
  return pattern.Pos == pattern.Length
}

//--- Template parsing and output generation

type TemplateEntry struct {
  SitePath, HardPath string
  FileInfo os.FileInfo
  InsertionLine int
}

func (entry TemplateEntry) String() string {
  if entry.InsertionLine == 0 {
    return entry.SitePath
  }
  return fmt.Sprintf("-> line %d: %s", entry.InsertionLine, entry.SitePath)
}

func makeTemplateEntry(siteRoot, startDir, sitePath string,
    insertionLine int) (*TemplateEntry, error) {
  hardPath := makeHardPath(siteRoot, startDir, sitePath)
  fileInfo, error := os.Stat(hardPath)
  if error != nil {
    return nil, error
  }
  entry := TemplateEntry{
      SitePath: sitePath,
      HardPath: hardPath,
      FileInfo: fileInfo,
      InsertionLine: insertionLine,
    }
  return &entry, nil
}

func makeHardPath(siteRoot, startDir, sitePath string) string {
  // A hard path names a location in the physical file system rather than
  //  in the website's directory structure. It is either an absolute path
  //  or a relative path with respect to the directory containing the
  //  top-level template that is being parsed.
  var dir string
  if filepath.IsAbs(sitePath) {
    dir = siteRoot
  } else {
    dir = startDir
  }
  // Note that filepath.Join automatically performs filepath.Clean, thus
  //  returning a lexically unique form of the path. However, the path
  //  does not uniquely identify a file if it includes a symbolic link.
  //  Therefore, we cannot rely on string comparison to prevent cycles.
  hardPath := filepath.Join(dir, sitePath)
  return hardPath
}

func parse(templatePath string) error {

  // We resolve absolute paths with the website root.
  siteRoot := "/var/www/dd1"  // Stub. We'll get the real value from Apache.
  // We resolve relative paths using the starting directory.
  startDir := filepath.Dir(templatePath)

  // Make an insertion stack with a top-level entry.
  entry, error := makeTemplateEntry(siteRoot, startDir, templatePath, 0)
  if error != nil {
    return error
  }
  stack := []*TemplateEntry{ entry }
  return doParse(siteRoot, startDir, stack)
}

func doParse(siteRoot, startDir string, stack []*TemplateEntry) error {
  current := stack[len(stack)-1]
  if verbose {
    fmt.Fprintf(&output, "// start \"%s\"\n", current.SitePath)
  }
  var topLevel bool
  if len(stack) == 1 {
    topLevel = true
  }

  // Check for an insertion cycle.
  for i := len(stack)-2; i >= 0; i-- {
    ancestor := stack[i]
    if os.SameFile(ancestor.FileInfo, current.FileInfo) {
      lines := []string{ "doParse: insertion cycle" }
      for j := i; j < len(stack); j++ {
        lines = append(lines, stack[j].String())
      }
      message := fmt.Sprintf(strings.Join(lines, "\n  "))
      return errors.New(message)
    }
  }

  var error error
  var file *os.File

  file, error = os.Open(current.HardPath)
  if error != nil {
    return error
  }

  reader := bufio.NewReader(file)
  writer := bufio.NewWriter(os.Stdout)
  defer writer.Flush()

  codePattern := newPattern("<?code")
  insertPattern := newPattern("<?insert")
  openPatterns := []*Pattern{ &codePattern, &insertPattern }
  var open *Pattern
  close := newPattern("?>")

  var buffer []rune
  var ch rune
  var size int
  countBytes, countRunes := 0, 0
  lineIndex := 1
  prefix := true

  for {
    ch, size, error = reader.ReadRune()
    if error == nil {
      buffer = append(buffer, ch)
      countBytes += size
      countRunes += 1
      if ch == '\n' {
        lineIndex += 1
      }
    } else {
      content := string(buffer)
      if topLevel {
        content = strings.TrimSpace(content)
      }
      emitStatic(content)
      break
    }

    if open == nil {
      for _, pattern := range openPatterns {
        if pattern.Next(ch) {
          open = pattern
          content := string(buffer[0:len(buffer)-open.Length])
          if prefix {
            if topLevel {
              content = strings.TrimSpace(content)
            }
            prefix = false
          }
          emitStatic(content)
          buffer = []rune{}
        }
      }
    } else {
      if close.Next(ch) {
        content := buffer[0:len(buffer)-close.Length]
        if open == &codePattern {
          emitCode(string(content))
        } else if open == &insertPattern {
          childPath := strings.TrimSpace(string(content))
          entry, error := makeTemplateEntry(siteRoot, startDir, childPath,
              lineIndex)
          if error != nil {
            return error
          }
          stack = append(stack, entry)
          error = doParse(siteRoot, startDir, stack)
          if error != nil {
            return error
          }
          stack = stack[0:len(stack)-1]
        }
        open = nil
        buffer = []rune{}
      }
    }
  }
  if verbose {
    fmt.Fprintf(&output, "// finish \"%s\"\n", current.SitePath)
    fmt.Fprintf(&output, "// read %d bytes, %d runes\n", countBytes, countRunes)
    fmt.Fprintf(&output, "// finished on line %d\n", lineIndex)
  }
  if error == io.EOF {
    return nil
  }
  return error
}

func emitCode(content string) {
  fmt.Fprint(&output, content)
}

func emitStatic(content string) {
  if len(content) == 0 {
    return
  }
  from := 0
  for pos, ch := range content {
    if ch == '`' {
      if pos != from {
        raw := fmt.Sprintf("`%s`", content[from:pos])
        emitRaw(raw)
      }
      emitRaw("'`'")
      from = pos+1
    }
  }
  if from != len(content) {
    raw := fmt.Sprintf("`%s`", content[from:len(content)])
    emitRaw(raw)
  }
}
func emitRaw(s string) {
  fmt.Fprintf(&output, "fmt.Print(%s)\n", s)
}

func main() {
  writer := bufio.NewWriter(os.Stdout)
  defer writer.Flush()

  numFiles := len(os.Args)-1
  if numFiles == 0 {
    writer.WriteString("No files specified.\n")
    return
  }
  for argIx := 1; argIx <= numFiles; argIx++ {
    path := os.Args[argIx]
    error := parse(path)
    if error == nil {
      fileSet := token.NewFileSet()
      source, error := parser.ParseFile(fileSet, "output", output.Bytes(), 0)
      if error == nil {
        printer.Fprint(writer, fileSet, source)
      } else {
        writer.Write(output.Bytes())
        writer.WriteString("\n----------\n")
        writer.WriteString(fmt.Sprintf("Go parsing error: %s\n", error))
      }
    } else {
      writer.WriteString(fmt.Sprintf("template parsing error: %s\n", error))
    }
  }
}
<|endoftext|>


<commit_before>package send

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/bluele/slack"
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
)

const (
	slackClientToken = "GRIP_SLACK_CLIENT_TOKEN"
)

type slackJournal struct {
	channel  string
	hostName string
	fallback *log.Logger
	client   *slack.Slack
	*base
}

// NewSlackLogger constructs a Sender that posts messages to a slack,
// given a slack API token.
//
// You must specify the channel that will receive the messages, and
// the hostname of the current machine, which is added to the logging
// metadata.
func NewSlackLogger(name, token, channel, hostname string, l LevelInfo) (Sender, error) {
	s := &slackJournal{
		hostName: hostname,
		client:   slack.New(token),
		base:     newBase(name),
	}

	if !strings.HasPrefix(channel, "#") {
		s.channel = "#" + channel
	} else {
		s.channel = channel
	}

	if err := s.SetLevel(l); err != nil {
		return nil, err
	}

	if _, err := s.client.AuthTest(); err != nil {
		return nil, fmt.Errorf("slack authentication error: %v", err)
	}

	s.reset = func() {
		s.fallback = log.New(os.Stdout, strings.Join([]string{"[", s.Name(), "] "}, ""), log.LstdFlags)
	}

	s.reset()

	return s, nil
}

// NewSlackDefault is equivalent to NewSlackLogger, but constructs a
// Sender reading the slack token from the environment variable
// "GRIP_SLACK_CLIENT_TOKEN".
func NewSlackDefault(name, channel string, l LevelInfo) (Sender, error) {
	token, ok := os.LookupEnv(slackClientToken)
	if !ok {
		return nil, fmt.Errorf("environment variable %s not defined, cannot create slack client",
			"foo")
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, fmt.Errorf("error resolving hostname for slack logger: %v", err)
	}

	return NewSlackLogger(name, token, channel, hostname, l)
}

func (s *slackJournal) Type() SenderType { return Slack }

func (s *slackJournal) Send(m message.Composer) {
	if !s.level.ShouldLog(m) {
		return
	}

	if fallback, err := s.doSend(m); err != nil {
		s.fallback.Println("slack error:", err.Error())
		s.fallback.Println(fallback)
	}
}

func (s *slackJournal) doSend(m message.Composer) (string, error) {
	msg := m.Resolve()

	s.RLock()
	var channel []byte
	copy(channel, s.channel)
	params := getParams(s.name, s.hostName, m.Priority())
	s.RUnlock()

	if err := s.client.ChatPostMessage(string(channel), msg, params); err != nil {
		return fmt.Sprintf("%s: %s\n", params.Attachments[0].Fallback, msg), err
	}

	return "", nil
}

func getParams(log, host string, p level.Priority) *slack.ChatPostMessageOpt {
	params := slack.ChatPostMessageOpt{
		Attachments: []*slack.Attachment{
			{
				Fallback: fmt.Sprintf("[level=%s, process=%s, host=%s]",
					p, log, host),
				Fields: []*slack.AttachmentField{
					{
						Title: "Host",
						Value: host,
						Short: true,
					},
					{
						Title: "Process",
						Value: log,
						Short: true,
					},
					{
						Title: "Priority",
						Value: p.String(),
						Short: true,
					},
				},
			},
		},
	}

	switch p {
	case level.Emergency, level.Alert, level.Critical:
		params.Attachments[0].Color = "danger"
	case level.Warning, level.Notice:
		params.Attachments[0].Color = "warning"
	default: // includes info/debug
		params.Attachments[0].Color = "good"

	}

	return &params
}
<commit_msg>fix go14 compat<commit_after>package send

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/bluele/slack"
	"github.com/tychoish/grip/level"
	"github.com/tychoish/grip/message"
)

const (
	slackClientToken = "GRIP_SLACK_CLIENT_TOKEN"
)

type slackJournal struct {
	channel  string
	hostName string
	fallback *log.Logger
	client   *slack.Slack
	*base
}

// NewSlackLogger constructs a Sender that posts messages to a slack,
// given a slack API token.
//
// You must specify the channel that will receive the messages, and
// the hostname of the current machine, which is added to the logging
// metadata.
func NewSlackLogger(name, token, channel, hostname string, l LevelInfo) (Sender, error) {
	s := &slackJournal{
		hostName: hostname,
		client:   slack.New(token),
		base:     newBase(name),
	}

	if !strings.HasPrefix(channel, "#") {
		s.channel = "#" + channel
	} else {
		s.channel = channel
	}

	if err := s.SetLevel(l); err != nil {
		return nil, err
	}

	if _, err := s.client.AuthTest(); err != nil {
		return nil, fmt.Errorf("slack authentication error: %v", err)
	}

	s.reset = func() {
		s.fallback = log.New(os.Stdout, strings.Join([]string{"[", s.Name(), "] "}, ""), log.LstdFlags)
	}

	s.reset()

	return s, nil
}

// NewSlackDefault is equivalent to NewSlackLogger, but constructs a
// Sender reading the slack token from the environment variable
// "GRIP_SLACK_CLIENT_TOKEN".
func NewSlackDefault(name, channel string, l LevelInfo) (Sender, error) {
	token := os.Getenv(slackClientToken)
	if token == "" {
		return nil, fmt.Errorf("environment variable %s not defined, cannot create slack client",
			"foo")
	}

	hostname, err := os.Hostname()
	if err != nil {
		return nil, fmt.Errorf("error resolving hostname for slack logger: %v", err)
	}

	return NewSlackLogger(name, token, channel, hostname, l)
}

func (s *slackJournal) Type() SenderType { return Slack }

func (s *slackJournal) Send(m message.Composer) {
	if !s.level.ShouldLog(m) {
		return
	}

	if fallback, err := s.doSend(m); err != nil {
		s.fallback.Println("slack error:", err.Error())
		s.fallback.Println(fallback)
	}
}

func (s *slackJournal) doSend(m message.Composer) (string, error) {
	msg := m.Resolve()

	s.RLock()
	var channel []byte
	copy(channel, s.channel)
	params := getParams(s.name, s.hostName, m.Priority())
	s.RUnlock()

	if err := s.client.ChatPostMessage(string(channel), msg, params); err != nil {
		return fmt.Sprintf("%s: %s\n", params.Attachments[0].Fallback, msg), err
	}

	return "", nil
}

func getParams(log, host string, p level.Priority) *slack.ChatPostMessageOpt {
	params := slack.ChatPostMessageOpt{
		Attachments: []*slack.Attachment{
			{
				Fallback: fmt.Sprintf("[level=%s, process=%s, host=%s]",
					p, log, host),
				Fields: []*slack.AttachmentField{
					{
						Title: "Host",
						Value: host,
						Short: true,
					},
					{
						Title: "Process",
						Value: log,
						Short: true,
					},
					{
						Title: "Priority",
						Value: p.String(),
						Short: true,
					},
				},
			},
		},
	}

	switch p {
	case level.Emergency, level.Alert, level.Critical:
		params.Attachments[0].Color = "danger"
	case level.Warning, level.Notice:
		params.Attachments[0].Color = "warning"
	default: // includes info/debug
		params.Attachments[0].Color = "good"

	}

	return &params
}
<|endoftext|>


<commit_before>// Go binding for libGammu (library to work with different cell phones)
package gammu

/*
#include <gammu.h>

void sendCallback(GSM_StateMachine *sm, int status, int msgRef, void *data) {
	if (status==0) {
		*((GSM_Error *) data) = ERR_NONE;
	} else {
		*((GSM_Error *) data) = ERR_UNKNOWN;
	}
}
void setStatusCallback(GSM_StateMachine *sm, GSM_Error *status) {
	GSM_SetSendSMSStatusCallback(sm, sendCallback, status);
}
GSM_Debug_Info *debug_info;
void setDebug() {
	debug_info = GSM_GetGlobalDebug();
	GSM_SetDebugFileDescriptor(stderr, TRUE, debug_info);
	GSM_SetDebugLevel("textall", debug_info);
}

#cgo pkg-config: gammu
*/
import "C"
import (
	"fmt"
	"io"
	"runtime"
	"time"
	"unsafe"
)

// Error
type Error struct {
	descr string
	g     C.GSM_Error
}

func (e Error) Error() string {
	return fmt.Sprintf(
		"[%s] %s", e.descr, C.GoString(C.GSM_ErrorString(C.GSM_Error(e.g))),
	)
}

// StateMachine
type StateMachine struct {
	g      *C.GSM_StateMachine
	smsc   C.GSM_SMSC
	status C.GSM_Error

	Timeout time.Duration // Default 15s
}

// Creates new state maschine using cf configuration file or default
// configuration file if cf == "".
func NewStateMachine(cf string) (*StateMachine, error) {
	//C.setDebug()
	var config *C.INI_Section
	if cf != "" {
		cs := C.CString(cf)
		defer C.free(unsafe.Pointer(cs))
		if e := C.GSM_FindGammuRC(&config, cs); e != C.ERR_NONE {
			return nil, Error{"FindGammuRC", e}
		}
	} else {
		if e := C.GSM_FindGammuRC(&config, nil); e != C.ERR_NONE {
			return nil, Error{"FindGammuRC", e}
		}
	}
	defer C.INI_Free(config)

	sm := new(StateMachine)
	sm.g = C.GSM_AllocStateMachine()
	if sm.g == nil {
		panic("out of memory")
	}

	if e := C.GSM_ReadConfig(config, C.GSM_GetConfig(sm.g, 0), 0); e != C.ERR_NONE {
		sm.free()
		return nil, Error{"ReadConfig", e}
	}
	C.GSM_SetConfigNum(sm.g, 1)
	sm.Timeout = 15 * time.Second

	runtime.SetFinalizer(sm, (*StateMachine).free)
	return sm, nil
}

func (sm *StateMachine) free() {
	if sm.IsConnected() {
		sm.Disconnect()
	}
	C.GSM_FreeStateMachine(sm.g)
	sm.g = nil
}

func (sm *StateMachine) Connect() error {
	if e := C.GSM_InitConnection(sm.g, 1); e != C.ERR_NONE {
		return Error{"InitConnection", e}
	}
	C.setStatusCallback(sm.g, &sm.status)
	sm.smsc.Location = 1
	if e := C.GSM_GetSMSC(sm.g, &sm.smsc); e != C.ERR_NONE {
		return Error{"GetSMSC", e}
	}
	return nil
}

func (sm *StateMachine) IsConnected() bool {
	return C.GSM_IsConnected(sm.g) != 0
}

func (sm *StateMachine) Disconnect() error {
	if e := C.GSM_TerminateConnection(sm.g); e != C.ERR_NONE {
		return Error{"TerminateConnection", e}
	}
	return nil
}

func (sm *StateMachine) Reset() error {
	if e := C.GSM_Reset(sm.g, 0); e != C.ERR_NONE {
		return Error{"Reset", e}
	}
	return nil
}

func (sm *StateMachine) HardReset() error {
	if e := C.GSM_Reset(sm.g, 1); e != C.ERR_NONE {
		return Error{"Reset", e}
	}
	return nil
}

func decodeUTF8(out *C.uchar, in string) {
	cn := C.CString(in)
	C.DecodeUTF8(out, cn, C.int(len(in)))
	C.free(unsafe.Pointer(cn))
}

func (sm *StateMachine) sendSMS(sms *C.GSM_SMSMessage, number string, report bool) error {
	C.CopyUnicodeString(&sms.SMSC.Number[0], &sm.smsc.Number[0])
	decodeUTF8(&sms.Number[0], number)
	if report {
		sms.PDU = C.SMS_Status_Report
	} else {
		sms.PDU = C.SMS_Submit
	}
	// Send mepssage
	sm.status = C.ERR_TIMEOUT
	if e := C.GSM_SendSMS(sm.g, sms); e != C.ERR_NONE {
		return Error{"SendSMS", e}
	}
	// Wait for reply
	t := time.Now()
	for time.Now().Sub(t) < sm.Timeout {
		C.GSM_ReadDevice(sm.g, C.TRUE)
		if sm.status == C.ERR_NONE {
			// Message sent OK
			break
		} else if sm.status != C.ERR_TIMEOUT {
			// Error
			break
		}
	}
	if sm.status != C.ERR_NONE {
		return Error{"ReadDevice", sm.status}
	}
	return nil
}

func (sm *StateMachine) SendSMS(number, text string, report bool) error {
	var sms C.GSM_SMSMessage
	decodeUTF8(&sms.Text[0], text)
	sms.UDH.Type = C.UDH_NoUDH
	sms.Coding = C.SMS_Coding_Default_No_Compression
	sms.Class = 1
	return sm.sendSMS(&sms, number, report)
}

func (sm *StateMachine) SendLongSMS(number, text string, report bool) error {
	// Fill in SMS info
	var smsInfo C.GSM_MultiPartSMSInfo
	C.GSM_ClearMultiPartSMSInfo(&smsInfo)
	smsInfo.Class = 1
	smsInfo.EntriesNum = 1
	smsInfo.UnicodeCoding = C.FALSE
	// Check for non-ASCII rune
	for _, r := range text {
		if r > 0x7F {
			smsInfo.UnicodeCoding = C.TRUE
			break
		}
	}
	smsInfo.Entries[0].ID = C.SMS_ConcatenatedTextLong
	msgUnicode := make([]C.uchar, (len(text)+1)*2)
	decodeUTF8(&msgUnicode[0], text)
	smsInfo.Entries[0].Buffer = &msgUnicode[0]
	// Prepare multipart message
	var msms C.GSM_MultiSMSMessage
	if e := C.GSM_EncodeMultiPartSMS(nil, &smsInfo, &msms); e != C.ERR_NONE {
		return Error{"EncodeMultiPartSMS", e}
	}
	// Send message
	for i := 0; i < int(msms.Number); i++ {
		if e := sm.sendSMS(&msms.SMS[i], number, report); e != nil {
			return e
		}
	}
	return nil
}

func encodeUTF8(in *C.uchar) string {
	out := make([]C.char, C.UnicodeLength(in)*2)
	C.EncodeUTF8(&out[0], in)
	return C.GoString(&out[0])
}

func goTime(t *C.GSM_DateTime) time.Time {
	return time.Date(
		int(t.Year), time.Month(t.Month), int(t.Day),
		int(t.Hour), int(t.Minute), int(t.Second), 0,
		time.UTC,
	).Add(-time.Second * time.Duration(t.Timezone)).Local()
}

type SMS struct {
	Time     time.Time
	SMSCTime time.Time
	Number   string
	Report   bool // True if this message is a delivery report
	Body     string
}

// Read and deletes first avaliable message.
// Returns io.EOF if there is no more messages to read
func (sm *StateMachine) GetSMS() (sms SMS, err error) {
	var msms C.GSM_MultiSMSMessage
	if e := C.GSM_GetNextSMS(sm.g, &msms, C.TRUE); e != C.ERR_NONE {
		if e == C.ERR_EMPTY {
			err = io.EOF
		} else {
			err = Error{"GetNextSMS", e}
		}
		return
	}
	s := msms.SMS[msms.Number-1]
	sms.Number = encodeUTF8(&s.Number[0])
	sms.Time = goTime(&s.DateTime)
	sms.SMSCTime = goTime(&s.SMSCTime)

	for i := 0; i < int(msms.Number); i++ {
		s = msms.SMS[i]
		if s.Coding == C.SMS_Coding_8bit {
			continue
		}
		sms.Body += encodeUTF8(&s.Text[0])
		if s.PDU == C.SMS_Status_Report {
			sms.Report = true
		}
		s.Folder = 0 // Flat
		if e := C.GSM_DeleteSMS(sm.g, &s); e != C.ERR_NONE {
			err = Error{"DeleteSMS", e}
			return
		}
	}
	return
}
<commit_msg>Fixed bug in encodeUTF8<commit_after>// Go binding for libGammu (library to work with different cell phones)
package gammu

/*
#include <gammu.h>

void sendCallback(GSM_StateMachine *sm, int status, int msgRef, void *data) {
	if (status==0) {
		*((GSM_Error *) data) = ERR_NONE;
	} else {
		*((GSM_Error *) data) = ERR_UNKNOWN;
	}
}
void setStatusCallback(GSM_StateMachine *sm, GSM_Error *status) {
	GSM_SetSendSMSStatusCallback(sm, sendCallback, status);
}
GSM_Debug_Info *debug_info;
void setDebug() {
	debug_info = GSM_GetGlobalDebug();
	GSM_SetDebugFileDescriptor(stderr, TRUE, debug_info);
	GSM_SetDebugLevel("textall", debug_info);
}

#cgo pkg-config: gammu
*/
import "C"
import (
	"fmt"
	"io"
	"runtime"
	"time"
	"unsafe"
)

// Error
type Error struct {
	descr string
	g     C.GSM_Error
}

func (e Error) Error() string {
	return fmt.Sprintf(
		"[%s] %s", e.descr, C.GoString(C.GSM_ErrorString(C.GSM_Error(e.g))),
	)
}

// StateMachine
type StateMachine struct {
	g      *C.GSM_StateMachine
	smsc   C.GSM_SMSC
	status C.GSM_Error

	Timeout time.Duration // Default 15s
}

// Creates new state maschine using cf configuration file or default
// configuration file if cf == "".
func NewStateMachine(cf string) (*StateMachine, error) {
	//C.setDebug()
	var config *C.INI_Section
	if cf != "" {
		cs := C.CString(cf)
		defer C.free(unsafe.Pointer(cs))
		if e := C.GSM_FindGammuRC(&config, cs); e != C.ERR_NONE {
			return nil, Error{"FindGammuRC", e}
		}
	} else {
		if e := C.GSM_FindGammuRC(&config, nil); e != C.ERR_NONE {
			return nil, Error{"FindGammuRC", e}
		}
	}
	defer C.INI_Free(config)

	sm := new(StateMachine)
	sm.g = C.GSM_AllocStateMachine()
	if sm.g == nil {
		panic("out of memory")
	}

	if e := C.GSM_ReadConfig(config, C.GSM_GetConfig(sm.g, 0), 0); e != C.ERR_NONE {
		sm.free()
		return nil, Error{"ReadConfig", e}
	}
	C.GSM_SetConfigNum(sm.g, 1)
	sm.Timeout = 15 * time.Second

	runtime.SetFinalizer(sm, (*StateMachine).free)
	return sm, nil
}

func (sm *StateMachine) free() {
	if sm.IsConnected() {
		sm.Disconnect()
	}
	C.GSM_FreeStateMachine(sm.g)
	sm.g = nil
}

func (sm *StateMachine) Connect() error {
	if e := C.GSM_InitConnection(sm.g, 1); e != C.ERR_NONE {
		return Error{"InitConnection", e}
	}
	C.setStatusCallback(sm.g, &sm.status)
	sm.smsc.Location = 1
	if e := C.GSM_GetSMSC(sm.g, &sm.smsc); e != C.ERR_NONE {
		return Error{"GetSMSC", e}
	}
	return nil
}

func (sm *StateMachine) IsConnected() bool {
	return C.GSM_IsConnected(sm.g) != 0
}

func (sm *StateMachine) Disconnect() error {
	if e := C.GSM_TerminateConnection(sm.g); e != C.ERR_NONE {
		return Error{"TerminateConnection", e}
	}
	return nil
}

func (sm *StateMachine) Reset() error {
	if e := C.GSM_Reset(sm.g, 0); e != C.ERR_NONE {
		return Error{"Reset", e}
	}
	return nil
}

func (sm *StateMachine) HardReset() error {
	if e := C.GSM_Reset(sm.g, 1); e != C.ERR_NONE {
		return Error{"Reset", e}
	}
	return nil
}

func decodeUTF8(out *C.uchar, in string) {
	cn := C.CString(in)
	C.DecodeUTF8(out, cn, C.int(len(in)))
	C.free(unsafe.Pointer(cn))
}

func (sm *StateMachine) sendSMS(sms *C.GSM_SMSMessage, number string, report bool) error {
	C.CopyUnicodeString(&sms.SMSC.Number[0], &sm.smsc.Number[0])
	decodeUTF8(&sms.Number[0], number)
	if report {
		sms.PDU = C.SMS_Status_Report
	} else {
		sms.PDU = C.SMS_Submit
	}
	// Send mepssage
	sm.status = C.ERR_TIMEOUT
	if e := C.GSM_SendSMS(sm.g, sms); e != C.ERR_NONE {
		return Error{"SendSMS", e}
	}
	// Wait for reply
	t := time.Now()
	for time.Now().Sub(t) < sm.Timeout {
		C.GSM_ReadDevice(sm.g, C.TRUE)
		if sm.status == C.ERR_NONE {
			// Message sent OK
			break
		} else if sm.status != C.ERR_TIMEOUT {
			// Error
			break
		}
	}
	if sm.status != C.ERR_NONE {
		return Error{"ReadDevice", sm.status}
	}
	return nil
}

func (sm *StateMachine) SendSMS(number, text string, report bool) error {
	var sms C.GSM_SMSMessage
	decodeUTF8(&sms.Text[0], text)
	sms.UDH.Type = C.UDH_NoUDH
	sms.Coding = C.SMS_Coding_Default_No_Compression
	sms.Class = 1
	return sm.sendSMS(&sms, number, report)
}

func (sm *StateMachine) SendLongSMS(number, text string, report bool) error {
	// Fill in SMS info
	var smsInfo C.GSM_MultiPartSMSInfo
	C.GSM_ClearMultiPartSMSInfo(&smsInfo)
	smsInfo.Class = 1
	smsInfo.EntriesNum = 1
	smsInfo.UnicodeCoding = C.FALSE
	// Check for non-ASCII rune
	for _, r := range text {
		if r > 0x7F {
			smsInfo.UnicodeCoding = C.TRUE
			break
		}
	}
	smsInfo.Entries[0].ID = C.SMS_ConcatenatedTextLong
	msgUnicode := make([]C.uchar, (len(text)+1)*2)
	decodeUTF8(&msgUnicode[0], text)
	smsInfo.Entries[0].Buffer = &msgUnicode[0]
	// Prepare multipart message
	var msms C.GSM_MultiSMSMessage
	if e := C.GSM_EncodeMultiPartSMS(nil, &smsInfo, &msms); e != C.ERR_NONE {
		return Error{"EncodeMultiPartSMS", e}
	}
	// Send message
	for i := 0; i < int(msms.Number); i++ {
		if e := sm.sendSMS(&msms.SMS[i], number, report); e != nil {
			return e
		}
	}
	return nil
}

func encodeUTF8(in *C.uchar) string {
	l := C.UnicodeLength(in)
	if l == 0 {
		return ""
	}
	out := make([]C.char, C.UnicodeLength(in)*2)
	C.EncodeUTF8(&out[0], in)
	return C.GoString(&out[0])
}

func goTime(t *C.GSM_DateTime) time.Time {
	return time.Date(
		int(t.Year), time.Month(t.Month), int(t.Day),
		int(t.Hour), int(t.Minute), int(t.Second), 0,
		time.UTC,
	).Add(-time.Second * time.Duration(t.Timezone)).Local()
}

type SMS struct {
	Time     time.Time
	SMSCTime time.Time
	Number   string
	Report   bool // True if this message is a delivery report
	Body     string
}

// Read and deletes first avaliable message.
// Returns io.EOF if there is no more messages to read
func (sm *StateMachine) GetSMS() (sms SMS, err error) {
	var msms C.GSM_MultiSMSMessage
	if e := C.GSM_GetNextSMS(sm.g, &msms, C.TRUE); e != C.ERR_NONE {
		if e == C.ERR_EMPTY {
			err = io.EOF
		} else {
			err = Error{"GetNextSMS", e}
		}
		return
	}
	s := msms.SMS[msms.Number-1]
	sms.Number = encodeUTF8(&s.Number[0])
	sms.Time = goTime(&s.DateTime)
	sms.SMSCTime = goTime(&s.SMSCTime)

	for i := 0; i < int(msms.Number); i++ {
		s = msms.SMS[i]
		if s.Coding == C.SMS_Coding_8bit {
			continue
		}
		sms.Body += encodeUTF8(&s.Text[0])
		if s.PDU == C.SMS_Status_Report {
			sms.Report = true
		}
		s.Folder = 0 // Flat
		if e := C.GSM_DeleteSMS(sm.g, &s); e != C.ERR_NONE {
			err = Error{"DeleteSMS", e}
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

type stats map[string]float64

func main() {
	color.Yellow("Generating statistics")

	userStats := getUserStats()
	animeStats := getAnimeStats()

	arn.PanicOnError(arn.DB.Set("Cache", "user statistics", &arn.StatisticsCategory{
		Name:      "Users",
		PieCharts: userStats,
	}))

	arn.PanicOnError(arn.DB.Set("Cache", "anime statistics", &arn.StatisticsCategory{
		Name:      "Anime",
		PieCharts: animeStats,
	}))

	color.Green("Finished.")
}

func getUserStats() []*arn.PieChart {
	println("Generating user statistics")

	analytics, err := arn.AllAnalytics()
	arn.PanicOnError(err)

	screenSize := stats{}
	pixelRatio := stats{}
	browser := stats{}
	country := stats{}
	gender := stats{}
	os := stats{}
	notifications := stats{}
	activity := stats{}
	avatar := stats{}

	for _, info := range analytics {
		pixelRatio[fmt.Sprintf("%.1f", info.Screen.PixelRatio)]++

		size := arn.ToString(info.Screen.Width) + " x " + arn.ToString(info.Screen.Height)
		screenSize[size]++
	}

	for user := range arn.MustStreamUsers() {
		if user.Gender != "" && user.Gender != "other" {
			gender[user.Gender]++
		}

		if user.Browser.Name != "" {
			browser[user.Browser.Name]++
		}

		if user.Location.CountryName != "" {
			country[user.Location.CountryName]++
		}

		if user.OS.Name != "" {
			if strings.HasPrefix(user.OS.Name, "CrOS") {
				user.OS.Name = "Chrome OS"
			}

			os[user.OS.Name]++
		}

		if len(user.PushSubscriptions().Items) > 0 {
			notifications["Enabled"]++
		} else {
			notifications["Disabled"]++
		}

		if user.IsActive() {
			activity["Active last week"]++
		} else {
			activity["Inactive"]++
		}

		if user.Avatar.Source == "" {
			avatar["none"]++
		} else {
			avatar[user.Avatar.Source]++
		}
	}

	println("Finished user statistics")

	return []*arn.PieChart{
		arn.NewPieChart("OS", os),
		arn.NewPieChart("Screen size", screenSize),
		arn.NewPieChart("Browser", browser),
		arn.NewPieChart("Country", country),
		arn.NewPieChart("Avatar", avatar),
		arn.NewPieChart("Activity", activity),
		arn.NewPieChart("Notifications", notifications),
		arn.NewPieChart("Gender", gender),
		arn.NewPieChart("Pixel ratio", pixelRatio),
	}
}

func getAnimeStats() []*arn.PieChart {
	println("Generating anime statistics")

	allAnime, err := arn.AllAnime()
	arn.PanicOnError(err)

	shoboi := stats{}
	anilist := stats{}
	mal := stats{}
	anidb := stats{}
	status := stats{}
	types := stats{}
	shoboiEdits := stats{}
	anilistEdits := stats{}
	malEdits := stats{}
	anidbEdits := stats{}
	rating := stats{}
	twist := stats{}

	for _, anime := range allAnime {
		for _, external := range anime.Mappings {
			if external.Service == "shoboi/anime" {
				if external.CreatedBy == "" {
					shoboiEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					shoboiEdits[user.Nick]++
				}
			}

			if external.Service == "anilist/anime" {
				if external.CreatedBy == "" {
					anilistEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anilistEdits[user.Nick]++
				}
			}

			if external.Service == "myanimelist/anime" {
				if external.CreatedBy == "" {
					malEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					malEdits[user.Nick]++
				}
			}

			if external.Service == "anidb/anime" {
				if external.CreatedBy == "" {
					anidbEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anidbEdits[user.Nick]++
				}
			}
		}

		if anime.GetMapping("shoboi/anime") != "" {
			shoboi["Connected with Shoboi"]++
		} else {
			shoboi["Not connected with Shoboi"]++
		}

		if anime.GetMapping("anilist/anime") != "" {
			anilist["Connected with AniList"]++
		} else {
			anilist["Not connected with AniList"]++
		}

		if anime.GetMapping("myanimelist/anime") != "" {
			mal["Connected with MyAnimeList"]++
		} else {
			mal["Not connected with MyAnimeList"]++
		}

		if anime.GetMapping("anidb/anime") != "" {
			anidb["Connected with AniDB"]++
		} else {
			anidb["Not connected with AniDB"]++
		}

		rating[arn.ToString(int(anime.Rating.Overall+0.5))]++

		found := false
		for _, episode := range anime.Episodes().Items {
			if episode.Links != nil && episode.Links["twist.moe"] != "" {
				found = true
				break
			}
		}

		if found {
			twist["Connected with AnimeTwist"]++
		} else {
			twist["Not connected with AnimeTwist"]++
		}

		status[anime.Status]++
		types[anime.Type]++
	}

	println("Finished anime statistics")

	return []*arn.PieChart{
		arn.NewPieChart("Type", types),
		arn.NewPieChart("Status", status),
		arn.NewPieChart("Rating", rating),
		arn.NewPieChart("MyAnimeList", mal),
		arn.NewPieChart("AniList", anilist),
		arn.NewPieChart("AniDB", anidb),
		arn.NewPieChart("Shoboi", shoboi),
		arn.NewPieChart("AnimeTwist", twist),
		// arn.NewPieChart("MyAnimeList Editors", malEdits),
		arn.NewPieChart("AniList Editors", anilistEdits),
		// arn.NewPieChart("AniDB Editors", anidbEdits),
		arn.NewPieChart("Shoboi Editors", shoboiEdits),
	}
}
<commit_msg>Change statistics<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/animenotifier/arn"
	"github.com/fatih/color"
)

type stats map[string]float64

func main() {
	color.Yellow("Generating statistics")

	userStats := getUserStats()
	animeStats := getAnimeStats()

	arn.PanicOnError(arn.DB.Set("Cache", "user statistics", &arn.StatisticsCategory{
		Name:      "Users",
		PieCharts: userStats,
	}))

	arn.PanicOnError(arn.DB.Set("Cache", "anime statistics", &arn.StatisticsCategory{
		Name:      "Anime",
		PieCharts: animeStats,
	}))

	color.Green("Finished.")
}

func getUserStats() []*arn.PieChart {
	println("Generating user statistics")

	analytics, err := arn.AllAnalytics()
	arn.PanicOnError(err)

	screenSize := stats{}
	pixelRatio := stats{}
	browser := stats{}
	country := stats{}
	gender := stats{}
	os := stats{}
	notifications := stats{}
	avatar := stats{}

	for _, info := range analytics {
		pixelRatio[fmt.Sprintf("%.1f", info.Screen.PixelRatio)]++

		size := arn.ToString(info.Screen.Width) + " x " + arn.ToString(info.Screen.Height)
		screenSize[size]++
	}

	for user := range arn.MustStreamUsers() {
		if !user.IsActive() {
			continue
		}

		if user.Gender != "" && user.Gender != "other" {
			gender[user.Gender]++
		}

		if user.Browser.Name != "" {
			browser[user.Browser.Name]++
		}

		if user.Location.CountryName != "" {
			country[user.Location.CountryName]++
		}

		if user.OS.Name != "" {
			if strings.HasPrefix(user.OS.Name, "CrOS") {
				user.OS.Name = "Chrome OS"
			}

			os[user.OS.Name]++
		}

		if len(user.PushSubscriptions().Items) > 0 {
			notifications["Enabled"]++
		} else {
			notifications["Disabled"]++
		}

		if user.Avatar.Source == "" {
			avatar["none"]++
		} else {
			avatar[user.Avatar.Source]++
		}
	}

	println("Finished user statistics")

	return []*arn.PieChart{
		arn.NewPieChart("OS", os),
		arn.NewPieChart("Screen size", screenSize),
		arn.NewPieChart("Browser", browser),
		arn.NewPieChart("Country", country),
		arn.NewPieChart("Avatar", avatar),
		arn.NewPieChart("Notifications", notifications),
		arn.NewPieChart("Gender", gender),
		arn.NewPieChart("Pixel ratio", pixelRatio),
	}
}

func getAnimeStats() []*arn.PieChart {
	println("Generating anime statistics")

	allAnime, err := arn.AllAnime()
	arn.PanicOnError(err)

	shoboi := stats{}
	anilist := stats{}
	mal := stats{}
	anidb := stats{}
	status := stats{}
	types := stats{}
	shoboiEdits := stats{}
	anilistEdits := stats{}
	malEdits := stats{}
	anidbEdits := stats{}
	rating := stats{}
	twist := stats{}

	for _, anime := range allAnime {
		for _, external := range anime.Mappings {
			if external.Service == "shoboi/anime" {
				if external.CreatedBy == "" {
					shoboiEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					shoboiEdits[user.Nick]++
				}
			}

			if external.Service == "anilist/anime" {
				if external.CreatedBy == "" {
					anilistEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anilistEdits[user.Nick]++
				}
			}

			if external.Service == "myanimelist/anime" {
				if external.CreatedBy == "" {
					malEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					malEdits[user.Nick]++
				}
			}

			if external.Service == "anidb/anime" {
				if external.CreatedBy == "" {
					anidbEdits["(auto-generated)"]++
				} else {
					user, err := arn.GetUser(external.CreatedBy)
					arn.PanicOnError(err)
					anidbEdits[user.Nick]++
				}
			}
		}

		if anime.GetMapping("shoboi/anime") != "" {
			shoboi["Connected with Shoboi"]++
		} else {
			shoboi["Not connected with Shoboi"]++
		}

		if anime.GetMapping("anilist/anime") != "" {
			anilist["Connected with AniList"]++
		} else {
			anilist["Not connected with AniList"]++
		}

		if anime.GetMapping("myanimelist/anime") != "" {
			mal["Connected with MyAnimeList"]++
		} else {
			mal["Not connected with MyAnimeList"]++
		}

		if anime.GetMapping("anidb/anime") != "" {
			anidb["Connected with AniDB"]++
		} else {
			anidb["Not connected with AniDB"]++
		}

		rating[arn.ToString(int(anime.Rating.Overall+0.5))]++

		found := false
		for _, episode := range anime.Episodes().Items {
			if episode.Links != nil && episode.Links["twist.moe"] != "" {
				found = true
				break
			}
		}

		if found {
			twist["Connected with AnimeTwist"]++
		} else {
			twist["Not connected with AnimeTwist"]++
		}

		status[anime.Status]++
		types[anime.Type]++
	}

	println("Finished anime statistics")

	return []*arn.PieChart{
		arn.NewPieChart("Type", types),
		arn.NewPieChart("Status", status),
		arn.NewPieChart("Rating", rating),
		arn.NewPieChart("MyAnimeList", mal),
		arn.NewPieChart("AniList", anilist),
		arn.NewPieChart("AniDB", anidb),
		arn.NewPieChart("Shoboi", shoboi),
		arn.NewPieChart("AnimeTwist", twist),
		// arn.NewPieChart("MyAnimeList Editors", malEdits),
		arn.NewPieChart("AniList Editors", anilistEdits),
		// arn.NewPieChart("AniDB Editors", anidbEdits),
		arn.NewPieChart("Shoboi Editors", shoboiEdits),
	}
}
<|endoftext|>