<commit_before>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"net/rpc"
	"net/rpc/jsonrpc"
	"strconv"
	"strings"
	"time"

	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/block"
	"github.com/bitmark-inc/bitmarkd/blockheader"
	"github.com/bitmark-inc/bitmarkd/difficulty"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/peer"
	"github.com/bitmark-inc/bitmarkd/reservoir"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
)

// defaults
const (
	defaultCount = 10
	maximumCount = 100
)

// InternalConnection - type to allow rpc system to interface to http request
type InternalConnection struct {
	in  io.Reader
	out io.Writer
}

func (c *InternalConnection) Read(p []byte) (n int, err error) {
	return c.in.Read(p)
}
func (c *InternalConnection) Write(d []byte) (n int, err error) {
	return c.out.Write(d)
}
func (c *InternalConnection) Close() error {
	return nil
}

// the argument passed to the handlers
type httpHandler struct {
	log     *logger.L
	server  *rpc.Server
	start   time.Time
	version string
	allow   map[string][]*net.IPNet
}

// this matches anything not matched and returns error
func (s *httpHandler) root(w http.ResponseWriter, r *http.Request) {
	sendNotFound(w)
}

// performs a call to any normal RPC
func (s *httpHandler) rpc(w http.ResponseWriter, r *http.Request) {
	if http.MethodPost != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	server := s.server

	connectionCount.Increment()
	defer connectionCount.Decrement()

	serverCodec := jsonrpc.NewServerCodec(&InternalConnection{in: r.Body, out: w})
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(http.StatusOK)
	err := server.ServeRequest(serverCodec)
	if nil != err {
		sendInternalServerError(w)
		return
	}
}

// check if remote address is allowed
func (s *httpHandler) isAllowed(api string, r *http.Request) bool {
	last := strings.LastIndex(r.RemoteAddr, ":")
	if last <= 0 {
		return false
	}

	cidr, ok := s.allow[api]
	if !ok {
		return false
	}

	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if nil != err {
		return false
	}

	addr := net.ParseIP(host)
	if nil == addr {
		return false
	}

	for _, n := range cidr {
		if n.Contains(addr) {
			return true
		}
	}

	return false
}

// to allow a GET for the same response and Node.Info RPC
func (s *httpHandler) details(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("details", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	type lrCount struct {
		Local  uint64 `json:"local"`
		Remote uint64 `json:"remote"`
	}

	type blockInfo struct {
		LRCount lrCount `json:"count"`
		Hash    string  `json:"hash"`
	}

	type peerCounts struct {
		Incoming uint64 `json:"incoming"`
		Outgoing uint64 `json:"outgoing"`
	}

	type theReply struct {
		Chain               string     `json:"chain"`
		Mode                string     `json:"mode"`
		Block               blockInfo  `json:"block"`
		RPCs                uint64     `json:"rpcs"`
		Peers               peerCounts `json:"peers"`
		TransactionCounters Counters   `json:"transactionCounters"`
		Difficulty          float64    `json:"difficulty"`
		Hashrate            float64    `json:hashrate,omitempty`
		Version             string     `json:"version"`
		Uptime              string     `json:"uptime"`
		PublicKey           string     `json:"publicKey"`
	}

	reply := theReply{
		Chain: mode.ChainName(),
		Mode:  mode.String(),
		Block: blockInfo{
			LRCount: lrCount{
				Local:  blockheader.Height(),
				Remote: peer.BlockHeight(),
			},
			Hash: block.LastBlockHash(),
		},
		RPCs: connectionCount.Uint64(),
		// Miners : mine.ConnectionCount(),
		Difficulty: difficulty.Current.Value(),
		Hashrate:   difficulty.Hashrate(),
		Version:    s.version,
		Uptime:     time.Since(s.start).String(),
		PublicKey:  hex.EncodeToString(peer.PublicKey()),
	}

	reply.Peers.Incoming, reply.Peers.Outgoing = peer.GetCounts()
	reply.TransactionCounters.Pending, reply.TransactionCounters.Verified = reservoir.ReadCounters()

	sendReply(w, reply)
}

func (s *httpHandler) connections(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("connections", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	type reply struct {
		ConnectedTo []*zmqutil.Connected `json:"connectedTo"`
	}

	var info reply

	info.ConnectedTo = peer.FetchConnectors()

	sendReply(w, info)
}

// to output peer data
type entry struct {
	PublicKey string    `json:"publicKey"`
	Listeners []string  `json:"listeners"`
	Timestamp time.Time `json:"timestamp"`
}

// GET to find data on all peers seen in the announcer
// (restricted to local_allow)
//
// query parameters:
//   public_key=<64-hex-characters>   [32 byte public key in hex]
//   count=<int>                      [1..100  default: 10]
func (s *httpHandler) peers(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("peers", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	r.ParseForm()

	// public_key parsing
	startkey := []byte{}
	k, err := hex.DecodeString(r.Form.Get("public_key"))
	if nil == err && 32 == len(k) {
		startkey = k
	}

	// count parsing
	count := defaultCount
	n, err := strconv.Atoi(r.Form.Get("count"))
	if nil == err && n >= 1 && n <= maximumCount {
		count = n
	}

	peers := make([]entry, 0, count)

item_loop:
	for i := 0; i < count; i += 1 {
		publicKey, listeners, timestamp, err := announce.GetNext(startkey)
		if nil != err {
			sendInternalServerError(w)
			return
		}
		if bytes.Compare(publicKey, startkey) <= 0 {
			break item_loop
		}
		startkey = publicKey

		p := hex.EncodeToString(publicKey)

		lPack := util.PackedConnection(listeners)
		lc := make([]string, 0, 2)
	lc_loop:
		for {
			conn, n := lPack.Unpack()
			if nil == conn {
				break lc_loop
			}
			lc = append(lc, conn.String())
			lPack = lPack[n:]
		}

		peers = append(peers, entry{
			PublicKey: p,
			Listeners: lc,
			Timestamp: timestamp,
		})
	}

	sendReply(w, peers)
}

// send an JSON encoded reply
func sendReply(w http.ResponseWriter, data interface{}) {
	text, err := json.Marshal(data)
	if nil != err {
		sendInternalServerError(w)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(http.StatusOK)
	w.Write(text)
}

// selected errors as required above
func sendNotFound(w http.ResponseWriter) {
	sendError(w, "not found", http.StatusNotFound)
}
func sendMethodNotAllowed(w http.ResponseWriter) {
	sendError(w, "method not allowed", http.StatusMethodNotAllowed)
}
func sendForbidden(w http.ResponseWriter) {
	sendError(w, "forbidden", http.StatusForbidden)
}
func sendInternalServerError(w http.ResponseWriter) {
	sendError(w, "internal server error", http.StatusInternalServerError)
}

// to compose JSON error messages
type eType struct {
	Code  int    `json:"code"`
	Error string `json:"error"`
}

// output an error with a JSON body
func sendError(w http.ResponseWriter, message string, code int) {
	text, err := json.Marshal(eType{
		Code:  code,
		Error: message,
	})
	if nil != err {
		// manually composed error just incase JSON fails
		http.Error(w, `{"code":500,"error":"Internal Server Error"}`, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(code)
	w.Write(text)
}
<commit_msg>[rpc] quotes were missing on struct tag<commit_after>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"net/rpc"
	"net/rpc/jsonrpc"
	"strconv"
	"strings"
	"time"

	"github.com/bitmark-inc/bitmarkd/announce"
	"github.com/bitmark-inc/bitmarkd/block"
	"github.com/bitmark-inc/bitmarkd/blockheader"
	"github.com/bitmark-inc/bitmarkd/difficulty"
	"github.com/bitmark-inc/bitmarkd/mode"
	"github.com/bitmark-inc/bitmarkd/peer"
	"github.com/bitmark-inc/bitmarkd/reservoir"
	"github.com/bitmark-inc/bitmarkd/util"
	"github.com/bitmark-inc/bitmarkd/zmqutil"
	"github.com/bitmark-inc/logger"
)

// defaults
const (
	defaultCount = 10
	maximumCount = 100
)

// InternalConnection - type to allow rpc system to interface to http request
type InternalConnection struct {
	in  io.Reader
	out io.Writer
}

func (c *InternalConnection) Read(p []byte) (n int, err error) {
	return c.in.Read(p)
}
func (c *InternalConnection) Write(d []byte) (n int, err error) {
	return c.out.Write(d)
}
func (c *InternalConnection) Close() error {
	return nil
}

// the argument passed to the handlers
type httpHandler struct {
	log     *logger.L
	server  *rpc.Server
	start   time.Time
	version string
	allow   map[string][]*net.IPNet
}

// this matches anything not matched and returns error
func (s *httpHandler) root(w http.ResponseWriter, r *http.Request) {
	sendNotFound(w)
}

// performs a call to any normal RPC
func (s *httpHandler) rpc(w http.ResponseWriter, r *http.Request) {
	if http.MethodPost != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	server := s.server

	connectionCount.Increment()
	defer connectionCount.Decrement()

	serverCodec := jsonrpc.NewServerCodec(&InternalConnection{in: r.Body, out: w})
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(http.StatusOK)
	err := server.ServeRequest(serverCodec)
	if nil != err {
		sendInternalServerError(w)
		return
	}
}

// check if remote address is allowed
func (s *httpHandler) isAllowed(api string, r *http.Request) bool {
	last := strings.LastIndex(r.RemoteAddr, ":")
	if last <= 0 {
		return false
	}

	cidr, ok := s.allow[api]
	if !ok {
		return false
	}

	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if nil != err {
		return false
	}

	addr := net.ParseIP(host)
	if nil == addr {
		return false
	}

	for _, n := range cidr {
		if n.Contains(addr) {
			return true
		}
	}

	return false
}

// to allow a GET for the same response and Node.Info RPC
func (s *httpHandler) details(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("details", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	type lrCount struct {
		Local  uint64 `json:"local"`
		Remote uint64 `json:"remote"`
	}

	type blockInfo struct {
		LRCount lrCount `json:"count"`
		Hash    string  `json:"hash"`
	}

	type peerCounts struct {
		Incoming uint64 `json:"incoming"`
		Outgoing uint64 `json:"outgoing"`
	}

	type theReply struct {
		Chain               string     `json:"chain"`
		Mode                string     `json:"mode"`
		Block               blockInfo  `json:"block"`
		RPCs                uint64     `json:"rpcs"`
		Peers               peerCounts `json:"peers"`
		TransactionCounters Counters   `json:"transactionCounters"`
		Difficulty          float64    `json:"difficulty"`
		Hashrate            float64    `json:"hashrate,omitempty"`
		Version             string     `json:"version"`
		Uptime              string     `json:"uptime"`
		PublicKey           string     `json:"publicKey"`
	}

	reply := theReply{
		Chain: mode.ChainName(),
		Mode:  mode.String(),
		Block: blockInfo{
			LRCount: lrCount{
				Local:  blockheader.Height(),
				Remote: peer.BlockHeight(),
			},
			Hash: block.LastBlockHash(),
		},
		RPCs: connectionCount.Uint64(),
		// Miners : mine.ConnectionCount(),
		Difficulty: difficulty.Current.Value(),
		Hashrate:   difficulty.Hashrate(),
		Version:    s.version,
		Uptime:     time.Since(s.start).String(),
		PublicKey:  hex.EncodeToString(peer.PublicKey()),
	}

	reply.Peers.Incoming, reply.Peers.Outgoing = peer.GetCounts()
	reply.TransactionCounters.Pending, reply.TransactionCounters.Verified = reservoir.ReadCounters()

	sendReply(w, reply)
}

func (s *httpHandler) connections(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("connections", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	type reply struct {
		ConnectedTo []*zmqutil.Connected `json:"connectedTo"`
	}

	var info reply

	info.ConnectedTo = peer.FetchConnectors()

	sendReply(w, info)
}

// to output peer data
type entry struct {
	PublicKey string    `json:"publicKey"`
	Listeners []string  `json:"listeners"`
	Timestamp time.Time `json:"timestamp"`
}

// GET to find data on all peers seen in the announcer
// (restricted to local_allow)
//
// query parameters:
//   public_key=<64-hex-characters>   [32 byte public key in hex]
//   count=<int>                      [1..100  default: 10]
func (s *httpHandler) peers(w http.ResponseWriter, r *http.Request) {
	if http.MethodGet != r.Method {
		sendMethodNotAllowed(w)
		return
	}

	if !s.isAllowed("peers", r) {
		s.log.Warnf("Deny access: %q", r.RemoteAddr)
		sendForbidden(w)
		return
	}

	connectionCount.Increment()
	defer connectionCount.Decrement()

	r.ParseForm()

	// public_key parsing
	startkey := []byte{}
	k, err := hex.DecodeString(r.Form.Get("public_key"))
	if nil == err && 32 == len(k) {
		startkey = k
	}

	// count parsing
	count := defaultCount
	n, err := strconv.Atoi(r.Form.Get("count"))
	if nil == err && n >= 1 && n <= maximumCount {
		count = n
	}

	peers := make([]entry, 0, count)

item_loop:
	for i := 0; i < count; i += 1 {
		publicKey, listeners, timestamp, err := announce.GetNext(startkey)
		if nil != err {
			sendInternalServerError(w)
			return
		}
		if bytes.Compare(publicKey, startkey) <= 0 {
			break item_loop
		}
		startkey = publicKey

		p := hex.EncodeToString(publicKey)

		lPack := util.PackedConnection(listeners)
		lc := make([]string, 0, 2)
	lc_loop:
		for {
			conn, n := lPack.Unpack()
			if nil == conn {
				break lc_loop
			}
			lc = append(lc, conn.String())
			lPack = lPack[n:]
		}

		peers = append(peers, entry{
			PublicKey: p,
			Listeners: lc,
			Timestamp: timestamp,
		})
	}

	sendReply(w, peers)
}

// send an JSON encoded reply
func sendReply(w http.ResponseWriter, data interface{}) {
	text, err := json.Marshal(data)
	if nil != err {
		sendInternalServerError(w)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(http.StatusOK)
	w.Write(text)
}

// selected errors as required above
func sendNotFound(w http.ResponseWriter) {
	sendError(w, "not found", http.StatusNotFound)
}
func sendMethodNotAllowed(w http.ResponseWriter) {
	sendError(w, "method not allowed", http.StatusMethodNotAllowed)
}
func sendForbidden(w http.ResponseWriter) {
	sendError(w, "forbidden", http.StatusForbidden)
}
func sendInternalServerError(w http.ResponseWriter) {
	sendError(w, "internal server error", http.StatusInternalServerError)
}

// to compose JSON error messages
type eType struct {
	Code  int    `json:"code"`
	Error string `json:"error"`
}

// output an error with a JSON body
func sendError(w http.ResponseWriter, message string, code int) {
	text, err := json.Marshal(eType{
		Code:  code,
		Error: message,
	})
	if nil != err {
		// manually composed error just incase JSON fails
		http.Error(w, `{"code":500,"error":"Internal Server Error"}`, http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(code)
	w.Write(text)
}
<|endoftext|>


<commit_before>package rrdtool

import (
	"errors"
	"log"
	"math"
	"sync/atomic"
	"time"

	cmodel "github.com/open-falcon/common/model"
	"github.com/open-falcon/rrdlite"
	"github.com/toolkits/file"

	"github.com/open-falcon/graph/g"
	"github.com/open-falcon/graph/store"
)

var (
	disk_counter uint64
	net_counter  uint64
)

type fetch_t struct {
	filename string
	cf       string
	start    int64
	end      int64
	step     int
	data     []*cmodel.RRDData
}

type flushfile_t struct {
	filename string
	items    []*cmodel.GraphItem
}

type readfile_t struct {
	filename string
	data     []byte
}

func Start() {
	cfg := g.Config()
	var err error
	// check data dir
	if err = file.EnsureDirRW(cfg.RRD.Storage); err != nil {
		log.Fatalln("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err)
	}

	migrate_start(cfg)

	// sync disk
	go syncDisk()
	go ioWorker()
	log.Println("rrdtool.Start ok")
}

// RRA.Point.Size
const (
	RRA1PointCnt   = 720 // 1m一个点存12h
	RRA5PointCnt   = 576 // 5m一个点存2d
	RRA20PointCnt  = 504 // 20m一个点存7d
	RRA180PointCnt = 766 // 3h一个点存3month
	RRA720PointCnt = 730 // 12h一个点存1year
)

func create(filename string, item *cmodel.GraphItem) error {
	now := time.Now()
	start := now.Add(time.Duration(-24) * time.Hour)
	step := uint(item.Step)

	c := rrdlite.NewCreator(filename, start, step)
	c.DS("metric", item.DsType, item.Heartbeat, item.Min, item.Max)

	// 设置各种归档策略
	// 1分钟一个点存 12小时
	c.RRA("AVERAGE", 0.5, 1, RRA1PointCnt)

	// 5m一个点存2d
	c.RRA("AVERAGE", 0.5, 5, RRA5PointCnt)
	c.RRA("MAX", 0.5, 5, RRA5PointCnt)
	c.RRA("MIN", 0.5, 5, RRA5PointCnt)

	// 20m一个点存7d
	c.RRA("AVERAGE", 0.5, 20, RRA20PointCnt)
	c.RRA("MAX", 0.5, 20, RRA20PointCnt)
	c.RRA("MIN", 0.5, 20, RRA20PointCnt)

	// 3小时一个点存3个月
	c.RRA("AVERAGE", 0.5, 180, RRA180PointCnt)
	c.RRA("MAX", 0.5, 180, RRA180PointCnt)
	c.RRA("MIN", 0.5, 180, RRA180PointCnt)

	// 12小时一个点存1year
	c.RRA("AVERAGE", 0.5, 720, RRA720PointCnt)
	c.RRA("MAX", 0.5, 720, RRA720PointCnt)
	c.RRA("MIN", 0.5, 720, RRA720PointCnt)

	return c.Create(true)
}

func update(filename string, items []*cmodel.GraphItem) error {
	u := rrdlite.NewUpdater(filename)

	for _, item := range items {
		v := math.Abs(item.Value)
		if v > 1e+300 || (v < 1e-300 && v > 0) {
			continue
		}
		if item.DsType == "DERIVE" || item.DsType == "COUNTER" {
			u.Cache(item.Timestamp, int(item.Value))
		} else {
			u.Cache(item.Timestamp, item.Value)
		}
	}

	return u.Update()
}

// flush to disk from memory
// 最新的数据在列表的最后面
// TODO fix me, filename fmt from item[0], it's hard to keep consistent
func flushrrd(filename string, items []*cmodel.GraphItem) error {
	if items == nil || len(items) == 0 {
		return errors.New("empty items")
	}

	if !g.IsRrdFileExist(filename) {
		baseDir := file.Dir(filename)

		err := file.InsureDir(baseDir)
		if err != nil {
			return err
		}

		err = create(filename, items[0])
		if err != nil {
			return err
		}
	}

	return update(filename, items)
}

func ReadFile(filename string) ([]byte, error) {
	done := make(chan error, 1)
	task := &io_task_t{
		method: IO_TASK_M_READ,
		args:   &readfile_t{filename: filename},
		done:   done,
	}

	io_task_chan <- task
	err := <-done
	return task.args.(*readfile_t).data, err
}

func FlushFile(filename string, items []*cmodel.GraphItem) error {
	done := make(chan error, 1)
	io_task_chan <- &io_task_t{
		method: IO_TASK_M_FLUSH,
		args: &flushfile_t{
			filename: filename,
			items:    items,
		},
		done: done,
	}
	atomic.AddUint64(&disk_counter, 1)
	return <-done
}

func Fetch(filename string, cf string, start, end int64, step int) ([]*cmodel.RRDData, error) {
	done := make(chan error, 1)
	task := &io_task_t{
		method: IO_TASK_M_FETCH,
		args: &fetch_t{
			filename: filename,
			cf:       cf,
			start:    start,
			end:      end,
			step:     step,
		},
		done: done,
	}
	io_task_chan <- task
	err := <-done
	return task.args.(*fetch_t).data, err
}

func fetch(filename string, cf string, start, end int64, step int) ([]*cmodel.RRDData, error) {
	start_t := time.Unix(start, 0)
	end_t := time.Unix(end, 0)
	step_t := time.Duration(step) * time.Second

	fetchRes, err := rrdlite.Fetch(filename, cf, start_t, end_t, step_t)
	if err != nil {
		return []*cmodel.RRDData{}, err
	}

	defer fetchRes.FreeValues()

	values := fetchRes.Values()
	size := len(values)
	ret := make([]*cmodel.RRDData, size)

	start_ts := fetchRes.Start.Unix()
	step_s := fetchRes.Step.Seconds()

	for i, val := range values {
		ts := start_ts + int64(i+1)*int64(step_s)
		d := &cmodel.RRDData{
			Timestamp: ts,
			Value:     cmodel.JsonFloat(val),
		}
		ret[i] = d
	}

	return ret, nil
}

func FlushAll(force bool) {
	n := store.GraphItems.Size / 10
	for i := 0; i < store.GraphItems.Size; i++ {
		FlushRRD(i, force)
		if i%n == 0 {
			log.Printf("flush hash idx:%03d size:03d disk:%08d disk:%08ld net:%08ld\n",
				i, store.GraphItems.Size, disk_counter, net_counter)
		}
	}
	log.Printf("flush hash done (disk:%08ld net:%08ld)\n", disk_counter, net_counter)
}

func CommitByKey(key string) {

	md5, dsType, step, err := g.SplitRrdCacheKey(key)
	if err != nil {
		return
	}
	filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)

	items := store.GraphItems.PopAll(key)
	if len(items) == 0 {
		return
	}
	FlushFile(filename, items)
}

func PullByKey(key string) {
	done := make(chan error)

	item := store.GraphItems.First(key)
	if item == nil {
		return
	}
	node, err := Consistent.Get(item.PrimaryKey())
	if err != nil {
		return
	}
	Net_task_ch[node] <- &Net_task_t{
		Method: NET_TASK_M_PULL,
		Key:    key,
		Done:   done,
	}
	// net_task slow, shouldn't block syncDisk() or FlushAll()
	// warning: recev sigout when migrating, maybe lost memory data
	go func() {
		err := <-done
		if err != nil {
			log.Printf("get %s from remote err[%s]\n", key, err)
			return
		}
		atomic.AddUint64(&net_counter, 1)
		//todo: flushfile after getfile? not yet
	}()
}

func FlushRRD(idx int, force bool) {
	begin := time.Now()
	atomic.StoreInt32(&flushrrd_timeout, 0)

	keys := store.GraphItems.KeysByIndex(idx)
	if len(keys) == 0 {
		return
	}

	for _, key := range keys {
		flag, _ := store.GraphItems.GetFlag(key)

		//write err data to local filename
		if force == false && g.Config().Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
			if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP {
				atomic.StoreInt32(&flushrrd_timeout, 1)
			}
			PullByKey(key)
		} else {
			CommitByKey(key)
		}
	}
}
<commit_msg>fixbug print format<commit_after>package rrdtool

import (
	"errors"
	"log"
	"math"
	"sync/atomic"
	"time"

	cmodel "github.com/open-falcon/common/model"
	"github.com/open-falcon/rrdlite"
	"github.com/toolkits/file"

	"github.com/open-falcon/graph/g"
	"github.com/open-falcon/graph/store"
)

var (
	disk_counter uint64
	net_counter  uint64
)

type fetch_t struct {
	filename string
	cf       string
	start    int64
	end      int64
	step     int
	data     []*cmodel.RRDData
}

type flushfile_t struct {
	filename string
	items    []*cmodel.GraphItem
}

type readfile_t struct {
	filename string
	data     []byte
}

func Start() {
	cfg := g.Config()
	var err error
	// check data dir
	if err = file.EnsureDirRW(cfg.RRD.Storage); err != nil {
		log.Fatalln("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err)
	}

	migrate_start(cfg)

	// sync disk
	go syncDisk()
	go ioWorker()
	log.Println("rrdtool.Start ok")
}

// RRA.Point.Size
const (
	RRA1PointCnt   = 720 // 1m一个点存12h
	RRA5PointCnt   = 576 // 5m一个点存2d
	RRA20PointCnt  = 504 // 20m一个点存7d
	RRA180PointCnt = 766 // 3h一个点存3month
	RRA720PointCnt = 730 // 12h一个点存1year
)

func create(filename string, item *cmodel.GraphItem) error {
	now := time.Now()
	start := now.Add(time.Duration(-24) * time.Hour)
	step := uint(item.Step)

	c := rrdlite.NewCreator(filename, start, step)
	c.DS("metric", item.DsType, item.Heartbeat, item.Min, item.Max)

	// 设置各种归档策略
	// 1分钟一个点存 12小时
	c.RRA("AVERAGE", 0.5, 1, RRA1PointCnt)

	// 5m一个点存2d
	c.RRA("AVERAGE", 0.5, 5, RRA5PointCnt)
	c.RRA("MAX", 0.5, 5, RRA5PointCnt)
	c.RRA("MIN", 0.5, 5, RRA5PointCnt)

	// 20m一个点存7d
	c.RRA("AVERAGE", 0.5, 20, RRA20PointCnt)
	c.RRA("MAX", 0.5, 20, RRA20PointCnt)
	c.RRA("MIN", 0.5, 20, RRA20PointCnt)

	// 3小时一个点存3个月
	c.RRA("AVERAGE", 0.5, 180, RRA180PointCnt)
	c.RRA("MAX", 0.5, 180, RRA180PointCnt)
	c.RRA("MIN", 0.5, 180, RRA180PointCnt)

	// 12小时一个点存1year
	c.RRA("AVERAGE", 0.5, 720, RRA720PointCnt)
	c.RRA("MAX", 0.5, 720, RRA720PointCnt)
	c.RRA("MIN", 0.5, 720, RRA720PointCnt)

	return c.Create(true)
}

func update(filename string, items []*cmodel.GraphItem) error {
	u := rrdlite.NewUpdater(filename)

	for _, item := range items {
		v := math.Abs(item.Value)
		if v > 1e+300 || (v < 1e-300 && v > 0) {
			continue
		}
		if item.DsType == "DERIVE" || item.DsType == "COUNTER" {
			u.Cache(item.Timestamp, int(item.Value))
		} else {
			u.Cache(item.Timestamp, item.Value)
		}
	}

	return u.Update()
}

// flush to disk from memory
// 最新的数据在列表的最后面
// TODO fix me, filename fmt from item[0], it's hard to keep consistent
func flushrrd(filename string, items []*cmodel.GraphItem) error {
	if items == nil || len(items) == 0 {
		return errors.New("empty items")
	}

	if !g.IsRrdFileExist(filename) {
		baseDir := file.Dir(filename)

		err := file.InsureDir(baseDir)
		if err != nil {
			return err
		}

		err = create(filename, items[0])
		if err != nil {
			return err
		}
	}

	return update(filename, items)
}

func ReadFile(filename string) ([]byte, error) {
	done := make(chan error, 1)
	task := &io_task_t{
		method: IO_TASK_M_READ,
		args:   &readfile_t{filename: filename},
		done:   done,
	}

	io_task_chan <- task
	err := <-done
	return task.args.(*readfile_t).data, err
}

func FlushFile(filename string, items []*cmodel.GraphItem) error {
	done := make(chan error, 1)
	io_task_chan <- &io_task_t{
		method: IO_TASK_M_FLUSH,
		args: &flushfile_t{
			filename: filename,
			items:    items,
		},
		done: done,
	}
	atomic.AddUint64(&disk_counter, 1)
	return <-done
}

func Fetch(filename string, cf string, start, end int64, step int) ([]*cmodel.RRDData, error) {
	done := make(chan error, 1)
	task := &io_task_t{
		method: IO_TASK_M_FETCH,
		args: &fetch_t{
			filename: filename,
			cf:       cf,
			start:    start,
			end:      end,
			step:     step,
		},
		done: done,
	}
	io_task_chan <- task
	err := <-done
	return task.args.(*fetch_t).data, err
}

func fetch(filename string, cf string, start, end int64, step int) ([]*cmodel.RRDData, error) {
	start_t := time.Unix(start, 0)
	end_t := time.Unix(end, 0)
	step_t := time.Duration(step) * time.Second

	fetchRes, err := rrdlite.Fetch(filename, cf, start_t, end_t, step_t)
	if err != nil {
		return []*cmodel.RRDData{}, err
	}

	defer fetchRes.FreeValues()

	values := fetchRes.Values()
	size := len(values)
	ret := make([]*cmodel.RRDData, size)

	start_ts := fetchRes.Start.Unix()
	step_s := fetchRes.Step.Seconds()

	for i, val := range values {
		ts := start_ts + int64(i+1)*int64(step_s)
		d := &cmodel.RRDData{
			Timestamp: ts,
			Value:     cmodel.JsonFloat(val),
		}
		ret[i] = d
	}

	return ret, nil
}

func FlushAll(force bool) {
	n := store.GraphItems.Size / 10
	for i := 0; i < store.GraphItems.Size; i++ {
		FlushRRD(i, force)
		if i%n == 0 {
			log.Printf("flush hash idx:%03d size:%03d disk:%08d net:%08d\n",
				i, store.GraphItems.Size, disk_counter, net_counter)
		}
	}
	log.Printf("flush hash done (disk:%08ld net:%08ld)\n", disk_counter, net_counter)
}

func CommitByKey(key string) {

	md5, dsType, step, err := g.SplitRrdCacheKey(key)
	if err != nil {
		return
	}
	filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)

	items := store.GraphItems.PopAll(key)
	if len(items) == 0 {
		return
	}
	FlushFile(filename, items)
}

func PullByKey(key string) {
	done := make(chan error)

	item := store.GraphItems.First(key)
	if item == nil {
		return
	}
	node, err := Consistent.Get(item.PrimaryKey())
	if err != nil {
		return
	}
	Net_task_ch[node] <- &Net_task_t{
		Method: NET_TASK_M_PULL,
		Key:    key,
		Done:   done,
	}
	// net_task slow, shouldn't block syncDisk() or FlushAll()
	// warning: recev sigout when migrating, maybe lost memory data
	go func() {
		err := <-done
		if err != nil {
			log.Printf("get %s from remote err[%s]\n", key, err)
			return
		}
		atomic.AddUint64(&net_counter, 1)
		//todo: flushfile after getfile? not yet
	}()
}

func FlushRRD(idx int, force bool) {
	begin := time.Now()
	atomic.StoreInt32(&flushrrd_timeout, 0)

	keys := store.GraphItems.KeysByIndex(idx)
	if len(keys) == 0 {
		return
	}

	for _, key := range keys {
		flag, _ := store.GraphItems.GetFlag(key)

		//write err data to local filename
		if force == false && g.Config().Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
			if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP {
				atomic.StoreInt32(&flushrrd_timeout, 1)
			}
			PullByKey(key)
		} else {
			CommitByKey(key)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type ConfigGetter func() string
type RunModeGetter func() RunMode

type Base struct {
	appName    string
	getHome    ConfigGetter
	getRunMode RunModeGetter
	getLog     LogGetter
}

type HomeFinder interface {
	CacheDir() string
	ConfigDir() string
	Home(emptyOk bool) string
	DataDir() string
	RuntimeDir() string
	Normalize(s string) string
	LogDir() string
	ServiceSpawnDir() (string, error)
	SandboxCacheDir() string // For macOS
	InfoDir() string
}

func (b Base) Unsplit(v []string) string {
	if len(v) > 0 && len(v[0]) == 0 {
		v2 := make([]string, len(v))
		copy(v2, v)
		v[0] = string(filepath.Separator)
	}
	return filepath.Join(v...)
}

func (b Base) Join(elem ...string) string { return filepath.Join(elem...) }

type XdgPosix struct {
	Base
}

func (x XdgPosix) Normalize(s string) string { return s }

func (x XdgPosix) Home(emptyOk bool) string {
	var ret string
	if x.getHome != nil {
		ret = x.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret = os.Getenv("HOME")
	}
	return ret
}

func (x XdgPosix) dirHelper(env string, prefixDirs ...string) string {
	var prfx string
	prfx = os.Getenv(env)
	if len(prfx) == 0 {
		h := x.Home(false)
		v := append([]string{h}, prefixDirs...)
		prfx = x.Join(v...)
	}
	appName := x.appName
	if x.getRunMode() != ProductionRunMode {
		appName = appName + "." + string(x.getRunMode())
	}
	return x.Join(prfx, appName)
}

func (x XdgPosix) ConfigDir() string       { return x.dirHelper("XDG_CONFIG_HOME", ".config") }
func (x XdgPosix) CacheDir() string        { return x.dirHelper("XDG_CACHE_HOME", ".cache") }
func (x XdgPosix) SandboxCacheDir() string { return "" } // Unsupported
func (x XdgPosix) DataDir() string         { return x.dirHelper("XDG_DATA_HOME", ".local", "share") }

func (x XdgPosix) RuntimeDir() string { return x.dirHelper("XDG_RUNTIME_DIR", ".config") }
func (x XdgPosix) InfoDir() string    { return x.RuntimeDir() }

func (x XdgPosix) ServiceSpawnDir() (ret string, err error) {
	ret = x.RuntimeDir()
	if len(ret) == 0 {
		ret, err = ioutil.TempDir("", "keybase_service")
	}
	return
}

func (x XdgPosix) LogDir() string {
	// There doesn't seem to be an official place for logs in the XDG spec, but
	// according to http://stackoverflow.com/a/27965014/823869 at least, this
	// is the best compromise.
	return x.CacheDir()
}

type Darwin struct {
	Base
}

func toUpper(s string) string {
	if s == "" {
		return s
	}
	a := []rune(s)
	a[0] = unicode.ToUpper(a[0])
	return string(a)
}

func (d Darwin) appDir(dirs ...string) string {
	appName := toUpper(d.appName)
	runMode := d.getRunMode()
	if runMode != ProductionRunMode {
		appName = appName + toUpper(string(runMode))
	}
	dirs = append(dirs, appName)
	return filepath.Join(dirs...)
}

func (d Darwin) CacheDir() string { return d.appDir(d.Home(false), "Library", "Caches") }
func (d Darwin) SandboxCacheDir() string {
	if isIOS {
		return ""
	}
	// The container name "keybase" is the group name specified in the entitlement for sandboxed extensions
	// Note: this was added for kbfs finder integration, which was never activated.
	// keybased.sock and kbfsd.sock live in this directory.
	return d.appDir(d.Home(false), "Library", "Group Containers", "keybase", "Library", "Caches")
}
func (d Darwin) ConfigDir() string                { return d.appDir(d.Home(false), "Library", "Application Support") }
func (d Darwin) DataDir() string                  { return d.ConfigDir() }
func (d Darwin) RuntimeDir() string               { return d.CacheDir() }
func (d Darwin) InfoDir() string                  { return d.appDir(os.TempDir()) }
func (d Darwin) ServiceSpawnDir() (string, error) { return d.RuntimeDir(), nil }
func (d Darwin) LogDir() string {
	appName := toUpper(d.appName)
	runMode := d.getRunMode()
	dirs := []string{d.Home(false), "Library", "Logs"}
	if runMode != ProductionRunMode {
		dirs = append(dirs, appName+toUpper(string(runMode)))
	}
	return filepath.Join(dirs...)
}

func (d Darwin) Home(emptyOk bool) string {
	var ret string
	if d.getHome != nil {
		ret = d.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret = os.Getenv("HOME")
	}
	return ret
}

func (d Darwin) Normalize(s string) string { return s }

type Win32 struct {
	Base
}

var win32SplitRE = regexp.MustCompile(`[/\\]`)

func (w Win32) Split(s string) []string {
	return win32SplitRE.Split(s, -1)
}

func (w Win32) Normalize(s string) string {
	return w.Unsplit(w.Split(s))
}

func (w Win32) CacheDir() string                 { return w.Home(false) }
func (w Win32) SandboxCacheDir() string          { return "" } // Unsupported
func (w Win32) ConfigDir() string                { return w.Home(false) }
func (w Win32) DataDir() string                  { return w.Home(false) }
func (w Win32) RuntimeDir() string               { return w.Home(false) }
func (w Win32) InfoDir() string                  { return w.RuntimeDir() }
func (w Win32) ServiceSpawnDir() (string, error) { return w.RuntimeDir(), nil }
func (w Win32) LogDir() string                   { return w.Home(false) }

func (w Win32) Home(emptyOk bool) string {
	var ret string

	if w.getHome != nil {
		ret = w.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret, _ = LocalDataDir()
		if len(ret) == 0 {
			w.getLog().Info("APPDATA environment variable not found")
		}

	}
	if len(ret) == 0 && !emptyOk {
		tmp := os.Getenv("TEMP")
		if len(tmp) == 0 {
			w.getLog().Info("No 'TEMP' environment variable found")
			tmp = os.Getenv("TMP")
			if len(tmp) == 0 {
				w.getLog().Fatalf("No 'TMP' environment variable found")
			}
		}
		v := w.Split(tmp)
		if len(v) < 2 {
			w.getLog().Fatalf("Bad 'TEMP' variable found, no directory separators!")
		}
		last := strings.ToLower(v[len(v)-1])
		rest := v[0 : len(v)-1]
		if last != "temp" && last != "tmp" {
			w.getLog().Warning("TEMP directory didn't end in \\Temp: %s", last)
		}
		if strings.ToLower(rest[len(rest)-1]) == "local" {
			rest[len(rest)-1] = "Roaming"
		}
		ret = w.Unsplit(rest)
	}

	packageName := "Keybase"

	if w.getRunMode() == DevelRunMode || w.getRunMode() == StagingRunMode {
		runModeName := string(w.getRunMode())
		if runModeName != "" {
			// Capitalize the first letter
			r, n := utf8.DecodeRuneInString(runModeName)
			runModeName = string(unicode.ToUpper(r)) + runModeName[n:]
			packageName = packageName + runModeName
		}
	}

	ret = filepath.Join(ret, packageName)

	return ret
}

func NewHomeFinder(appName string, getHome ConfigGetter, osname string, getRunMode RunModeGetter, getLog LogGetter) HomeFinder {
	base := Base{appName, getHome, getRunMode, getLog}
	switch osname {
	case "windows":
		return Win32{base}
	case "darwin":
		return Darwin{base}
	default:
		return XdgPosix{base}
	}
}
<commit_msg>change infoDir behavior on darwin when --home is specified (#11252)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type ConfigGetter func() string
type RunModeGetter func() RunMode

type Base struct {
	appName    string
	getHome    ConfigGetter
	getRunMode RunModeGetter
	getLog     LogGetter
}

type HomeFinder interface {
	CacheDir() string
	ConfigDir() string
	Home(emptyOk bool) string
	DataDir() string
	RuntimeDir() string
	Normalize(s string) string
	LogDir() string
	ServiceSpawnDir() (string, error)
	SandboxCacheDir() string // For macOS
	InfoDir() string
}

func (b Base) Unsplit(v []string) string {
	if len(v) > 0 && len(v[0]) == 0 {
		v2 := make([]string, len(v))
		copy(v2, v)
		v[0] = string(filepath.Separator)
	}
	return filepath.Join(v...)
}

func (b Base) Join(elem ...string) string { return filepath.Join(elem...) }

type XdgPosix struct {
	Base
}

func (x XdgPosix) Normalize(s string) string { return s }

func (x XdgPosix) Home(emptyOk bool) string {
	var ret string
	if x.getHome != nil {
		ret = x.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret = os.Getenv("HOME")
	}
	return ret
}

func (x XdgPosix) dirHelper(env string, prefixDirs ...string) string {
	var prfx string
	prfx = os.Getenv(env)
	if len(prfx) == 0 {
		h := x.Home(false)
		v := append([]string{h}, prefixDirs...)
		prfx = x.Join(v...)
	}
	appName := x.appName
	if x.getRunMode() != ProductionRunMode {
		appName = appName + "." + string(x.getRunMode())
	}
	return x.Join(prfx, appName)
}

func (x XdgPosix) ConfigDir() string       { return x.dirHelper("XDG_CONFIG_HOME", ".config") }
func (x XdgPosix) CacheDir() string        { return x.dirHelper("XDG_CACHE_HOME", ".cache") }
func (x XdgPosix) SandboxCacheDir() string { return "" } // Unsupported
func (x XdgPosix) DataDir() string         { return x.dirHelper("XDG_DATA_HOME", ".local", "share") }

func (x XdgPosix) RuntimeDir() string { return x.dirHelper("XDG_RUNTIME_DIR", ".config") }
func (x XdgPosix) InfoDir() string    { return x.RuntimeDir() }

func (x XdgPosix) ServiceSpawnDir() (ret string, err error) {
	ret = x.RuntimeDir()
	if len(ret) == 0 {
		ret, err = ioutil.TempDir("", "keybase_service")
	}
	return
}

func (x XdgPosix) LogDir() string {
	// There doesn't seem to be an official place for logs in the XDG spec, but
	// according to http://stackoverflow.com/a/27965014/823869 at least, this
	// is the best compromise.
	return x.CacheDir()
}

type Darwin struct {
	Base
}

func toUpper(s string) string {
	if s == "" {
		return s
	}
	a := []rune(s)
	a[0] = unicode.ToUpper(a[0])
	return string(a)
}

func (d Darwin) appDir(dirs ...string) string {
	appName := toUpper(d.appName)
	runMode := d.getRunMode()
	if runMode != ProductionRunMode {
		appName = appName + toUpper(string(runMode))
	}
	dirs = append(dirs, appName)
	return filepath.Join(dirs...)
}

func (d Darwin) CacheDir() string { return d.appDir(d.Home(false), "Library", "Caches") }
func (d Darwin) SandboxCacheDir() string {
	if isIOS {
		return ""
	}
	// The container name "keybase" is the group name specified in the entitlement for sandboxed extensions
	// Note: this was added for kbfs finder integration, which was never activated.
	// keybased.sock and kbfsd.sock live in this directory.
	return d.appDir(d.Home(false), "Library", "Group Containers", "keybase", "Library", "Caches")
}
func (d Darwin) ConfigDir() string                { return d.appDir(d.Home(false), "Library", "Application Support") }
func (d Darwin) DataDir() string                  { return d.ConfigDir() }
func (d Darwin) RuntimeDir() string               { return d.CacheDir() }
func (d Darwin) ServiceSpawnDir() (string, error) { return d.RuntimeDir(), nil }
func (d Darwin) LogDir() string {
	appName := toUpper(d.appName)
	runMode := d.getRunMode()
	dirs := []string{d.Home(false), "Library", "Logs"}
	if runMode != ProductionRunMode {
		dirs = append(dirs, appName+toUpper(string(runMode)))
	}
	return filepath.Join(dirs...)
}

func (d Darwin) InfoDir() string {
	// If the user is explicitly passing in a HomeDirectory, make the PID file directory
	// local to that HomeDir. This way it's possible to have multiple keybases in parallel
	// running for a given run mode, without having to explicitly specify a PID file.
	if d.getHome != nil && d.getHome() != "" {
		return d.CacheDir()
	}
	return d.appDir(os.TempDir())
}

func (d Darwin) Home(emptyOk bool) string {
	var ret string
	if d.getHome != nil {
		ret = d.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret = os.Getenv("HOME")
	}
	return ret
}

func (d Darwin) Normalize(s string) string { return s }

type Win32 struct {
	Base
}

var win32SplitRE = regexp.MustCompile(`[/\\]`)

func (w Win32) Split(s string) []string {
	return win32SplitRE.Split(s, -1)
}

func (w Win32) Normalize(s string) string {
	return w.Unsplit(w.Split(s))
}

func (w Win32) CacheDir() string                 { return w.Home(false) }
func (w Win32) SandboxCacheDir() string          { return "" } // Unsupported
func (w Win32) ConfigDir() string                { return w.Home(false) }
func (w Win32) DataDir() string                  { return w.Home(false) }
func (w Win32) RuntimeDir() string               { return w.Home(false) }
func (w Win32) InfoDir() string                  { return w.RuntimeDir() }
func (w Win32) ServiceSpawnDir() (string, error) { return w.RuntimeDir(), nil }
func (w Win32) LogDir() string                   { return w.Home(false) }

func (w Win32) Home(emptyOk bool) string {
	var ret string

	if w.getHome != nil {
		ret = w.getHome()
	}
	if len(ret) == 0 && !emptyOk {
		ret, _ = LocalDataDir()
		if len(ret) == 0 {
			w.getLog().Info("APPDATA environment variable not found")
		}

	}
	if len(ret) == 0 && !emptyOk {
		tmp := os.Getenv("TEMP")
		if len(tmp) == 0 {
			w.getLog().Info("No 'TEMP' environment variable found")
			tmp = os.Getenv("TMP")
			if len(tmp) == 0 {
				w.getLog().Fatalf("No 'TMP' environment variable found")
			}
		}
		v := w.Split(tmp)
		if len(v) < 2 {
			w.getLog().Fatalf("Bad 'TEMP' variable found, no directory separators!")
		}
		last := strings.ToLower(v[len(v)-1])
		rest := v[0 : len(v)-1]
		if last != "temp" && last != "tmp" {
			w.getLog().Warning("TEMP directory didn't end in \\Temp: %s", last)
		}
		if strings.ToLower(rest[len(rest)-1]) == "local" {
			rest[len(rest)-1] = "Roaming"
		}
		ret = w.Unsplit(rest)
	}

	packageName := "Keybase"

	if w.getRunMode() == DevelRunMode || w.getRunMode() == StagingRunMode {
		runModeName := string(w.getRunMode())
		if runModeName != "" {
			// Capitalize the first letter
			r, n := utf8.DecodeRuneInString(runModeName)
			runModeName = string(unicode.ToUpper(r)) + runModeName[n:]
			packageName = packageName + runModeName
		}
	}

	ret = filepath.Join(ret, packageName)

	return ret
}

func NewHomeFinder(appName string, getHome ConfigGetter, osname string, getRunMode RunModeGetter, getLog LogGetter) HomeFinder {
	base := Base{appName, getHome, getRunMode, getLog}
	switch osname {
	case "windows":
		return Win32{base}
	case "darwin":
		return Darwin{base}
	default:
		return XdgPosix{base}
	}
}
<|endoftext|>


<commit_before>package msgclient

import (
	"context"

	"github.com/OneOfOne/xxhash"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/messaging/broker"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

type Publisher struct {
	publishClients     []messaging_pb.SeaweedMessaging_PublishClient
	topicConfiguration *messaging_pb.TopicConfiguration
	messageCount       uint64
	publisherId        string
}

/*
func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (*Publisher, error) {
	// read topic configuration
	topicConfiguration := &messaging_pb.TopicConfiguration{
		PartitionCount: 4,
	}
	publishClients := make([]messaging_pb.SeaweedMessaging_PublishClient, topicConfiguration.PartitionCount)
	for i := 0; i < int(topicConfiguration.PartitionCount); i++ {
		client, err := setupPublisherClient(broker.TopicPartition{
			Namespace: namespace,
			Topic: topic,
			Partition: int32(i),
		})
		if err != nil {
			return nil, err
		}
		publishClients[i] = client
	}
	return &Publisher{
		publishClients:     publishClients,
		topicConfiguration: topicConfiguration,
	}, nil
}
*/

func setupPublisherClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition) (messaging_pb.SeaweedMessaging_PublishClient, error) {

	stream, err := messaging_pb.NewSeaweedMessagingClient(grpcConnection).Publish(context.Background())
	if err != nil {
		return nil, err
	}

	// send init message
	err = stream.Send(&messaging_pb.PublishRequest{
		Init: &messaging_pb.PublishRequest_InitMessage{
			Namespace: tp.Namespace,
			Topic:     tp.Topic,
			Partition: tp.Partition,
		},
	})
	if err != nil {
		return nil, err
	}

	// process init response
	initResponse, err := stream.Recv()
	if err != nil {
		return nil, err
	}
	if initResponse.Redirect != nil {
		// TODO follow redirection
	}
	if initResponse.Config != nil {
	}

	// setup looks for control messages
	doneChan := make(chan error, 1)
	go func() {
		for {
			in, err := stream.Recv()
			if err != nil {
				doneChan <- err
				return
			}
			if in.Redirect != nil {
			}
			if in.Config != nil {
			}
		}
	}()

	return stream, nil

}

func (p *Publisher) Publish(m *messaging_pb.Message) error {
	hashValue := p.messageCount
	p.messageCount++
	if p.topicConfiguration.Partitoning == messaging_pb.TopicConfiguration_NonNullKeyHash {
		if m.Key != nil {
			hashValue = xxhash.Checksum64(m.Key)
		}
	} else if p.topicConfiguration.Partitoning == messaging_pb.TopicConfiguration_KeyHash {
		hashValue = xxhash.Checksum64(m.Key)
	} else {
		// round robin
	}

	idx := int(hashValue) % len(p.publishClients)
	if idx < 0 {
		idx += len(p.publishClients)
	}
	return p.publishClients[idx].Send(&messaging_pb.PublishRequest{
		Data: m,
	})
}

func (p *Publisher) Shutdown() {
	for _, client := range p.publishClients {
		client.CloseSend()
	}
}
<commit_msg>able to connect publisher<commit_after>package msgclient

import (
	"context"

	"github.com/OneOfOne/xxhash"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/messaging/broker"
	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
)

type Publisher struct {
	publishClients     []messaging_pb.SeaweedMessaging_PublishClient
	topicConfiguration *messaging_pb.TopicConfiguration
	messageCount       uint64
	publisherId        string
}

func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (*Publisher, error) {
	// read topic configuration
	topicConfiguration := &messaging_pb.TopicConfiguration{
		PartitionCount: 4,
	}
	publishClients := make([]messaging_pb.SeaweedMessaging_PublishClient, topicConfiguration.PartitionCount)
	for i := 0; i < int(topicConfiguration.PartitionCount); i++ {
		tp := broker.TopicPartition{
			Namespace: namespace,
			Topic: topic,
			Partition: int32(i),
		}
		grpcClientConn, err := mc.findBroker(tp)
		if err != nil {
			return nil, err
		}
		client, err := setupPublisherClient(grpcClientConn, tp)
		if err != nil {
			return nil, err
		}
		publishClients[i] = client
	}
	return &Publisher{
		publishClients:     publishClients,
		topicConfiguration: topicConfiguration,
	}, nil
}

func setupPublisherClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition) (messaging_pb.SeaweedMessaging_PublishClient, error) {

	stream, err := messaging_pb.NewSeaweedMessagingClient(grpcConnection).Publish(context.Background())
	if err != nil {
		return nil, err
	}

	// send init message
	err = stream.Send(&messaging_pb.PublishRequest{
		Init: &messaging_pb.PublishRequest_InitMessage{
			Namespace: tp.Namespace,
			Topic:     tp.Topic,
			Partition: tp.Partition,
		},
	})
	if err != nil {
		return nil, err
	}

	// process init response
	initResponse, err := stream.Recv()
	if err != nil {
		return nil, err
	}
	if initResponse.Redirect != nil {
		// TODO follow redirection
	}
	if initResponse.Config != nil {
	}

	// setup looks for control messages
	doneChan := make(chan error, 1)
	go func() {
		for {
			in, err := stream.Recv()
			if err != nil {
				doneChan <- err
				return
			}
			if in.Redirect != nil {
			}
			if in.Config != nil {
			}
		}
	}()

	return stream, nil

}

func (p *Publisher) Publish(m *messaging_pb.Message) error {
	hashValue := p.messageCount
	p.messageCount++
	if p.topicConfiguration.Partitoning == messaging_pb.TopicConfiguration_NonNullKeyHash {
		if m.Key != nil {
			hashValue = xxhash.Checksum64(m.Key)
		}
	} else if p.topicConfiguration.Partitoning == messaging_pb.TopicConfiguration_KeyHash {
		hashValue = xxhash.Checksum64(m.Key)
	} else {
		// round robin
	}

	idx := int(hashValue) % len(p.publishClients)
	if idx < 0 {
		idx += len(p.publishClients)
	}
	return p.publishClients[idx].Send(&messaging_pb.PublishRequest{
		Data: m,
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	namespace = "prometheus"
	job       = "target_job"
	instance  = "target_instance"
	failure   = "failure"
	outcome   = "outcome"
	success   = "success"
	interval  = "interval"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_operation_latency_milliseconds",
			Help:       "The latencies for target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{job, instance, outcome},
	)
	targetIntervalLength = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_interval_length_seconds",
			Help:       "Actual intervals between scrapes.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{interval},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
	prometheus.MustRegister(targetIntervalLength)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Return the last encountered scrape error, if any.
	LastError() error
	// Return the health of the target.
	State() TargetState
	// Return the last time a scrape was attempted.
	LastScrape() time.Time
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// Merge a new externally supplied target definition (e.g. with changed base
	// labels) into an old target definition for the same endpoint. Preserve
	// remaining information - like health state - from the old target.
	Merge(newTarget Target)
	// Scrape target at the specified interval.
	RunScraper(extraction.Ingester, time.Duration)
	// Stop scraping, synchronous.
	StopScraper()
	// Do a single scrape.
	scrape(ingester extraction.Ingester) error
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error
	// The last time a scrape was attempted.
	lastScrape time.Time
	// Channel to signal RunScraper should stop, holds a channel
	// to notify once stopped.
	stopScraper chan bool

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:     address,
		Deadline:    deadline,
		baseLabels:  baseLabels,
		httpClient:  utility.NewDeadlineClient(deadline),
		stopScraper: make(chan bool),
	}

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

func (t *target) RunScraper(ingester extraction.Ingester, interval time.Duration) {
	jitterTimer := time.NewTimer(time.Duration(float64(interval) * rand.Float64()))
	select {
	case <-jitterTimer.C:
	case <-t.stopScraper:
		return
	}
	jitterTimer.Stop()

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	t.lastScrape = time.Now()
	t.scrape(ingester)
	for {
		select {
		case <-ticker.C:
			targetIntervalLength.WithLabelValues(interval.String()).Observe(float64(time.Since(t.lastScrape) / time.Second))
			t.lastScrape = time.Now()
			t.scrape(ingester)
		case <-t.stopScraper:
			return
		}
	}
}

func (t *target) StopScraper() {
	t.stopScraper <- true
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema=prometheus/telemetry;version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(ingester extraction.Ingester) (err error) {
	timestamp := clientmodel.Now()
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{
			job:      string(t.baseLabels[clientmodel.JobLabel]),
			instance: t.Address(),
			outcome:  success,
		}
		if err == nil {
			t.state = ALIVE
			t.recordScrapeHealth(ingester, timestamp, true)
			labels[outcome] = failure
		} else {
			t.state = UNREACHABLE
			t.recordScrapeHealth(ingester, timestamp, false)
		}
		targetOperationLatencies.With(labels).Observe(ms)
		t.lastError = err
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	// XXX: This is a wart; we need to handle this more gracefully down the
	//      road, especially once we have service discovery support.
	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

func (t *target) LastError() error {
	return t.lastError
}

func (t *target) State() TargetState {
	return t.state
}

func (t *target) LastScrape() time.Time {
	return t.lastScrape
}

func (t *target) Address() string {
	return t.address
}

func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

func (t *target) BaseLabels() clientmodel.LabelSet {
	return t.baseLabels
}

// Merge a new externally supplied target definition (e.g. with changed base
// labels) into an old target definition for the same endpoint. Preserve
// remaining information - like health state - from the old target.
func (t *target) Merge(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.baseLabels = newTarget.BaseLabels()
}

type targets []Target
<commit_msg>Fix the accept header.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package retrieval

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/extraction"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	InstanceLabel clientmodel.LabelName = "instance"
	// The metric name for the synthetic health variable.
	ScrapeHealthMetricName clientmodel.LabelValue = "up"

	// Constants for instrumentation.
	namespace = "prometheus"
	job       = "target_job"
	instance  = "target_instance"
	failure   = "failure"
	outcome   = "outcome"
	success   = "success"
	interval  = "interval"
)

var (
	localhostRepresentations = []string{"http://127.0.0.1", "http://localhost"}

	targetOperationLatencies = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_operation_latency_milliseconds",
			Help:       "The latencies for target operations.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{job, instance, outcome},
	)
	targetIntervalLength = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  namespace,
			Name:       "target_interval_length_seconds",
			Help:       "Actual intervals between scrapes.",
			Objectives: []float64{0.01, 0.05, 0.5, 0.90, 0.99},
		},
		[]string{interval},
	)
)

func init() {
	prometheus.MustRegister(targetOperationLatencies)
	prometheus.MustRegister(targetIntervalLength)
}

// The state of the given Target.
type TargetState int

func (t TargetState) String() string {
	switch t {
	case UNKNOWN:
		return "UNKNOWN"
	case ALIVE:
		return "ALIVE"
	case UNREACHABLE:
		return "UNREACHABLE"
	}

	panic("unknown state")
}

const (
	// The Target has not been seen; we know nothing about it, except that it is
	// on our docket for examination.
	UNKNOWN TargetState = iota
	// The Target has been found and successfully queried.
	ALIVE
	// The Target was either historically found or not found and then determined
	// to be unhealthy by either not responding or disappearing.
	UNREACHABLE
)

// A Target represents an endpoint that should be interrogated for metrics.
//
// The protocol described by this type will likely change in future iterations,
// as it offers no good support for aggregated targets and fan out.  Thusly,
// it is likely that the current Target and target uses will be
// wrapped with some resolver type.
//
// For the future, the Target protocol will abstract away the exact means that
// metrics are retrieved and deserialized from the given instance to which it
// refers.
type Target interface {
	// Return the last encountered scrape error, if any.
	LastError() error
	// Return the health of the target.
	State() TargetState
	// Return the last time a scrape was attempted.
	LastScrape() time.Time
	// The address to which the Target corresponds.  Out of all of the available
	// points in this interface, this one is the best candidate to change given
	// the ways to express the endpoint.
	Address() string
	// The address as seen from other hosts. References to localhost are resolved
	// to the address of the prometheus server.
	GlobalAddress() string
	// Return the target's base labels.
	BaseLabels() clientmodel.LabelSet
	// Merge a new externally supplied target definition (e.g. with changed base
	// labels) into an old target definition for the same endpoint. Preserve
	// remaining information - like health state - from the old target.
	Merge(newTarget Target)
	// Scrape target at the specified interval.
	RunScraper(extraction.Ingester, time.Duration)
	// Stop scraping, synchronous.
	StopScraper()
	// Do a single scrape.
	scrape(ingester extraction.Ingester) error
}

// target is a Target that refers to a singular HTTP or HTTPS endpoint.
type target struct {
	// The current health state of the target.
	state TargetState
	// The last encountered scrape error, if any.
	lastError error
	// The last time a scrape was attempted.
	lastScrape time.Time
	// Channel to signal RunScraper should stop, holds a channel
	// to notify once stopped.
	stopScraper chan bool

	address string
	// What is the deadline for the HTTP or HTTPS against this endpoint.
	Deadline time.Duration
	// Any base labels that are added to this target and its metrics.
	baseLabels clientmodel.LabelSet
	// The HTTP client used to scrape the target's endpoint.
	httpClient *http.Client
}

// Furnish a reasonably configured target for querying.
func NewTarget(address string, deadline time.Duration, baseLabels clientmodel.LabelSet) Target {
	target := &target{
		address:     address,
		Deadline:    deadline,
		baseLabels:  baseLabels,
		httpClient:  utility.NewDeadlineClient(deadline),
		stopScraper: make(chan bool),
	}

	return target
}

func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool) {
	metric := clientmodel.Metric{}
	for label, value := range t.baseLabels {
		metric[label] = value
	}
	metric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(ScrapeHealthMetricName)
	metric[InstanceLabel] = clientmodel.LabelValue(t.Address())

	healthValue := clientmodel.SampleValue(0)
	if healthy {
		healthValue = clientmodel.SampleValue(1)
	}

	sample := &clientmodel.Sample{
		Metric:    metric,
		Timestamp: timestamp,
		Value:     healthValue,
	}

	ingester.Ingest(&extraction.Result{
		Err:     nil,
		Samples: clientmodel.Samples{sample},
	})
}

func (t *target) RunScraper(ingester extraction.Ingester, interval time.Duration) {
	jitterTimer := time.NewTimer(time.Duration(float64(interval) * rand.Float64()))
	select {
	case <-jitterTimer.C:
	case <-t.stopScraper:
		return
	}
	jitterTimer.Stop()

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	t.lastScrape = time.Now()
	t.scrape(ingester)
	for {
		select {
		case <-ticker.C:
			targetIntervalLength.WithLabelValues(interval.String()).Observe(float64(time.Since(t.lastScrape) / time.Second))
			t.lastScrape = time.Now()
			t.scrape(ingester)
		case <-t.stopScraper:
			return
		}
	}
}

func (t *target) StopScraper() {
	t.stopScraper <- true
}

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,application/json;schema="prometheus/telemetry";version=0.0.2;q=0.2,*/*;q=0.1`

func (t *target) scrape(ingester extraction.Ingester) (err error) {
	timestamp := clientmodel.Now()
	defer func(start time.Time) {
		ms := float64(time.Since(start)) / float64(time.Millisecond)
		labels := prometheus.Labels{
			job:      string(t.baseLabels[clientmodel.JobLabel]),
			instance: t.Address(),
			outcome:  success,
		}
		if err == nil {
			t.state = ALIVE
			t.recordScrapeHealth(ingester, timestamp, true)
			labels[outcome] = failure
		} else {
			t.state = UNREACHABLE
			t.recordScrapeHealth(ingester, timestamp, false)
		}
		targetOperationLatencies.With(labels).Observe(ms)
		t.lastError = err
	}(time.Now())

	req, err := http.NewRequest("GET", t.Address(), nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Accept", acceptHeader)

	resp, err := t.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("server returned HTTP status %s", resp.Status)
	}

	processor, err := extraction.ProcessorForRequestHeader(resp.Header)
	if err != nil {
		return err
	}

	// XXX: This is a wart; we need to handle this more gracefully down the
	//      road, especially once we have service discovery support.
	baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
	for baseLabel, baseValue := range t.baseLabels {
		baseLabels[baseLabel] = baseValue
	}

	i := &MergeLabelsIngester{
		Labels:          baseLabels,
		CollisionPrefix: clientmodel.ExporterLabelPrefix,

		Ingester: ingester,
	}
	processOptions := &extraction.ProcessOptions{
		Timestamp: timestamp,
	}
	return processor.ProcessSingle(resp.Body, i, processOptions)
}

func (t *target) LastError() error {
	return t.lastError
}

func (t *target) State() TargetState {
	return t.state
}

func (t *target) LastScrape() time.Time {
	return t.lastScrape
}

func (t *target) Address() string {
	return t.address
}

func (t *target) GlobalAddress() string {
	address := t.address
	hostname, err := os.Hostname()
	if err != nil {
		glog.Warningf("Couldn't get hostname: %s, returning target.Address()", err)
		return address
	}
	for _, localhostRepresentation := range localhostRepresentations {
		address = strings.Replace(address, localhostRepresentation, fmt.Sprintf("http://%s", hostname), -1)
	}
	return address
}

func (t *target) BaseLabels() clientmodel.LabelSet {
	return t.baseLabels
}

// Merge a new externally supplied target definition (e.g. with changed base
// labels) into an old target definition for the same endpoint. Preserve
// remaining information - like health state - from the old target.
func (t *target) Merge(newTarget Target) {
	if t.Address() != newTarget.Address() {
		panic("targets don't refer to the same endpoint")
	}
	t.baseLabels = newTarget.BaseLabels()
}

type targets []Target
<|endoftext|>


<commit_before>package main

import (
        "flag"
        "kindi"
        "log"
        "os"
)

const baseUrl = "https://uwe-oauth.appspot.com"

func main() {
	configDir := flag.String("config", "", "path to config directory")
	flag.Parse()

	err := kindi.InitKeychain(*configDir)
	if err != nil {
		log.Fatalf("Error: Initializing keychain: %v", err)
	}

        flag.Parse()
        subcmd := flag.Arg(0)

        switch subcmd {
        case "encrypt" : doEncrypt()
        case "decrypt" : doDecrypt()
        }
}

func doEncrypt() {
	in := flag.String("in", "", "file to be encrypted")
	to := flag.String("to", "", "recipient gmail address")
        os.Args = flag.Args()
        flag.Parse()

	if len(*in) == 0 {
		log.Println("--in cmd line argument required")
	}
	if len(*to) == 0 {
		log.Println("--to cmd line argument required")
	}

	log.Printf("encrypting %v\n", *in)

	err := kindi.EncryptFile([]byte(*to), *in)
	if err != nil {
		log.Fatalf("Error: encrypting file %v: %v", *in, err)
	}
	log.Println("finished")
}

func doDecrypt() {
        in := flag.String("in", "", "file to be decrypted")
        os.Args = flag.Args()
        flag.Parse()

	if len(*in) == 0 {
		log.Println("--in cmd line argument required")
	}

	log.Printf("decrypting %v\n", *in)

	out, err := kindi.DecryptFile(*in)
	if err != nil {
		log.Fatalf("Error: decrypting file %v: %v", *in, err)
	}
	log.Printf("finished decrypting into %v\n", out)
}

<commit_msg>refined usage message and flags<commit_after>package main

import (
        "flag"
	"fmt"
        "kindi"
        "log"
        "os"
)

const baseUrl = "https://uwe-oauth.appspot.com"

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s <options> command <command options>\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\tValid options: --help\n")
	fmt.Fprintf(os.Stderr, "\tValid commands: encrypt, decrypt\n")
	fmt.Fprintf(os.Stderr, "\t\tOptions for encrypt command: --in <path to file to be encrypted> --to <gmail address of recipient>\n")
	fmt.Fprintf(os.Stderr, "\t\tOptions for decrypt command: --in <path to file to be decrypted>\n")
	fmt.Fprintf(os.Stderr, "\tExample usage:\n")
	fmt.Fprintf(os.Stderr, "\t\tEncrypting a file: %s encrypt --in foo.txt --to johndoe@gmail.com\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t\tDecrypting a file: %s decrypt --in foo.txt.kindi\n", os.Args[0])
}

func main() {
	flag.Usage = usage

	configDir := flag.String("config", "", "path to config directory")
	help := flag.Bool("help", false, "show this message")

	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(0)
	}

	err := kindi.InitKeychain(*configDir)
	if err != nil {
		log.Fatalf("Error: Initializing keychain: %v", err)
	}

        flag.Parse()
        subcmd := flag.Arg(0)

        switch subcmd {
        case "encrypt" : doEncrypt()
        case "decrypt" : doDecrypt()
	default: flag.Usage()
        }
}

func doEncrypt() {
	in := flag.String("in", "", "file to be encrypted")
	to := flag.String("to", "", "recipient gmail address")
        os.Args = flag.Args()
        flag.Parse()

	if len(*in) == 0 {
		log.Println("--in cmd line argument required")
	}
	if len(*to) == 0 {
		log.Println("--to cmd line argument required")
	}

	log.Printf("encrypting %v\n", *in)

	err := kindi.EncryptFile([]byte(*to), *in)
	if err != nil {
		log.Fatalf("Error: encrypting file %v: %v", *in, err)
	}
	log.Println("finished")
}

func doDecrypt() {
        in := flag.String("in", "", "file to be decrypted")
        os.Args = flag.Args()
        flag.Parse()

	if len(*in) == 0 {
		log.Println("--in cmd line argument required")
	}

	log.Printf("decrypting %v\n", *in)

	out, err := kindi.DecryptFile(*in)
	if err != nil {
		log.Fatalf("Error: decrypting file %v: %v", *in, err)
	}
	log.Printf("finished decrypting into %v\n", out)
}

<|endoftext|>


<commit_before>// Copyright 2010  The "goheader" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package main

import (
	"bufio"
	//"bytes"
	"flag"
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

const C_LINE_COMMENT = "//!!! "

var header = `// {cmd}
// MACHINE GENERATED BY THE ABOVE COMMAND

package {pkg}
`

var (
	fOS      = flag.String("s", "", "The operating system")
	fPackage = flag.String("p", "", "The name of the package")
	fListOS  = flag.Bool("ls", false, "List of valid systems")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: goheader -s system -g package [defs.h...]\n")
	flag.PrintDefaults()
	os.Exit(1)
}

func turn(fname string) os.Error {
	reSkipLine := regexp.MustCompile(`^#(ifndef|endif|include|define)[ \t\n]`)
	reDefine := regexp.MustCompile(`^(#[ \t]*define)[ \t]+([^ \t]+)[ \t]+(.+)`)
	reMacroInDefine := regexp.MustCompile(`^.*(\(.*\))`)

	reSingleComment := regexp.MustCompile(`^(.+)?/\*[ \t]*(.+)[ \t]*\*/`)
	reStartMultipleComment := regexp.MustCompile(`^/\*(.+)?`)
	reMiddleMultipleComment := regexp.MustCompile(`^[ \t*]*(.+)`)
	reEndMultipleComment := regexp.MustCompile(`^(.+)?\*/`)

	file, err := os.Open(fname, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer file.Close()

	r := bufio.NewReader(file)

	fmt.Println(header)
	var isMultipleComment bool

	for {
		line, err := r.ReadString('\n')
		if err == os.EOF {
			break
		}
		line = strings.TrimSpace(line) + "\n"
		isSingleComment := false

		// === Convert comment of single line.
		if !isMultipleComment {
			if fields := reSingleComment.FindStringSubmatch(line); fields != nil {
				isSingleComment = true
				line = "// " + fields[2] + "\n"

				if fields[1] != "" {
					line = fields[1] + line
				}
			}
		}

		if !isSingleComment && !isMultipleComment && strings.HasPrefix(line, "/*") {
			isMultipleComment = true
		}

		// === Convert comments of multiple line.
		if isMultipleComment {
			if fields := reStartMultipleComment.FindStringSubmatch(line); fields != nil {
				if fields[1] != "\n" {
					line = "// " + fields[1]
					fmt.Print(line)
				}
				continue
			}

			if fields := reEndMultipleComment.FindStringSubmatch(line); fields != nil {
				if fields[1] != "" {
					line = "// " + fields[1] + "\n"
					fmt.Print(line)
				}
				isMultipleComment = false
				continue
			}

			if fields := reMiddleMultipleComment.FindStringSubmatch(line); fields != nil {
				line = "// " + fields[1]
				fmt.Print(line)
				continue
			}
		}

		// === Turn defines.
		if fields := reDefine.FindStringSubmatch(line); fields != nil {
			line = fmt.Sprintf("%s = %s", fields[2], fields[3])

			// Removes comment (if any) to ckeck if it is a macro.
			lastField := strings.Split(fields[3], "//", -1)[0]
			if reMacroInDefine.MatchString(lastField) {
				line = C_LINE_COMMENT + line
			}

			fmt.Print(line)
			continue
		}

		// === Comment another lines.
		if reSkipLine.MatchString(line) {
			line = C_LINE_COMMENT + line
			fmt.Print(line)
			continue
		}

		fmt.Print(line)
	}

	return nil
}

func main() {
	var isValidOS bool
	validOS := []string{"darwin", "freebsd", "linux", "windows"}

	// === Parse the flags
	flag.Usage = usage
	flag.Parse()

	if *fListOS {
		fmt.Print("  = Systems\n\n  ")
		fmt.Println(validOS)
		os.Exit(0)
	}

	if len(os.Args) == 1 || *fOS == "" || *fPackage == "" {
		usage()
	}

	*fOS = strings.ToLower(*fOS)

	for _, v := range validOS {
		if v == *fOS {
			isValidOS = true
			break
		}
	}

	if !isValidOS {
		fmt.Fprintf(os.Stderr, "ERROR: System passed in flag 's' is invalid\n")
		os.Exit(1)
	}

	// === Update header
	cmd := strings.Join(os.Args, " ")
	header = strings.Replace(header, "{cmd}", path.Base(cmd), 1)
	header = strings.Replace(header, "{pkg}", *fPackage, 1)

var File = "/usr/include/asm-generic/ioctls.h"
	if err := turn(File); err != nil {
		fmt.Fprintf(os.Stderr, err.String())
		os.Exit(1)
	}
}

<commit_msg>Add const to groups.<commit_after>// Copyright 2010  The "goheader" Authors
//
// Use of this source code is governed by the Simplified BSD License
// that can be found in the LICENSE file.
//
// This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
// OR CONDITIONS OF ANY KIND, either express or implied. See the License
// for more details.

package main

import (
	"bufio"
	//"bytes"
	"flag"
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"
)

const C_LINE_COMMENT = "//!!! "

var header = `// {cmd}
// MACHINE GENERATED BY THE ABOVE COMMAND

package {pkg}
`

var (
	fOS      = flag.String("s", "", "The operating system")
	fPackage = flag.String("p", "", "The name of the package")
	fListOS  = flag.Bool("ls", false, "List of valid systems")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: goheader -s system -g package [defs.h...]\n")
	flag.PrintDefaults()
	os.Exit(1)
}

func turn(fname string) os.Error {
	reSkipLine := regexp.MustCompile(`^#(ifndef|endif|include|define)[ \t\n]`)
	reDefine := regexp.MustCompile(`^(#[ \t]*define)[ \t]+([^ \t]+)[ \t]+(.+)`)
	reMacroInDefine := regexp.MustCompile(`^.*(\(.*\))`)

	reSingleComment := regexp.MustCompile(`^(.+)?/\*[ \t]*(.+)[ \t]*\*/`)
	reStartMultipleComment := regexp.MustCompile(`^/\*(.+)?`)
	reMiddleMultipleComment := regexp.MustCompile(`^[ \t*]*(.+)`)
	reEndMultipleComment := regexp.MustCompile(`^(.+)?\*/`)

	file, err := os.Open(fname, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer file.Close()

	r := bufio.NewReader(file)

	fmt.Println(header)
	var isMultipleComment, isDefine bool

	for {
		line, err := r.ReadString('\n')
		if err == os.EOF {
			break
		}
		line = strings.TrimSpace(line) + "\n"
		isSingleComment := false

		// === Convert comment of single line.
		if !isMultipleComment {
			if fields := reSingleComment.FindStringSubmatch(line); fields != nil {
				isSingleComment = true
				line = "// " + fields[2] + "\n"

				if fields[1] != "" {
					line = fields[1] + line
				}
			}
		}

		if !isSingleComment && !isMultipleComment && strings.HasPrefix(line, "/*") {
			isMultipleComment = true
		}

		// === Convert comments of multiple line.
		if isMultipleComment {
			if fields := reStartMultipleComment.FindStringSubmatch(line); fields != nil {
				if fields[1] != "\n" {
					line = "// " + fields[1]
					fmt.Print(line)
				}
				continue
			}

			if fields := reEndMultipleComment.FindStringSubmatch(line); fields != nil {
				if fields[1] != "" {
					line = "// " + fields[1] + "\n"
					fmt.Print(line)
				}
				isMultipleComment = false
				continue
			}

			if fields := reMiddleMultipleComment.FindStringSubmatch(line); fields != nil {
				line = "// " + fields[1]
				fmt.Print(line)
				continue
			}
		}

		// === Turn defines.
		if fields := reDefine.FindStringSubmatch(line); fields != nil {
			if !isDefine {
				isDefine = true
				fmt.Print("const (\n")
			}
			line = fmt.Sprintf("%s = %s", fields[2], fields[3])

			// Removes comment (if any) to ckeck if it is a macro.
			lastField := strings.Split(fields[3], "//", -1)[0]
			if reMacroInDefine.MatchString(lastField) {
				line = C_LINE_COMMENT + line
			}

			fmt.Print(line)
			continue
		}

		if isDefine && line == "\n" {
			fmt.Print(")\n\n")
			isDefine = false
			continue
		}

		// === Comment another lines.
		if reSkipLine.MatchString(line) {
			line = C_LINE_COMMENT + line
			fmt.Print(line)
			continue
		}

		fmt.Print(line)
	}

	return nil
}

func main() {
	var isValidOS bool
	validOS := []string{"darwin", "freebsd", "linux", "windows"}

	// === Parse the flags
	flag.Usage = usage
	flag.Parse()

	if *fListOS {
		fmt.Print("  = Systems\n\n  ")
		fmt.Println(validOS)
		os.Exit(0)
	}

	if len(os.Args) == 1 || *fOS == "" || *fPackage == "" {
		usage()
	}

	*fOS = strings.ToLower(*fOS)

	for _, v := range validOS {
		if v == *fOS {
			isValidOS = true
			break
		}
	}

	if !isValidOS {
		fmt.Fprintf(os.Stderr, "ERROR: System passed in flag 's' is invalid\n")
		os.Exit(1)
	}

	// === Update header
	cmd := strings.Join(os.Args, " ")
	header = strings.Replace(header, "{cmd}", path.Base(cmd), 1)
	header = strings.Replace(header, "{pkg}", *fPackage, 1)

var File = "/usr/include/asm-generic/ioctls.h"
	if err := turn(File); err != nil {
		fmt.Fprintf(os.Stderr, err.String())
		os.Exit(1)
	}
}

<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
)

type debugger bool

func (d debugger) Printf(format string, args ...interface{}) {
	if d {
		log.Printf(format, args...)
	}
}

func (d debugger) Print(args ...interface{}) {
	if d {
		log.Print(args...)
	}
}

const debug debugger = false
const info debugger = true

func main() {
	hostname, _ := os.Hostname()

	var source = flag.String("source", hostname, "source (for reporting)")
	var interval = flag.Int("interval", 10, "stats interval (in seconds)")
	var reg_addr = flag.String("registration", "tcp://127.0.0.1:5867", "address to which clients register")
	var librato_email = flag.String("librato_email", "", "librato email")
	var librato_token = flag.String("librato_token", "", "librato token")

	flag.Parse()

	reg_chan := make(chan Registration)
	stats_channels := NewStatsChannels()
	ts_complete := make(chan int64)
	ts_new := make(chan int64)

	go StartRegistration(*reg_addr, reg_chan)

	go StartClientManager(*interval, reg_chan, stats_channels, ts_complete, ts_new)

	output_chan := make(chan *TimestampedStats)

	go RunAggregator(stats_channels, ts_complete, ts_new, output_chan)

	if len(*librato_email) > 0 && len(*librato_token) > 0 {
		librato := NewLibrato(*source, *librato_email, *librato_token)
		go RunOutput(output_chan, librato)
	} else {
		go RunOutput(output_chan, nil)
	}

	// Just used for debugging
	go log.Fatal(http.ListenAndServe(":8080", nil))

	// Exit cleanly
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c
	info.Print("[main] Exiting cleanly")
}
<commit_msg>Log at startup<commit_after>package main

import (
	"flag"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
)

type debugger bool

func (d debugger) Printf(format string, args ...interface{}) {
	if d {
		log.Printf(format, args...)
	}
}

func (d debugger) Print(args ...interface{}) {
	if d {
		log.Print(args...)
	}
}

const debug debugger = false
const info debugger = true

func main() {
	hostname, _ := os.Hostname()

	var source = flag.String("source", hostname, "source (for reporting)")
	var interval = flag.Int("interval", 10, "stats interval (in seconds)")
	var reg_addr = flag.String("registration", "tcp://127.0.0.1:5867", "address to which clients register")
	var librato_email = flag.String("librato_email", "", "librato email")
	var librato_token = flag.String("librato_token", "", "librato token")

	flag.Parse()

	reg_chan := make(chan Registration)
	stats_channels := NewStatsChannels()
	ts_complete := make(chan int64)
	ts_new := make(chan int64)

	go StartRegistration(*reg_addr, reg_chan)

	go StartClientManager(*interval, reg_chan, stats_channels, ts_complete, ts_new)

	output_chan := make(chan *TimestampedStats)

	go RunAggregator(stats_channels, ts_complete, ts_new, output_chan)

	if len(*librato_email) > 0 && len(*librato_token) > 0 {
		librato := NewLibrato(*source, *librato_email, *librato_token)
		go RunOutput(output_chan, librato)
	} else {
		go RunOutput(output_chan, nil)
	}

	info.Printf("Stagger running")

	// Just used for debugging
	go log.Fatal(http.ListenAndServe(":8080", nil))

	// Exit cleanly
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	<-c
	info.Print("[main] Exiting cleanly")
}
<|endoftext|>