<commit_before>package sum

import "strconv"
import "fmt"

func ConvertInt(input1 string) int64{
  value, err := strconv.ParseInt(input1, 10, 64)
  if err == nil {
    fmt.Println(value)
  }else{
    fmt.Println("error")
  }
  return value
  }

func ConvertUint(input2 string) uint64{
    value, err := strconv.ParseUint(input2, 10, 64)
    if err == nil {
      fmt.Println(value)
    }else{
      fmt.Println("error")
    }
    return value
    }
func ConvertFloat(input3 string) float64{
    value, err := strconv.ParseFloat(input3, 10, 64)
    if err == nil {
        fmt.Println(value)
     }else{
          fmt.Println("error")
      }
      return value
      }
<commit_msg>fjernet parameter som ikke skulle være i strconv.ParseFloat<commit_after>package sum

import "strconv"
import "fmt"

func ConvertInt(input1 string) int64{
  value, err := strconv.ParseInt(input1, 10, 64)
  if err == nil {
    fmt.Println(value)
  }else{
    fmt.Println("error")
  }
  return value
  }

func ConvertUint(input2 string) uint64{
    value, err := strconv.ParseUint(input2, 10, 64)
    if err == nil {
      fmt.Println(value)
    }else{
      fmt.Println("error")
    }
    return value
    }
func ConvertFloat(input3 string) float64{
    value, err := strconv.ParseFloat(input3, 64)
    if err == nil {
        fmt.Println(value)
     }else{
          fmt.Println("error")
      }
      return value
      }
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/shadyoak/grpc-counter/client"
	"google.golang.org/grpc/grpclog"
)

const (
	host = "localhost"
	port = 5000
)

func listenForUpdates(client *client.CounterClient) chan bool {
	c := make(chan bool)

	printChan := make(chan int)
	go printCounts(printChan)

	go func() {
	Loop:
		for {
			select {
			case count := <-client.Receive.CounterUpdate:
				printChan <- count
			case err := <-client.Receive.Error:
				close(printChan)
				grpclog.Fatalf("fatal error: %v", err)
			case <-client.Receive.Done:
				grpclog.Printf("updates complete")
				close(printChan)
				close(c)
				break Loop
			}
		}
	}()
	return c
}

func printCounts(c chan int) {

	flush := time.Tick(25 * time.Millisecond)

	var count *int
	for {
		select {
		case counter := <-c:
			count = &counter
		case <-flush:
			if count != nil {
				grpclog.Printf("got count: %v", *count)
				count = nil
			}
		}
	}
}

func main() {
	client := client.New(host, port)
	if err := client.Start(); err != nil {
		grpclog.Fatalf("unable to connect to client: %v", err)
	}
	defer client.Stop()
	grpclog.Println("connected to counter server:", client.Address())

	listen := listenForUpdates(client)

	for i := 0; i < 1000000; i++ {
		if err := client.IncrementCounter(1); err != nil {
			grpclog.Fatalf("increment counter error: %v", err)
		}
		//time.Sleep(500 * time.Millisecond)
	}

	<-listen
}
<commit_msg>Refactored demo<commit_after>package main

import (
	"time"

	"github.com/shadyoak/grpc-counter/client"
	"google.golang.org/grpc/grpclog"
)

const (
	host    = "localhost"
	port    = 5000
	countTo = 10
	delay   = 500 /* milliseconds */
)

func listenForUpdates(client *client.CounterClient) chan bool {
	c := make(chan bool)

	printChan := make(chan int)
	go printCounts(printChan)

	go func() {
	Loop:
		for {
			select {
			case count := <-client.Receive.CounterUpdate:
				printChan <- count
			case err := <-client.Receive.Error:
				close(printChan)
				grpclog.Fatalf("fatal error: %v", err)
			case <-client.Receive.Done:
				grpclog.Printf("updates complete")
				close(printChan)
				close(c)
				break Loop
			}
		}
	}()
	return c
}

func printCounts(c chan int) {

	flush := time.Tick(25 * time.Millisecond)

	var count *int
	for {
		select {
		case counter := <-c:
			count = &counter
		case <-flush:
			if count != nil {
				grpclog.Printf("got count: %v", *count)
				count = nil
			}
		}
	}
}

func main() {
	client := client.New(host, port)
	if err := client.Start(); err != nil {
		grpclog.Fatalf("unable to connect to client: %v", err)
	}
	defer client.Stop()
	grpclog.Println("connected to counter server:", client.Address())

	listen := listenForUpdates(client)

	for i := 0; i < countTo; i++ {
		if err := client.IncrementCounter(1); err != nil {
			grpclog.Fatalf("increment counter error: %v", err)
		}
		time.Sleep(delay * time.Millisecond)
	}

	<-listen
}
<|endoftext|>


<commit_before>package output
<commit_msg>Test coverage for export template helper<commit_after>package output

import (
	"testing"
)

func TestExport(t *testing.T) {
	if r := export(""); r != "" {
		t.Errorf(`Empty input, the same string was expected as a result. Got "%s".`, r)
	}
	if r := export("a"); r != "A" {
		t.Errorf(`Incorrect result. Expected "A", got "%s".`, r)
	}
	if r := export("int"); r != "Int" {
		t.Errorf(`Incorrect result. Expected "Int", got "%s".`, r)
	}
}
<|endoftext|>


<commit_before>// +build android

package libkb

import "sync"

// This represents the interface to the actual keystore
type ExternalKeyStore interface {
	RetrieveSecret(username string) ([]byte, error)
	StoreSecret(username string, secret []byte) error
	ClearSecret(username string) error
	GetUsersWithStoredSecretsMsgPack() ([]byte, error)
	SetupKeyStore(username string) error
	GetTerminalPrompt() string
}

// Represents interface to some external key store
var GlobalExternalKeyStore ExternalKeyStore

var l sync.Mutex

// This is called by Android to register Android's KeyStore with Go
func SetGlobalExternalKeyStore(s ExternalKeyStore) {
	l.Lock()
	defer l.Unlock()
	GlobalExternalKeyStore = s
}

func GetGlobalExternalKeyStore() ExternalKeyStore {
	l.Lock()
	defer l.Unlock()
	return GlobalExternalKeyStore
}

type SecretStoreAccountName struct {
	accountName      string
	externalKeyStore ExternalKeyStore
}

func (s SecretStoreAccountName) StoreSecret(secret []byte) (err error) {
	return s.externalKeyStore.StoreSecret(s.accountName, secret)
}

func (s SecretStoreAccountName) RetrieveSecret() ([]byte, error) {
	return s.externalKeyStore.RetrieveSecret(s.accountName)
}

func (s SecretStoreAccountName) ClearSecret() (err error) {
	return s.externalKeyStore.ClearSecret(s.accountName)
}

func NewSecretStore(username NormalizedUsername) SecretStore {
	externalKeyStore := GetGlobalExternalKeyStore()
	if externalKeyStore == nil {
		return nil
	}
	externalKeyStore.SetupKeyStore(string(username))
	return SecretStoreAccountName{string(username), externalKeyStore}
}

func HasSecretStore() bool {
	return true
}

func GetUsersWithStoredSecrets() ([]string, error) {
	usersMsgPack, err := GetGlobalExternalKeyStore().GetUsersWithStoredSecretsMsgPack()
	if err != nil {
		return nil, err
	}
	var users []string
	ch := codecHandle()
	err = MsgpackDecodeAll(usersMsgPack, ch, &users)
	return users, err
}

func GetTerminalPrompt() string {
	return GetGlobalExternalKeyStore().GetTerminalPrompt()
}
<commit_msg>Export fewer things<commit_after>// +build android

package libkb

import "sync"

// ExternalKeyStore is the interface for the actual (external) keystore.
type ExternalKeyStore interface {
	RetrieveSecret(username string) ([]byte, error)
	StoreSecret(username string, secret []byte) error
	ClearSecret(username string) error
	GetUsersWithStoredSecretsMsgPack() ([]byte, error)
	SetupKeyStore(username string) error
	GetTerminalPrompt() string
}

// externalKeyStore is the reference to some external key store
var externalKeyStore ExternalKeyStore

var externalKeyStoreMu sync.Mutex

// SetGlobalExternalKeyStore is called by Android to register Android's KeyStore with Go
func SetGlobalExternalKeyStore(s ExternalKeyStore) {
	externalKeyStoreMu.Lock()
	defer externalKeyStoreMu.Unlock()
	externalKeyStore = s
}

func getGlobalExternalKeyStore() ExternalKeyStore {
	externalKeyStoreMu.Lock()
	defer externalKeyStoreMu.Unlock()
	return externalKeyStore
}

type secretStoreAccountName struct {
	externalKeyStore ExternalKeyStore
	accountName      string
}

func (s secretStoreAccountName) StoreSecret(secret []byte) (err error) {
	return s.externalKeyStore.StoreSecret(s.accountName, secret)
}

func (s secretStoreAccountName) RetrieveSecret() ([]byte, error) {
	return s.externalKeyStore.RetrieveSecret(s.accountName)
}

func (s secretStoreAccountName) ClearSecret() (err error) {
	return s.externalKeyStore.ClearSecret(s.accountName)
}

func NewSecretStore(username NormalizedUsername) SecretStore {
	externalKeyStore := getGlobalExternalKeyStore()
	if externalKeyStore == nil {
		return nil
	}
	externalKeyStore.SetupKeyStore(string(username))
	return secretStoreAccountName{externalKeyStore, string(username)}
}

func HasSecretStore() bool {
	return getGlobalExternalKeyStore() != nil
}

func GetUsersWithStoredSecrets() ([]string, error) {
	externalKeyStore := getGlobalExternalKeyStore()
	if externalKeyStore == nil {
		return nil, nil
	}
	usersMsgPack, err := externalKeyStore.GetUsersWithStoredSecretsMsgPack()
	if err != nil {
		return nil, err
	}
	var users []string
	ch := codecHandle()
	err = MsgpackDecodeAll(usersMsgPack, ch, &users)
	return users, err
}

func GetTerminalPrompt() string {
	return getGlobalExternalKeyStore().GetTerminalPrompt()
}
<|endoftext|>


<commit_before>/*
Copyright 2012, Google Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
    * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package tabletserver

import (
	"code.google.com/p/vitess/go/pools"
	"code.google.com/p/vitess/go/relog"
	"code.google.com/p/vitess/go/timer"
	"fmt"
	"sync/atomic"
	"time"
)

type ActivePool struct {
	pool     *pools.Numbered
	timeout  int64
	connPool *ConnectionPool
	ticks    *timer.Timer
}

func NewActivePool(queryTimeout, idleTimeout time.Duration) *ActivePool {
	return &ActivePool{
		pool:     pools.NewNumbered(),
		timeout:  int64(queryTimeout),
		connPool: NewConnectionPool(1, idleTimeout),
		ticks:    timer.NewTimer(idleTimeout / 10),
	}
}

func (self *ActivePool) Open(ConnFactory CreateConnectionFunc) {
	self.connPool.Open(ConnFactory)
	go self.QueryKiller()
}

func (self *ActivePool) Close() {
	self.ticks.Close()
	self.connPool.Close()
	self.pool = pools.NewNumbered()
}

func (self *ActivePool) QueryKiller() {
	for self.ticks.Next() {
		for _, v := range self.pool.GetTimedout(time.Duration(self.Timeout())) {
			self.kill(v.(int64))
		}
	}
}

func (self *ActivePool) kill(connid int64) {
	self.Remove(connid)
	killStats.Add("Queries", 1)
	relog.Info("killing query %d", connid)
	killConn := self.connPool.Get()
	defer killConn.Recycle()
	sql := []byte(fmt.Sprintf("kill %d", connid))
	if _, err := killConn.ExecuteFetch(sql, 10000, false); err != nil {
		relog.Error("Could not kill query %d: %v", connid, err)
	}
}

func (self *ActivePool) Put(id int64) {
	self.pool.Register(id, id)
}

func (self *ActivePool) Remove(id int64) {
	self.pool.Unregister(id)
}

func (self *ActivePool) Timeout() time.Duration {
	return time.Duration(atomic.LoadInt64(&self.timeout))
}

func (self *ActivePool) SetTimeout(timeout time.Duration) {
	atomic.StoreInt64(&self.timeout, int64(timeout))
	self.ticks.SetInterval(timeout / 10)
}

func (self *ActivePool) SetIdleTimeout(idleTimeout time.Duration) {
	self.connPool.SetIdleTimeout(idleTimeout)
}

func (self *ActivePool) StatsJSON() string {
	s, t := self.Stats()
	return fmt.Sprintf("{\"Size\": %v, \"Timeout\": %v}", s, float64(t)/1e9)
}

func (self *ActivePool) Stats() (size int, timeout time.Duration) {
	return self.pool.Stats(), self.Timeout()
}
<commit_msg>bug fix: active pool was using the wrong time out value to poll for long running queries.<commit_after>/*
Copyright 2012, Google Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

    * Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
    * Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package tabletserver

import (
	"code.google.com/p/vitess/go/pools"
	"code.google.com/p/vitess/go/relog"
	"code.google.com/p/vitess/go/timer"
	"fmt"
	"sync/atomic"
	"time"
)

type ActivePool struct {
	pool     *pools.Numbered
	timeout  int64
	connPool *ConnectionPool
	ticks    *timer.Timer
}

func NewActivePool(queryTimeout, idleTimeout time.Duration) *ActivePool {
	return &ActivePool{
		pool:     pools.NewNumbered(),
		timeout:  int64(queryTimeout),
		connPool: NewConnectionPool(1, idleTimeout),
		ticks:    timer.NewTimer(queryTimeout / 10),
	}
}

func (self *ActivePool) Open(ConnFactory CreateConnectionFunc) {
	self.connPool.Open(ConnFactory)
	go self.QueryKiller()
}

func (self *ActivePool) Close() {
	self.ticks.Close()
	self.connPool.Close()
	self.pool = pools.NewNumbered()
}

func (self *ActivePool) QueryKiller() {
	for self.ticks.Next() {
		for _, v := range self.pool.GetTimedout(time.Duration(self.Timeout())) {
			self.kill(v.(int64))
		}
	}
}

func (self *ActivePool) kill(connid int64) {
	self.Remove(connid)
	killStats.Add("Queries", 1)
	relog.Info("killing query %d", connid)
	killConn := self.connPool.Get()
	defer killConn.Recycle()
	sql := []byte(fmt.Sprintf("kill %d", connid))
	if _, err := killConn.ExecuteFetch(sql, 10000, false); err != nil {
		relog.Error("Could not kill query %d: %v", connid, err)
	}
}

func (self *ActivePool) Put(id int64) {
	self.pool.Register(id, id)
}

func (self *ActivePool) Remove(id int64) {
	self.pool.Unregister(id)
}

func (self *ActivePool) Timeout() time.Duration {
	return time.Duration(atomic.LoadInt64(&self.timeout))
}

func (self *ActivePool) SetTimeout(timeout time.Duration) {
	atomic.StoreInt64(&self.timeout, int64(timeout))
	self.ticks.SetInterval(timeout / 10)
}

func (self *ActivePool) SetIdleTimeout(idleTimeout time.Duration) {
	self.connPool.SetIdleTimeout(idleTimeout)
}

func (self *ActivePool) StatsJSON() string {
	s, t := self.Stats()
	return fmt.Sprintf("{\"Size\": %v, \"Timeout\": %v}", s, float64(t)/1e9)
}

func (self *ActivePool) Stats() (size int, timeout time.Duration) {
	return self.pool.Stats(), self.Timeout()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/user"
)

func main() {
	fmt.Println("Hello")
	currentuser, err := user.Current()
	if err == nil {
		fmt.Println("Current user", currentuser)
	} else {
		fmt.Println("Error", err)
	}
}
<commit_msg>Show struct with keywords<commit_after>package main

import (
	"fmt"
	"os/user"
)

func main() {
	fmt.Println("Hello")
	current, err := user.Current()
	if err == nil {
		fmt.Printf("Current user %+v\n", current)
	} else {
		fmt.Println("Error", err)
	}
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod Moderation
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // thumbnail generator for images
  img_thm ThumbnailGenerator

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  allow_tor := self.conf.daemon["allow_tor"]
  allow_tor_attachments := self.conf.daemon["allow_tor_attachments"]
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy,  make(chan string, 512), self.store, self.store, allow_tor == "1", allow_tor_attachments == "1"}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      // start syncing in background
      go func() {
        if self.sync_on_start {
          log.Println("sync on start")
          // get every article
          articles := self.database.GetAllArticles()
          // wait 5 seconds for feed to handshake
          time.Sleep(5 * time.Second)
          log.Println("outfeed begin sync")
          for _, result := range articles {
            msgid := result[0]
            group := result[1]
            if policy.AllowsNewsgroup(group) {
              //XXX: will this crash if interrupted?
              nntp.sync <- msgid
            }
          }
          log.Println("outfeed end sync")
        }
      }()
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    select {
    case nntp := <- chnl:
      // new post from frontend
      log.Println("frontend post", nntp.MessageID())
      self.infeed <- nntp
    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  var chnl chan NNTPMessage
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
              feed.sync <- nntp.MessageID()
            } else {
              log.Println("not syncing", msgid)
            }
          }
        }
      }
      break;
    case msgid := <- self.infeed_load:
      log.Println("load from infeed", msgid)
      msg := self.store.ReadTempMessage(msgid)
      if msg != nil {
        self.infeed <- msg
      }
      break;
    case nntp := <- self.infeed:
      // ammend path
      nntp.AppendPath(self.instance_name)
      msgid := nntp.MessageID()
      log.Println("daemon got", msgid)

      // store article and attachments
      // register with database
      // this also generates thumbnails
      go self.store.StorePost(nntp)

      // prepare for content rollover
      // fallback rollover
      rollover := 100

      group := nntp.Newsgroup()
      tpp, err := self.database.GetThreadsPerPage(group)
      ppb, err := self.database.GetPagesPerBoard(group)
      if err == nil {
        rollover = tpp * ppb
      }

      // roll over old content
      self.expire.ExpireGroup(group, rollover)
      if err == nil {
        // queue to all outfeeds
        self.send_all_feeds <- msgid
        // tell frontend
        if chnl != nil {
          chnl <- nntp
        }
      } else {
        log.Printf("%s failed to store: %s", msgid, err)
      }
      break;
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

func (self *NNTPDaemon) Setup() {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {

  // set up daemon configs
  self.Setup()

  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan string)
  self.feeds = make(map[NNTPConnection]bool)

  self.bind_addr = self.conf.daemon["bind"]

  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return false
  }

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // initialize moderation engine
  self.mod.Init(self)

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }
  return true
}
<commit_msg>try again<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod Moderation
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // thumbnail generator for images
  img_thm ThumbnailGenerator

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  allow_tor := self.conf.daemon["allow_tor"]
  allow_tor_attachments := self.conf.daemon["allow_tor_attachments"]
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy,  make(chan string, 512), self.store, self.store, allow_tor == "1", allow_tor_attachments == "1"}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      // start syncing in background
      go func() {
        if self.sync_on_start {
          log.Println("sync on start")
          // get every article
          articles := self.database.GetAllArticles()
          // wait 5 seconds for feed to handshake
          time.Sleep(5 * time.Second)
          log.Println("outfeed begin sync")
          for _, result := range articles {
            msgid := result[0]
            group := result[1]
            if policy.AllowsNewsgroup(group) {
              //XXX: will this crash if interrupted?
              nntp.sync <- msgid
            }
          }
          log.Println("outfeed end sync")
        }
      }()
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    select {
    case nntp := <- chnl:
      // new post from frontend
      log.Println("frontend post", nntp.MessageID())
      self.infeed <- nntp
    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  var chnl chan NNTPMessage
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
              feed.sync <- nntp.MessageID()
            } else {
              log.Println("not syncing", msgid)
            }
          }
        }
      }
      break;
    case msgid := <- self.infeed_load:
      log.Println("load from infeed", msgid)
      msg := self.store.ReadTempMessage(msgid)
      if msg != nil {
        self.infeed <- msg
      }
      break;
    case nntp := <- self.infeed:
      // ammend path
      nntp.AppendPath(self.instance_name)
      msgid := nntp.MessageID()
      log.Println("daemon got", msgid)

      // store article and attachments
      // register with database
      // this also generates thumbnails
      go self.store.StorePost(nntp)

      // prepare for content rollover
      // fallback rollover
      rollover := 100

      group := nntp.Newsgroup()
      tpp, err := self.database.GetThreadsPerPage(group)
      ppb, err := self.database.GetPagesPerBoard(group)
      if err == nil {
        rollover = tpp * ppb
      }

      // roll over old content
      self.expire.ExpireGroup(group, rollover)
      if err == nil {
        // queue to all outfeeds
        self.send_all_feeds <- msgid
        // tell frontend
        if chnl != nil {
          chnl <- nntp
        }
      } else {
        log.Printf("%s failed to store: %s", msgid, err)
      }
      break;
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

func (self *NNTPDaemon) Setup() {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {

  // set up daemon configs
  self.Setup()

  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan string, 64)
  self.feeds = make(map[NNTPConnection]bool)

  self.bind_addr = self.conf.daemon["bind"]

  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return false
  }

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // initialize moderation engine
  self.mod.Init(self)

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }
  return true
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	%e for large exponents, %f otherwise
		%G	%E for large exponents, %F otherwise
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	The default format for %v is:
		bool:                    %t
		int, int8 etc.:          %d
		uint, uint8 etc.:        %d, %x if printed with %#v
		float32, complex64, etc: %g
		string:                  %s
		chan:                    %p
		pointer:                 %p
	For compound objects, the elements are printed using these rules, recursively,
	laid out like this:
		struct:             {field0 field1 ...}
		array, slice:       [elem0  elem1 ...]
		maps:               map[key1:value1 key2:value2]
		pointer to above:   &{}, &[], &map[]

	Width is specified by an optional decimal number immediately preceding the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f     default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points,
	that is, runes. (This differs from C's printf where the
	units are always measured in bytes.) Either or both of the flags
	may be replaced with the character '*', causing their values to be
	obtained from the next operand, which must be of type int.

	For most values, width is the minimum number of runes to output,
	padding the formatted form with spaces if necessary.

	For strings, byte slices and byte arrays, however, precision
	limits the length of the input to be formatted (not the size of
	the output), truncating if necessary. Normally it is measured in
	runes, but for these types when formatted with the %x or %X format
	it is measured in bytes.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthesized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }
	Infinite recursion can also be triggered by self-referential data
	structures, such as a slice that contains itself as an element, if
	that type has a String method. Such pathologies are rare, however,
	and the package does not protect against them.

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], arguments n+1,
	n+2, etc. will be processed unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<commit_msg>fmt: reword the document for [n].<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package fmt implements formatted I/O with functions analogous
	to C's printf and scanf.  The format 'verbs' are derived from C's but
	are simpler.

	Printing

	The verbs:

	General:
		%v	the value in a default format
			when printing structs, the plus flag (%+v) adds field names
		%#v	a Go-syntax representation of the value
		%T	a Go-syntax representation of the type of the value
		%%	a literal percent sign; consumes no value

	Boolean:
		%t	the word true or false
	Integer:
		%b	base 2
		%c	the character represented by the corresponding Unicode code point
		%d	base 10
		%o	base 8
		%q	a single-quoted character literal safely escaped with Go syntax.
		%x	base 16, with lower-case letters for a-f
		%X	base 16, with upper-case letters for A-F
		%U	Unicode format: U+1234; same as "U+%04X"
	Floating-point and complex constituents:
		%b	decimalless scientific notation with exponent a power of two,
			in the manner of strconv.FormatFloat with the 'b' format,
			e.g. -123456p-78
		%e	scientific notation, e.g. -1234.456e+78
		%E	scientific notation, e.g. -1234.456E+78
		%f	decimal point but no exponent, e.g. 123.456
		%F	synonym for %f
		%g	%e for large exponents, %f otherwise
		%G	%E for large exponents, %F otherwise
	String and slice of bytes:
		%s	the uninterpreted bytes of the string or slice
		%q	a double-quoted string safely escaped with Go syntax
		%x	base 16, lower-case, two characters per byte
		%X	base 16, upper-case, two characters per byte
	Pointer:
		%p	base 16 notation, with leading 0x

	There is no 'u' flag.  Integers are printed unsigned if they have unsigned type.
	Similarly, there is no need to specify the size of the operand (int8, int64).

	The default format for %v is:
		bool:                    %t
		int, int8 etc.:          %d
		uint, uint8 etc.:        %d, %x if printed with %#v
		float32, complex64, etc: %g
		string:                  %s
		chan:                    %p
		pointer:                 %p
	For compound objects, the elements are printed using these rules, recursively,
	laid out like this:
		struct:             {field0 field1 ...}
		array, slice:       [elem0  elem1 ...]
		maps:               map[key1:value1 key2:value2]
		pointer to above:   &{}, &[], &map[]

	Width is specified by an optional decimal number immediately preceding the verb.
	If absent, the width is whatever is necessary to represent the value.
	Precision is specified after the (optional) width by a period followed by a
	decimal number. If no period is present, a default precision is used.
	A period with no following number specifies a precision of zero.
	Examples:
		%f     default width, default precision
		%9f    width 9, default precision
		%.2f   default width, precision 2
		%9.2f  width 9, precision 2
		%9.f   width 9, precision 0

	Width and precision are measured in units of Unicode code points,
	that is, runes. (This differs from C's printf where the
	units are always measured in bytes.) Either or both of the flags
	may be replaced with the character '*', causing their values to be
	obtained from the next operand, which must be of type int.

	For most values, width is the minimum number of runes to output,
	padding the formatted form with spaces if necessary.

	For strings, byte slices and byte arrays, however, precision
	limits the length of the input to be formatted (not the size of
	the output), truncating if necessary. Normally it is measured in
	runes, but for these types when formatted with the %x or %X format
	it is measured in bytes.

	For floating-point values, width sets the minimum width of the field and
	precision sets the number of places after the decimal, if appropriate,
	except that for %g/%G it sets the total number of digits. For example,
	given 123.45 the format %6.2f prints 123.45 while %.4g prints 123.5.
	The default precision for %e and %f is 6; for %g it is the smallest
	number of digits necessary to identify the value uniquely.

	For complex numbers, the width and precision apply to the two
	components independently and the result is parenthesized, so %f applied
	to 1.2+3.4i produces (1.200000+3.400000i).

	Other flags:
		+	always print a sign for numeric values;
			guarantee ASCII-only output for %q (%+q)
		-	pad with spaces on the right rather than the left (left-justify the field)
		#	alternate format: add leading 0 for octal (%#o), 0x for hex (%#x);
			0X for hex (%#X); suppress 0x for %p (%#p);
			for %q, print a raw (backquoted) string if strconv.CanBackquote
			returns true;
			write e.g. U+0078 'x' if the character is printable for %U (%#U).
		' '	(space) leave a space for elided sign in numbers (% d);
			put spaces between bytes printing strings or slices in hex (% x, % X)
		0	pad with leading zeros rather than spaces;
			for numbers, this moves the padding after the sign

	Flags are ignored by verbs that do not expect them.
	For example there is no alternate decimal format, so %#d and %d
	behave identically.

	For each Printf-like function, there is also a Print function
	that takes no format and is equivalent to saying %v for every
	operand.  Another variant Println inserts blanks between
	operands and appends a newline.

	Regardless of the verb, if an operand is an interface value,
	the internal concrete value is used, not the interface itself.
	Thus:
		var i interface{} = 23
		fmt.Printf("%v\n", i)
	will print 23.

	Except when printed using the verbs %T and %p, special
	formatting considerations apply for operands that implement
	certain interfaces. In order of application:

	1. If an operand implements the Formatter interface, it will
	be invoked. Formatter provides fine control of formatting.

	2. If the %v verb is used with the # flag (%#v) and the operand
	implements the GoStringer interface, that will be invoked.

	If the format (which is implicitly %v for Println etc.) is valid
	for a string (%s %q %v %x %X), the following two rules apply:

	3. If an operand implements the error interface, the Error method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	4. If an operand implements method String() string, that method
	will be invoked to convert the object to a string, which will then
	be formatted as required by the verb (if any).

	For compound operands such as slices and structs, the format
	applies to the elements of each operand, recursively, not to the
	operand as a whole. Thus %q will quote each element of a slice
	of strings, and %6.2f will control formatting for each element
	of a floating-point array.

	To avoid recursion in cases such as
		type X string
		func (x X) String() string { return Sprintf("<%s>", x) }
	convert the value before recurring:
		func (x X) String() string { return Sprintf("<%s>", string(x)) }
	Infinite recursion can also be triggered by self-referential data
	structures, such as a slice that contains itself as an element, if
	that type has a String method. Such pathologies are rare, however,
	and the package does not protect against them.

	Explicit argument indexes:

	In Printf, Sprintf, and Fprintf, the default behavior is for each
	formatting verb to format successive arguments passed in the call.
	However, the notation [n] immediately before the verb indicates that the
	nth one-indexed argument is to be formatted instead. The same notation
	before a '*' for a width or precision selects the argument index holding
	the value. After processing a bracketed expression [n], subsequent verbs
	will use arguments n+1, n+2, etc. unless otherwise directed.

	For example,
		fmt.Sprintf("%[2]d %[1]d\n", 11, 22)
	will yield "22 11", while
		fmt.Sprintf("%[3]*.[2]*[1]f", 12.0, 2, 6),
	equivalent to
		fmt.Sprintf("%6.2f", 12.0),
	will yield " 12.00". Because an explicit index affects subsequent verbs,
	this notation can be used to print the same values multiple times
	by resetting the index for the first argument to be repeated:
		fmt.Sprintf("%d %d %#[1]x %#x", 16, 17)
	will yield "16 17 0x10 0x11".

	Format errors:

	If an invalid argument is given for a verb, such as providing
	a string to %d, the generated string will contain a
	description of the problem, as in these examples:

		Wrong type or unknown verb: %!verb(type=value)
			Printf("%d", hi):          %!d(string=hi)
		Too many arguments: %!(EXTRA type=value)
			Printf("hi", "guys"):      hi%!(EXTRA string=guys)
		Too few arguments: %!verb(MISSING)
			Printf("hi%d"):            hi %!d(MISSING)
		Non-int for width or precision: %!(BADWIDTH) or %!(BADPREC)
			Printf("%*s", 4.5, "hi"):  %!(BADWIDTH)hi
			Printf("%.*s", 4.5, "hi"): %!(BADPREC)hi
		Invalid or invalid use of argument index: %!(BADINDEX)
			Printf("%*[2]d", 7):       %!d(BADINDEX)
			Printf("%.[2]d", 7):       %!d(BADINDEX)

	All errors begin with the string "%!" followed sometimes
	by a single character (the verb) and end with a parenthesized
	description.

	If an Error or String method triggers a panic when called by a
	print routine, the fmt package reformats the error message
	from the panic, decorating it with an indication that it came
	through the fmt package.  For example, if a String method
	calls panic("bad"), the resulting formatted message will look
	like
		%!s(PANIC=bad)

	The %!s just shows the print verb in use when the failure
	occurred.

	Scanning

	An analogous set of functions scans formatted text to yield
	values.  Scan, Scanf and Scanln read from os.Stdin; Fscan,
	Fscanf and Fscanln read from a specified io.Reader; Sscan,
	Sscanf and Sscanln read from an argument string.  Scanln,
	Fscanln and Sscanln stop scanning at a newline and require that
	the items be followed by one; Scanf, Fscanf and Sscanf require
	newlines in the input to match newlines in the format; the other
	routines treat newlines as spaces.

	Scanf, Fscanf, and Sscanf parse the arguments according to a
	format string, analogous to that of Printf.  For example, %x
	will scan an integer as a hexadecimal number, and %v will scan
	the default representation format for the value.

	The formats behave analogously to those of Printf with the
	following exceptions:

		%p is not implemented
		%T is not implemented
		%e %E %f %F %g %G are all equivalent and scan any floating point or complex value
		%s and %v on strings scan a space-delimited token
		Flags # and + are not implemented.

	The familiar base-setting prefixes 0 (octal) and 0x
	(hexadecimal) are accepted when scanning integers without a
	format or with the %v verb.

	Width is interpreted in the input text (%5s means at most
	five runes of input will be read to scan a string) but there
	is no syntax for scanning with a precision (no %5.2f, just
	%5f).

	When scanning with a format, all non-empty runs of space
	characters (except newline) are equivalent to a single
	space in both the format and the input.  With that proviso,
	text in the format string must match the input text; scanning
	stops if it does not, with the return value of the function
	indicating the number of arguments scanned.

	In all the scanning functions, a carriage return followed
	immediately by a newline is treated as a plain newline
	(\r\n means the same as \n).

	In all the scanning functions, if an operand implements method
	Scan (that is, it implements the Scanner interface) that
	method will be used to scan the text for that operand.  Also,
	if the number of arguments scanned is less than the number of
	arguments provided, an error is returned.

	All arguments to be scanned must be either pointers to basic
	types or implementations of the Scanner interface.

	Note: Fscan etc. can read one character (rune) past the input
	they return, which means that a loop calling a scan routine
	may skip some of the input.  This is usually a problem only
	when there is no space between input values.  If the reader
	provided to Fscan implements ReadRune, that method will be used
	to read characters.  If the reader also implements UnreadRune,
	that method will be used to save the character and successive
	calls will not lose data.  To attach ReadRune and UnreadRune
	methods to a reader without that capability, use
	bufio.NewReader.
*/
package fmt
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"compress/gzip"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"time"

	cm "cleafy.com/replayer/model"

	"github.com/mattetti/filebuffer"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/sirupsen/logrus"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
	filetype "gopkg.in/h2non/filetype.v1"
)

var replayType = filetype.NewType("rep", "application/replay")

func replayMatcher(buf []byte) bool {
	header, err := cm.ReadFrameHeader(filebuffer.New(buf))
	if err != nil {
		return false
	}
	return cm.CheckVersion(header)
}

var (
	debug            = kingpin.Flag("debug", "Enable debug mode.").Bool()
	nopromcfg        = kingpin.Flag("nopromcfg", "Disable the generation of the prometheus cfg file (prometheus.yml)").Bool()
	dir              = kingpin.Flag("dir", "Input directory.").Short('d').OverrideDefaultFromEnvar("INPUT_DIRECTORY").Default(".").String()
	framereader      = make(<-chan cm.Frame)
	version          = "0.0.1"
	cfgMemoryStorage = local.MemorySeriesStorageOptions{
		MemoryChunks:       1024,
		MaxChunksToPersist: 1024,
		//PersistenceStoragePath:
		//PersistenceRetentionPeriod:
		//CheckpointInterval:         time.Minute*30,
		//CheckpointDirtySeriesLimit: 10000,
		Dirty:          true,
		PedanticChecks: true,
		SyncStrategy:   local.Always,
	}
)

func osfile2fname(fss []os.FileInfo, dir string) []string {
	out := make([]string, len(fss))
	for i, fin := range fss {
		out[i] = dir + "/" + fin.Name()
	}
	return out
}

func generateFramereader() {
	// 1. Check for every file that is GZip or csave format and create the filemap
	files, err := ioutil.ReadDir(*dir)
	if err != nil {
		logrus.Fatalf("generateFilemap: %v", err)
	}
	readers := make([]io.Reader, 0)

	fnames := osfile2fname(files, *dir)
	sort.Sort(cm.ByNumber(fnames))

	logrus.Debugf("fnames: %v", fnames)

	for _, filepath := range fnames {
		logrus.Debugf("filepath: %v", filepath)
		ftype, err := filetype.MatchFile(filepath)
		if err != nil {
			logrus.Debugf("err %v", err)
		}
		if ftype.MIME.Value == "application/replay" {
			f, _ := os.Open(filepath)
			readers = append(readers, f)
		}

		if ftype.MIME.Value == "application/gzip" {
			f, _ := os.Open(filepath)
			logrus.Debugf("reading header: %v", filepath)
			gz, _ := gzip.NewReader(f)
			header, err := cm.ReadFrameHeader(gz)
			if err == nil && cm.CheckVersion(header) {
				f.Seek(0, io.SeekStart)
				gz, _ = gzip.NewReader(f)
				readers = append(readers, gz)
			}
		}
	}

	framereader = cm.NewMultiReader(readers)
}

func updateURLTimestamp(timestamp int64, name string, url string, body io.Reader) io.Reader {
	dec := expfmt.NewDecoder(body, expfmt.FmtText)
	pr, pw := io.Pipe()
	//ts := timestamp * 1000

	go func() {
		for {
			var metrics dto.MetricFamily
			err := dec.Decode(&metrics)
			if err == io.EOF {
				break
			}
			if err != nil {
				logrus.Error(err)
				break
			}

			lpName := "job"
			urlName := "url"

			for _, metric := range metrics.GetMetric() {
				lp := dto.LabelPair{
					Name:  &lpName,
					Value: &name,
				}
				metric.Label = append(metric.Label, &lp)
				urlp := dto.LabelPair{
					Name:  &urlName,
					Value: &url,
				}
				metric.Label = append(metric.Label, &urlp)
			}

			enc := expfmt.NewEncoder(pw, expfmt.FmtText)

			enc.Encode(&metrics)
		}
		pw.Close()
	}()

	return pr
}

func main() {
	kingpin.Version(version)

	kingpin.Flag("storage.path", "Directory path to create and fill the data store under.").Default("data").StringVar(&cfgMemoryStorage.PersistenceStoragePath)
	kingpin.Flag("storage.retention-period", "Period of time to store data for").Default("360h").DurationVar(&cfgMemoryStorage.PersistenceRetentionPeriod)

	kingpin.Flag("storage.checkpoint-interval", "Period of time to store data for").Default("30m").DurationVar(&cfgMemoryStorage.CheckpointInterval)
	kingpin.Flag("storage.checkpoint-dirty-series-limit", "Period of time to store data for").Default("10000").IntVar(&cfgMemoryStorage.CheckpointDirtySeriesLimit)

	kingpin.Parse()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
		flag.Set("log.level", "debug")
	}

	log.Infoln("Prefilling into", cfgMemoryStorage.PersistenceStoragePath)
	localStorage := local.NewMemorySeriesStorage(&cfgMemoryStorage)

	sampleAppender := localStorage

	log.Infoln("Starting the storage engine")
	if err := localStorage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		os.Exit(1)
	}
	defer func() {
		if err := localStorage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	filetype.AddMatcher(replayType, replayMatcher)

	generateFramereader()
	logrus.Debug("frameReader %+v", framereader)

	sout := bufio.NewWriter(os.Stdout)
	defer sout.Flush()

	r := &http.Request{}

	for frame := range framereader {
		response, err := http.ReadResponse(bufio.NewReader(filebuffer.New(frame.Data)), r)
		if err != nil {
			logrus.Error(err)
			return
		}
		bytesReader := updateURLTimestamp(frame.Header.Timestamp, frame.NameString(), frame.URIString(), response.Body)

		sdec := expfmt.SampleDecoder{
			Dec: expfmt.NewDecoder(bytesReader, expfmt.FmtText),
			Opts: &expfmt.DecodeOptions{
				Timestamp: model.TimeFromUnix(frame.Header.Timestamp),
			},
		}

		decSamples := make(model.Vector, 0, 1)

		if err := sdec.Decode(&decSamples); err != nil {
			log.Errorln("Could not decode metric:", err)
			continue
		}

		log.Debugln("Ingested", len(decSamples), "metrics")

		for sampleAppender.NeedsThrottling() {
			log.Debugln("Waiting 100ms for appender to be ready for more data")
			time.Sleep(time.Millisecond * 100)
		}

		var (
			numOutOfOrder = 0
			numDuplicates = 0
		)

		for _, s := range model.Samples(decSamples) {
			if err := sampleAppender.Append(s); err != nil {
				switch err {
				case local.ErrOutOfOrderSample:
					numOutOfOrder++
					log.With("sample", s).With("error", err).Info("Sample discarded")
				case local.ErrDuplicateSampleForTimestamp:
					numDuplicates++
					log.With("sample", s).With("error", err).Info("Sample discarded")
				default:
					log.With("sample", s).With("error", err).Info("Sample discarded")
				}
			}
		}
	}
	// Generate the prometheus.yml in case it does not exist
	promcfgpath := cfgMemoryStorage.PersistenceStoragePath + "/../prometheus.yml"
	if _, err := os.Stat(promcfgpath); os.IsNotExist(err) && !*nopromcfg {
		if f, err := os.Create(promcfgpath); err != nil {
			defer f.Close()
			w := bufio.NewWriter(f)
			w.WriteString("global: {}")
		}
	}

	logrus.Info("Exiting! :)")
}
<commit_msg>promplay: write to the prometheus.yml for real<commit_after>package main

import (
	"bufio"
	"compress/gzip"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"time"

	cm "cleafy.com/replayer/model"

	"github.com/mattetti/filebuffer"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/local"
	"github.com/sirupsen/logrus"
	kingpin "gopkg.in/alecthomas/kingpin.v2"
	filetype "gopkg.in/h2non/filetype.v1"
)

var replayType = filetype.NewType("rep", "application/replay")

func replayMatcher(buf []byte) bool {
	header, err := cm.ReadFrameHeader(filebuffer.New(buf))
	if err != nil {
		return false
	}
	return cm.CheckVersion(header)
}

var (
	debug            = kingpin.Flag("debug", "Enable debug mode.").Bool()
	nopromcfg        = kingpin.Flag("nopromcfg", "Disable the generation of the prometheus cfg file (prometheus.yml)").Bool()
	dir              = kingpin.Flag("dir", "Input directory.").Short('d').OverrideDefaultFromEnvar("INPUT_DIRECTORY").Default(".").String()
	framereader      = make(<-chan cm.Frame)
	version          = "0.0.1"
	cfgMemoryStorage = local.MemorySeriesStorageOptions{
		MemoryChunks:       1024,
		MaxChunksToPersist: 1024,
		//PersistenceStoragePath:
		//PersistenceRetentionPeriod:
		//CheckpointInterval:         time.Minute*30,
		//CheckpointDirtySeriesLimit: 10000,
		Dirty:          true,
		PedanticChecks: true,
		SyncStrategy:   local.Always,
	}
)

func osfile2fname(fss []os.FileInfo, dir string) []string {
	out := make([]string, len(fss))
	for i, fin := range fss {
		out[i] = dir + "/" + fin.Name()
	}
	return out
}

func generateFramereader() {
	// 1. Check for every file that is GZip or csave format and create the filemap
	files, err := ioutil.ReadDir(*dir)
	if err != nil {
		logrus.Fatalf("generateFilemap: %v", err)
	}
	readers := make([]io.Reader, 0)

	fnames := osfile2fname(files, *dir)
	sort.Sort(cm.ByNumber(fnames))

	logrus.Debugf("fnames: %v", fnames)

	for _, filepath := range fnames {
		logrus.Debugf("filepath: %v", filepath)
		ftype, err := filetype.MatchFile(filepath)
		if err != nil {
			logrus.Debugf("err %v", err)
		}
		if ftype.MIME.Value == "application/replay" {
			f, _ := os.Open(filepath)
			readers = append(readers, f)
		}

		if ftype.MIME.Value == "application/gzip" {
			f, _ := os.Open(filepath)
			logrus.Debugf("reading header: %v", filepath)
			gz, _ := gzip.NewReader(f)
			header, err := cm.ReadFrameHeader(gz)
			if err == nil && cm.CheckVersion(header) {
				f.Seek(0, io.SeekStart)
				gz, _ = gzip.NewReader(f)
				readers = append(readers, gz)
			}
		}
	}

	framereader = cm.NewMultiReader(readers)
}

func updateURLTimestamp(timestamp int64, name string, url string, body io.Reader) io.Reader {
	dec := expfmt.NewDecoder(body, expfmt.FmtText)
	pr, pw := io.Pipe()
	//ts := timestamp * 1000

	go func() {
		for {
			var metrics dto.MetricFamily
			err := dec.Decode(&metrics)
			if err == io.EOF {
				break
			}
			if err != nil {
				logrus.Error(err)
				break
			}

			lpName := "job"
			urlName := "url"

			for _, metric := range metrics.GetMetric() {
				lp := dto.LabelPair{
					Name:  &lpName,
					Value: &name,
				}
				metric.Label = append(metric.Label, &lp)
				urlp := dto.LabelPair{
					Name:  &urlName,
					Value: &url,
				}
				metric.Label = append(metric.Label, &urlp)
			}

			enc := expfmt.NewEncoder(pw, expfmt.FmtText)

			enc.Encode(&metrics)
		}
		pw.Close()
	}()

	return pr
}

func main() {
	kingpin.Version(version)

	kingpin.Flag("storage.path", "Directory path to create and fill the data store under.").Default("data").StringVar(&cfgMemoryStorage.PersistenceStoragePath)
	kingpin.Flag("storage.retention-period", "Period of time to store data for").Default("360h").DurationVar(&cfgMemoryStorage.PersistenceRetentionPeriod)

	kingpin.Flag("storage.checkpoint-interval", "Period of time to store data for").Default("30m").DurationVar(&cfgMemoryStorage.CheckpointInterval)
	kingpin.Flag("storage.checkpoint-dirty-series-limit", "Period of time to store data for").Default("10000").IntVar(&cfgMemoryStorage.CheckpointDirtySeriesLimit)

	kingpin.Parse()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
		flag.Set("log.level", "debug")
	}

	log.Infoln("Prefilling into", cfgMemoryStorage.PersistenceStoragePath)
	localStorage := local.NewMemorySeriesStorage(&cfgMemoryStorage)

	sampleAppender := localStorage

	log.Infoln("Starting the storage engine")
	if err := localStorage.Start(); err != nil {
		log.Errorln("Error opening memory series storage:", err)
		os.Exit(1)
	}
	defer func() {
		if err := localStorage.Stop(); err != nil {
			log.Errorln("Error stopping storage:", err)
		}
	}()

	filetype.AddMatcher(replayType, replayMatcher)

	generateFramereader()
	logrus.Debug("frameReader %+v", framereader)

	sout := bufio.NewWriter(os.Stdout)
	defer sout.Flush()

	r := &http.Request{}

	for frame := range framereader {
		response, err := http.ReadResponse(bufio.NewReader(filebuffer.New(frame.Data)), r)
		if err != nil {
			logrus.Error(err)
			return
		}
		bytesReader := updateURLTimestamp(frame.Header.Timestamp, frame.NameString(), frame.URIString(), response.Body)

		sdec := expfmt.SampleDecoder{
			Dec: expfmt.NewDecoder(bytesReader, expfmt.FmtText),
			Opts: &expfmt.DecodeOptions{
				Timestamp: model.TimeFromUnix(frame.Header.Timestamp),
			},
		}

		decSamples := make(model.Vector, 0, 1)

		if err := sdec.Decode(&decSamples); err != nil {
			log.Errorln("Could not decode metric:", err)
			continue
		}

		log.Debugln("Ingested", len(decSamples), "metrics")

		for sampleAppender.NeedsThrottling() {
			log.Debugln("Waiting 100ms for appender to be ready for more data")
			time.Sleep(time.Millisecond * 100)
		}

		var (
			numOutOfOrder = 0
			numDuplicates = 0
		)

		for _, s := range model.Samples(decSamples) {
			if err := sampleAppender.Append(s); err != nil {
				switch err {
				case local.ErrOutOfOrderSample:
					numOutOfOrder++
					log.With("sample", s).With("error", err).Info("Sample discarded")
				case local.ErrDuplicateSampleForTimestamp:
					numDuplicates++
					log.With("sample", s).With("error", err).Info("Sample discarded")
				default:
					log.With("sample", s).With("error", err).Info("Sample discarded")
				}
			}
		}
	}
	// Generate the prometheus.yml in case it does not exist
	promcfgpath := cfgMemoryStorage.PersistenceStoragePath + "/../prometheus.yml"
	if _, err := os.Stat(promcfgpath); os.IsNotExist(err) && !*nopromcfg {
		if err = ioutil.WriteFile(promcfgpath, []byte("global: {}"), os.ModeExclusive|0644); err != nil {
			logrus.Error(err)
		}
	}

	logrus.Info("Exiting! :)")
}
<|endoftext|>


<commit_before>package blockchain

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"io"
	"math/big"
)

const (
	// CoordLen is the length in bytes of coordinates with our ECC curve.
	CoordLen = 32
	// AddrLen is the length in bytes of addresses.
	AddrLen = 2 * CoordLen
	// SigLen is the length in bytes of signatures.
	SigLen = AddrLen
)

var (
	// The curve we use for our ECC crypto.
	curve = elliptic.P256()
	// NilSig is a signature representing a failed Sign operation
	NilSig = Signature{big.NewInt(0), big.NewInt(0)}
	// NilAddr is an address representing no address
	NilAddr = Address{}
)

// Address represents a wallet that can be a recipient in a transaction.
type Address [AddrLen]byte

// Marshal converts an Address to a byte slice.
func (a Address) Marshal() []byte {
	buf := make([]byte, AddrLen)
	for i, b := range a {
		buf[i] = b
	}
	return buf
}

// Wallet represents a wallet that we have the ability to sign for.
type Wallet interface {
	Public() Address
	Sign(digest Hash, random io.Reader) (Signature, error)
}

// Internal representation of a wallet.
type wallet ecdsa.PrivateKey

// Key retreives the underlying private key from a wallet.
func (w *wallet) key() *ecdsa.PrivateKey {
	return (*ecdsa.PrivateKey)(w)
}

// Public returns the public key as byte array, or address, of the wallet.
func (w *wallet) Public() Address {
	addr := Address{}
	x := w.PublicKey.X.Bytes()
	y := w.PublicKey.Y.Bytes()

	if len(x) != CoordLen || len(y) != CoordLen {
		// Invalid wallet
		return NilAddr
	}

	for i, b := range x {
		addr[i] = b
	}
	for i, b := range y {
		addr[CoordLen+i] = b
	}

	return addr
}

// Sign returns a signature of the digest.
func (w *wallet) Sign(digest Hash, random io.Reader) (Signature, error) {
	r, s, err := ecdsa.Sign(random, w.key(), digest.Marshal())
	return Signature{r, s}, err
}

// Signature represents a signature of a transaction.
type Signature struct {
	R *big.Int
	S *big.Int
}

// Marshal converts a signature to a byte slice. Should be 64 bytes long.
func (s *Signature) Marshal() []byte {
	buf := make([]byte, 0, SigLen)
	buf = append(buf, s.R.Bytes()...)
	buf = append(buf, s.S.Bytes()...)
	return buf
}
<commit_msg>Update wallet (#37)<commit_after>package blockchain

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"io"
	"math/big"
)

const (
	// CoordLen is the length in bytes of coordinates with our ECC curve.
	CoordLen = 32
	// AddrLen is the length in bytes of addresses.
	AddrLen = 2 * CoordLen
	// SigLen is the length in bytes of signatures.
	SigLen = AddrLen
)

var (
	// The curve we use for our ECC crypto.
	curve = elliptic.P256()
	// NilSig is a signature representing a failed Sign operation
	NilSig = Signature{big.NewInt(0), big.NewInt(0)}
	// NilAddr is an address representing no address
	NilAddr = Address{}
)

// Address represents a wallet that can be a recipient in a transaction.
type Address struct {
	X, Y *big.Int
}

// Marshal converts an Address to a byte slice.
func (a Address) Marshal() []byte {
	buf := make([]byte, AddrLen)
	buf = append(buf, a.X.Bytes()...)
	buf = append(buf, a.Y.Bytes()...)
	return buf
}

// Key returns the ECDSA public key representation of the address.
func (a Address) Key() *ecdsa.PublicKey {
	return &ecdsa.PublicKey{
		Curve: curve,
		X:     a.X,
		Y:     a.X,
	}
}

// Wallet represents a wallet that we have the ability to sign for.
type Wallet interface {
	Public() Address
	Sign(digest Hash, random io.Reader) (Signature, error)
}

// Internal representation of a wallet.
type wallet ecdsa.PrivateKey

// Key retreives the underlying private key from a wallet.
func (w *wallet) key() *ecdsa.PrivateKey {
	return (*ecdsa.PrivateKey)(w)
}

// Public returns the public key as byte array, or address, of the wallet.
func (w *wallet) Public() Address {
	return Address{X: w.PublicKey.X, Y: w.PublicKey.Y}
}

// Sign returns a signature of the digest.
func (w *wallet) Sign(digest Hash, random io.Reader) (Signature, error) {
	r, s, err := ecdsa.Sign(random, w.key(), digest.Marshal())
	return Signature{r, s}, err
}

// Signature represents a signature of a transaction.
type Signature struct {
	R *big.Int
	S *big.Int
}

// Marshal converts a signature to a byte slice. Should be 64 bytes long.
func (s *Signature) Marshal() []byte {
	buf := make([]byte, 0, SigLen)
	buf = append(buf, s.R.Bytes()...)
	buf = append(buf, s.S.Bytes()...)
	return buf
}
<|endoftext|>