<commit_before>package torrent

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/anacrolix/missinggo"

	"github.com/anacrolix/torrent/tracker"
)

// Announces a torrent to a tracker at regular intervals, when peers are
// required.
type trackerScraper struct {
	url string
	// Causes the trackerScraper to stop running.
	stop         missinggo.Event
	t            *Torrent
	lastAnnounce trackerAnnounceResult
}

func (ts *trackerScraper) statusLine() string {
	var w bytes.Buffer
	fmt.Fprintf(&w, "%q\t%s\t%s",
		ts.url,
		func() string {
			// return ts.lastAnnounce.Completed.Add(ts.lastAnnounce.Interval).Format("2006-01-02 15:04:05 -0700 MST")
			na := ts.lastAnnounce.Completed.Add(ts.lastAnnounce.Interval).Sub(time.Now())
			na /= time.Second
			na *= time.Second
			if na > 0 {
				return na.String()
			} else {
				return "anytime"
			}
		}(),
		func() string {
			if ts.lastAnnounce.Err != nil {
				return ts.lastAnnounce.Err.Error()
			}
			if ts.lastAnnounce.Completed.IsZero() {
				return "never"
			}
			return fmt.Sprintf("%d peers", ts.lastAnnounce.NumPeers)
		}())
	return w.String()
}

type trackerAnnounceResult struct {
	Err       error
	NumPeers  int
	Interval  time.Duration
	Completed time.Time
}

func trackerToTorrentPeers(ps []tracker.Peer) (ret []Peer) {
	ret = make([]Peer, 0, len(ps))
	for _, p := range ps {
		ret = append(ret, Peer{
			IP:     p.IP,
			Port:   p.Port,
			Source: peerSourceTracker,
		})
	}
	return
}

// Return how long to wait before trying again. For most errors, we return 5
// minutes, a relatively quick turn around for DNS changes.
func (me *trackerScraper) announce() (ret trackerAnnounceResult) {
	defer func() {
		ret.Completed = time.Now()
	}()
	ret.Interval = 5 * time.Minute
	blocked, urlToUse, host, err := me.t.cl.prepareTrackerAnnounceUnlocked(me.url)
	if err != nil {
		ret.Err = err
		return
	}
	if blocked {
		ret.Err = errors.New("blocked by IP")
		return
	}
	me.t.cl.mu.Lock()
	req := me.t.announceRequest()
	me.t.cl.mu.Unlock()
	res, err := tracker.AnnounceHost(urlToUse, &req, host)
	if err != nil {
		ret.Err = err
		return
	}
	me.t.AddPeers(trackerToTorrentPeers(res.Peers))
	ret.NumPeers = len(res.Peers)
	ret.Interval = time.Duration(res.Interval) * time.Second
	return
}

func (me *trackerScraper) Run() {
	for {
		select {
		case <-me.t.closed.LockedChan(&me.t.cl.mu):
			return
		case <-me.stop.LockedChan(&me.t.cl.mu):
			return
		case <-me.t.wantPeersEvent.LockedChan(&me.t.cl.mu):
		}

		ar := me.announce()
		me.t.cl.mu.Lock()
		me.lastAnnounce = ar
		me.t.cl.mu.Unlock()

		intervalChan := time.After(ar.Completed.Add(ar.Interval).Sub(time.Now()))

		select {
		case <-me.t.closed.LockedChan(&me.t.cl.mu):
			return
		case <-me.stop.LockedChan(&me.t.cl.mu):
			return
		case <-intervalChan:
		}
	}
}
<commit_msg>Tidy the next announce tracker status field<commit_after>package torrent

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/anacrolix/missinggo"

	"github.com/anacrolix/torrent/tracker"
)

// Announces a torrent to a tracker at regular intervals, when peers are
// required.
type trackerScraper struct {
	url string
	// Causes the trackerScraper to stop running.
	stop         missinggo.Event
	t            *Torrent
	lastAnnounce trackerAnnounceResult
}

func (ts *trackerScraper) statusLine() string {
	var w bytes.Buffer
	fmt.Fprintf(&w, "%q\t%s\t%s",
		ts.url,
		func() string {
			na := time.Until(ts.lastAnnounce.Completed.Add(ts.lastAnnounce.Interval))
			if na > 0 {
				na /= time.Second
				na *= time.Second
				return na.String()
			} else {
				return "anytime"
			}
		}(),
		func() string {
			if ts.lastAnnounce.Err != nil {
				return ts.lastAnnounce.Err.Error()
			}
			if ts.lastAnnounce.Completed.IsZero() {
				return "never"
			}
			return fmt.Sprintf("%d peers", ts.lastAnnounce.NumPeers)
		}())
	return w.String()
}

type trackerAnnounceResult struct {
	Err       error
	NumPeers  int
	Interval  time.Duration
	Completed time.Time
}

func trackerToTorrentPeers(ps []tracker.Peer) (ret []Peer) {
	ret = make([]Peer, 0, len(ps))
	for _, p := range ps {
		ret = append(ret, Peer{
			IP:     p.IP,
			Port:   p.Port,
			Source: peerSourceTracker,
		})
	}
	return
}

// Return how long to wait before trying again. For most errors, we return 5
// minutes, a relatively quick turn around for DNS changes.
func (me *trackerScraper) announce() (ret trackerAnnounceResult) {
	defer func() {
		ret.Completed = time.Now()
	}()
	ret.Interval = 5 * time.Minute
	blocked, urlToUse, host, err := me.t.cl.prepareTrackerAnnounceUnlocked(me.url)
	if err != nil {
		ret.Err = err
		return
	}
	if blocked {
		ret.Err = errors.New("blocked by IP")
		return
	}
	me.t.cl.mu.Lock()
	req := me.t.announceRequest()
	me.t.cl.mu.Unlock()
	res, err := tracker.AnnounceHost(urlToUse, &req, host)
	if err != nil {
		ret.Err = err
		return
	}
	me.t.AddPeers(trackerToTorrentPeers(res.Peers))
	ret.NumPeers = len(res.Peers)
	ret.Interval = time.Duration(res.Interval) * time.Second
	return
}

func (me *trackerScraper) Run() {
	for {
		select {
		case <-me.t.closed.LockedChan(&me.t.cl.mu):
			return
		case <-me.stop.LockedChan(&me.t.cl.mu):
			return
		case <-me.t.wantPeersEvent.LockedChan(&me.t.cl.mu):
		}

		ar := me.announce()
		me.t.cl.mu.Lock()
		me.lastAnnounce = ar
		me.t.cl.mu.Unlock()

		intervalChan := time.After(ar.Completed.Add(ar.Interval).Sub(time.Now()))

		select {
		case <-me.t.closed.LockedChan(&me.t.cl.mu):
			return
		case <-me.stop.LockedChan(&me.t.cl.mu):
			return
		case <-intervalChan:
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/go-martini/martini"
	"log"
	"net/http"
)

func main() {
	m := martini.Classic()
	//中间件处理器是工作于请求和路由之间的. 本质上来说和Martini其他的处理器没有分别.
	/*
		你可以通过Handlers函数对中间件堆有完全的控制. 它将会替换掉之前的任何设置过的处理器
		m.Handlers(
		  Middleware1,
		  Middleware2,
		  Middleware3,
		)
		中间件处理器可以非常好处理一些功能，像logging(日志), authorization(授权), authentication(认证), sessions(会话), error pages(错误页面),
		以及任何其他的操作需要在http请求发生之前或者之后的:
	*/
	// 验证api密匙
	// m.Use(func(res http.ResponseWriter, req *http.Request) {
	// 	if req.Header.Get("X-API-KEY") != "secret123" {
	// 		res.WriteHeader(http.StatusUnauthorized)
	// 	}
	// })

	//Context.Next()是一个可选的函数用于中间件处理器暂时放弃执行直到其他的处理器都执行完毕. 这样就可以很好的处理在http请求完成后需要做的操作.
	// log 记录请求完成前后  (*译者注: 很巧妙，掌声鼓励.)
	m.Use(func(c martini.Context, log *log.Logger) {
		log.Println("before a request")

		c.Next()

		log.Println("after a request")
	})

	//路由匹配的顺序是按照他们被定义的顺序执行的. 最先被定义的路由将会首先被用户请求匹配并调用
	m.Get("/", func() (int, string) {
		return 418, "I'm a teaport" //状态码418
	})

	m.Patch("/", func() {
		// 更新
	})

	m.Post("/", func() {
		// 创建
	})

	m.Put("/", func() {
		// 替换
	})

	m.Delete("/", func() {
		// 删除
	})

	m.Options("/", func() {
		// http 选项
	})

	m.NotFound(func() {
		// 处理 404
	})

	//路由模型可能包含参数列表, 可以通过martini.Params服务来获取
	m.Get("/hello/:name", func(params martini.Params) string {
		return "Hello " + params["name"]
	})

	//路由匹配可以通过正则表达式或者glob的形式
	m.Get("/hello2/**", func(params martini.Params) string {
		log.Println("hello2")
		return "Hello " + params["_1"]
	})
	//m.Run()
	//修改端口
	http.ListenAndServe(":8080", m)
}
<commit_msg>martini整合到现有服务当中<commit_after>package main

import (
	"github.com/go-martini/martini"
	"log"
	"net/http"
)

func main() {
	m := martini.Classic()
	//中间件处理器是工作于请求和路由之间的. 本质上来说和Martini其他的处理器没有分别.
	/*
		你可以通过Handlers函数对中间件堆有完全的控制. 它将会替换掉之前的任何设置过的处理器
		m.Handlers(
		  Middleware1,
		  Middleware2,
		  Middleware3,
		)
		中间件处理器可以非常好处理一些功能，像logging(日志), authorization(授权), authentication(认证), sessions(会话), error pages(错误页面),
		以及任何其他的操作需要在http请求发生之前或者之后的:
	*/
	// 验证api密匙
	// m.Use(func(res http.ResponseWriter, req *http.Request) {
	// 	if req.Header.Get("X-API-KEY") != "secret123" {
	// 		res.WriteHeader(http.StatusUnauthorized)
	// 	}
	// })

	//Context.Next()是一个可选的函数用于中间件处理器暂时放弃执行直到其他的处理器都执行完毕. 这样就可以很好的处理在http请求完成后需要做的操作.
	// log 记录请求完成前后  (*译者注: 很巧妙，掌声鼓励.)
	m.Use(func(c martini.Context, log *log.Logger) {
		log.Println("before a request")

		c.Next()

		log.Println("after a request")
	})

	//路由匹配的顺序是按照他们被定义的顺序执行的. 最先被定义的路由将会首先被用户请求匹配并调用
	m.Get("/", func() (int, string) {
		return 418, "I'm a teaport" //状态码418
	})

	m.Patch("/", func() {
		// 更新
	})

	m.Post("/", func() {
		// 创建
	})

	m.Put("/", func() {
		// 替换
	})

	m.Delete("/", func() {
		// 删除
	})

	m.Options("/", func() {
		// http 选项
	})

	m.NotFound(func() {
		// 处理 404
	})

	//路由模型可能包含参数列表, 可以通过martini.Params服务来获取
	m.Get("/hello/:name", func(params martini.Params) string {
		return "Hello " + params["name"]
	})

	//路由匹配可以通过正则表达式或者glob的形式
	m.Get("/hello2/**", func(params martini.Params) string {
		log.Println("hello2")
		return "Hello " + params["_1"]
	})

	//martini整合到现有服务当中
	http.Handle("/", m)
	//m.Run()
	//修改端口
	http.ListenAndServe(":8080", m)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"errors"
	"fmt"
	"mime"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/mux"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/zzk"
	"github.com/control-center/serviced/zzk/registry"
	"github.com/control-center/serviced/zzk/service"
	"github.com/zenoss/glog"
)

type StoppableListener struct {
	*net.TCPListener
	stopChan chan bool
}

func NewStoppableListener(listener net.Listener) *StoppableListener {
	return &StoppableListener{listener.(*net.TCPListener), make(chan bool)}
}

func (sl StoppableListener) Accept() (c net.Conn, err error) {
	for {
		sl.SetDeadline(time.Now().Add(time.Second))
		select {
		case <-sl.stopChan:
			return nil, StoppedListenerError{errors.New("Port Closed")}
		default:
		}

		tc, err := sl.TCPListener.AcceptTCP()
		if err != nil {
			switch err := err.(type) {
			case net.Error:
				if err.Timeout() {
					continue
				}
			}
			return nil, err
		}

		tc.SetKeepAlive(true)
		tc.SetKeepAlivePeriod(3 * time.Minute)
		return tc, nil
	}
}

func (sl StoppableListener) Stop() {
	close(sl.stopChan)
}

type StoppedListenerError struct {
	error
}

func (sle StoppedListenerError) Timeout() bool   { return false }
func (sle StoppedListenerError) Temporary() bool { return false }

var (
	allportsLock sync.RWMutex
	allports     map[string]chan int // map of port number to channel that destroys the server
)

func init() {
	allports = make(map[string]chan int)
}

func (sc *ServiceConfig) ServePublicPorts(shutdown <-chan (interface{})) {
	go sc.syncAllPublicPorts(shutdown)
}

func (sc *ServiceConfig) CreatePublicPortServer(publicEndpointKey service.PublicEndpointKey, stopChan <-chan int, shutdown <-chan (interface{})) {
	mime.AddExtensionType(".json", "application/json")
	mime.AddExtensionType(".woff", "application/font-woff")

	httphandler := func(w http.ResponseWriter, r *http.Request) {
		glog.V(2).Infof("httphandler handling request: %+v", r)
		svcIds := make(map[string]struct{})
		var emptyStruct struct{}
		svcIds[publicEndpointKey.ServiceID()] = emptyStruct
		registryKey := registry.GetPublicEndpointKey(publicEndpointKey.Name(), publicEndpointKey.Type())
		sc.publicendpointhandler(w, r, registryKey, svcIds)
	}

	r := mux.NewRouter()
	r.HandleFunc("/", httphandler)
	r.HandleFunc("/{path:.*}", httphandler)

	go func() {
		port := fmt.Sprintf(":%s", publicEndpointKey.Name())
		server := &http.Server{Addr: port, Handler: r}
		listener, err := net.Listen("tcp", server.Addr)
		stoppableListener := NewStoppableListener(listener)
		if err != nil {
			glog.Errorf("Could not setup HTTP webserver on port %s: %s", port, err)
		}

		glog.Infof("Listening on port %s", port)

		go func() {
			server.Serve(stoppableListener)
		}()

		<-stopChan
		listener.Close()
		stoppableListener.Stop()
		glog.Infof("Closed port %s", port)
		return
	}()
}

func (sc *ServiceConfig) syncAllPublicPorts(shutdown <-chan interface{}) error {
	rootConn, err := zzk.GetLocalConnection("/")
	if err != nil {
		glog.Errorf("syncAllPublicPorts - Error getting root zk connection: %v", err)
		return err
	}

	cancelChan := make(chan interface{})
	syncPorts := func(conn client.Connection, parentPath string, childIDs ...string) {
		glog.V(1).Infof("syncPorts STARTING for parentPath:%s childIDs:%v", parentPath, childIDs)

		// start all servers that have been not started and enabled
		newPorts := make(map[string]chan int)
		for _, sv := range childIDs {
			publicEndpointKey := service.PublicEndpointKey(sv)
			if publicEndpointKey.Type() == registry.EPTypePort && publicEndpointKey.IsEnabled() {
				port := publicEndpointKey.Name()
				stopChan, running := allports[port]

				if !running {
					// recently enabled port - port should be opened
					stopChan = make(chan int)
					sc.CreatePublicPortServer(publicEndpointKey, stopChan, shutdown)
				}

				newPorts[port] = stopChan
			}
		}

		// stop all servers that have been deleted or disabled
		for port, stopChan := range allports {
			_, found := newPorts[port]
			if !found {
				stopChan <- 0
			}
		}

		//lock for as short a time as possible
		allportsLock.Lock()
		defer allportsLock.Unlock()
		allports = newPorts
		glog.V(1).Infof("allports: %+v", allports)
	}

	for {
		zkServicePEPService := service.ZKServicePublicEndpoints
		glog.V(1).Infof("Running registry.WatchChildren for zookeeper path: %s", zkServicePEPService)
		err := registry.WatchChildren(rootConn, zkServicePEPService, cancelChan, syncPorts, pepWatchError)
		if err != nil {
			glog.V(1).Infof("Will retry in 10 seconds to WatchChildren(%s) due to error: %v", zkServicePEPService, err)
			<-time.After(time.Second * 10)
			continue
		}
		select {
		case <-shutdown:
			close(cancelChan)
			return nil
		default:
		}
	}
}
<commit_msg>remove stoppable listener code.  we will use a basic listener and session timeouts to handle connection management.<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"fmt"
	"mime"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/mux"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/zzk"
	"github.com/control-center/serviced/zzk/registry"
	"github.com/control-center/serviced/zzk/service"
	"github.com/zenoss/glog"
)

var (
	allportsLock sync.RWMutex
	allports     map[string]chan int // map of port number to channel that destroys the server
)

func init() {
	allports = make(map[string]chan int)
}

func (sc *ServiceConfig) ServePublicPorts(shutdown <-chan (interface{})) {
	go sc.syncAllPublicPorts(shutdown)
}

func (sc *ServiceConfig) CreatePublicPortServer(publicEndpointKey service.PublicEndpointKey, stopChan <-chan int, shutdown <-chan (interface{})) {
	mime.AddExtensionType(".json", "application/json")
	mime.AddExtensionType(".woff", "application/font-woff")

	httphandler := func(w http.ResponseWriter, r *http.Request) {
		glog.V(2).Infof("httphandler handling request: %+v", r)
		svcIds := make(map[string]struct{})
		var emptyStruct struct{}
		svcIds[publicEndpointKey.ServiceID()] = emptyStruct
		registryKey := registry.GetPublicEndpointKey(publicEndpointKey.Name(), publicEndpointKey.Type())
		sc.publicendpointhandler(w, r, registryKey, svcIds)
	}

	r := mux.NewRouter()
	r.HandleFunc("/", httphandler)
	r.HandleFunc("/{path:.*}", httphandler)

	go func() {
		port := fmt.Sprintf(":%s", publicEndpointKey.Name())
		server := &http.Server{Addr: port, Handler: r}
		listener, err := net.Listen("tcp", server.Addr)
		if err != nil {
			glog.Errorf("Could not setup HTTP webserver on port %s: %s", port, err)
		}

		glog.Infof("Listening on port %s", port)

		go func() {
			server.Serve(listener)
		}()

		<-stopChan
		listener.Close()
		glog.Infof("Closed port %s", port)
		return
	}()
}

func (sc *ServiceConfig) syncAllPublicPorts(shutdown <-chan interface{}) error {
	rootConn, err := zzk.GetLocalConnection("/")
	if err != nil {
		glog.Errorf("syncAllPublicPorts - Error getting root zk connection: %v", err)
		return err
	}

	cancelChan := make(chan interface{})
	syncPorts := func(conn client.Connection, parentPath string, childIDs ...string) {
		glog.V(1).Infof("syncPorts STARTING for parentPath:%s childIDs:%v", parentPath, childIDs)

		// start all servers that have been not started and enabled
		newPorts := make(map[string]chan int)
		for _, sv := range childIDs {
			publicEndpointKey := service.PublicEndpointKey(sv)
			if publicEndpointKey.Type() == registry.EPTypePort && publicEndpointKey.IsEnabled() {
				port := publicEndpointKey.Name()
				stopChan, running := allports[port]

				if !running {
					// recently enabled port - port should be opened
					stopChan = make(chan int)
					sc.CreatePublicPortServer(publicEndpointKey, stopChan, shutdown)
				}

				newPorts[port] = stopChan
			}
		}

		// stop all servers that have been deleted or disabled
		for port, stopChan := range allports {
			_, found := newPorts[port]
			if !found {
				stopChan <- 0
			}
		}

		//lock for as short a time as possible
		allportsLock.Lock()
		defer allportsLock.Unlock()
		allports = newPorts
		glog.V(1).Infof("allports: %+v", allports)
	}

	for {
		zkServicePEPService := service.ZKServicePublicEndpoints
		glog.V(1).Infof("Running registry.WatchChildren for zookeeper path: %s", zkServicePEPService)
		err := registry.WatchChildren(rootConn, zkServicePEPService, cancelChan, syncPorts, pepWatchError)
		if err != nil {
			glog.V(1).Infof("Will retry in 10 seconds to WatchChildren(%s) due to error: %v", zkServicePEPService, err)
			<-time.After(time.Second * 10)
			continue
		}
		select {
		case <-shutdown:
			close(cancelChan)
			return nil
		default:
		}
	}
}
<|endoftext|>


<commit_before>package web

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"testing"
	"time"
)

// These tests can probably be DRY'd up a bunch

func makeRouter() *router {
	return &router{
		routes:   make([]route, 0),
		notFound: parseHandler(http.NotFound),
	}
}

func chHandler(ch chan string, s string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- s
	})
}

var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH",
	"POST", "PUT", "TRACE", "OTHER"}

func TestMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Connect("/", chHandler(ch, "CONNECT"))
	rt.Delete("/", chHandler(ch, "DELETE"))
	rt.Get("/", chHandler(ch, "GET"))
	rt.Head("/", chHandler(ch, "HEAD"))
	rt.Options("/", chHandler(ch, "OPTIONS"))
	rt.Patch("/", chHandler(ch, "PATCH"))
	rt.Post("/", chHandler(ch, "POST"))
	rt.Put("/", chHandler(ch, "PUT"))
	rt.Trace("/", chHandler(ch, "TRACE"))
	rt.Handle("/", chHandler(ch, "OTHER"))

	for _, method := range methods {
		r, _ := http.NewRequest(method, "/", nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case val := <-ch:
			if val != method {
				t.Error("Got %q, expected %q", val, method)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for method %q", method)
		}
	}
}

type testPattern struct{}

func (t testPattern) Prefix() string {
	return ""
}

func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool {
	return true
}

var _ Pattern = testPattern{}

func TestPatternTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	rt.Get("/hello/carl", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Get(regexp.MustCompile(`^/hello/(?P<name>.+)$`), http.NotFound)
	rt.Get(testPattern{}, http.NotFound)
}

type testHandler chan string

func (t testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t <- "http"
}
func (t testHandler) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	t <- "httpc"
}

var testHandlerTable = map[string]string{
	"/a": "http fn",
	"/b": "http handler",
	"/c": "web fn",
	"/d": "web handler",
	"/e": "httpc",
}

func TestHandlerTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Get("/a", func(w http.ResponseWriter, r *http.Request) {
		ch <- "http fn"
	})
	rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- "http handler"
	}))
	rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web fn"
	})
	rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web handler"
	}))
	rt.Get("/e", testHandler(ch))

	for route, response := range testHandlerTable {
		r, _ := http.NewRequest("gEt", route, nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case resp := <-ch:
			if resp != response {
				t.Errorf("Got %q, expected %q", resp, response)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for path %q", route)
		}

	}
}

func TestNotFound(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	r, _ := http.NewRequest("post", "/", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != 404 {
		t.Errorf("Expected 404, got %d", w.Code)
	}

	rt.NotFound(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "I'm a teapot!", http.StatusTeapot)
	})

	r, _ = http.NewRequest("post", "/", nil)
	w = httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != http.StatusTeapot {
		t.Errorf("Expected a teapot, got %d", w.Code)
	}
}

func TestSub(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Sub("/hello", func(w http.ResponseWriter, r *http.Request) {
		ch <- r.URL.Path
	})

	r, _ := http.NewRequest("GET", "/hello/world", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	select {
	case val := <-ch:
		if val != "/hello/world" {
			t.Error("Got %q, expected /hello/world", val)
		}
	case <-time.After(5 * time.Millisecond):
		t.Errorf("Timeout waiting for hello")
	}
}

var validMethodsTable = map[string][]string{
	"/hello/carl":       {"DELETE", "GET", "PATCH", "POST", "PUT"},
	"/hello/bob":        {"DELETE", "GET", "HEAD", "PATCH", "PUT"},
	"/hola/carl":        {"DELETE", "GET", "PUT"},
	"/hola/bob":         {"DELETE"},
	"/does/not/compute": {},
}

func TestValidMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan []string, 1)

	rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			ch <- []string{}
			return
		}
		methods, ok := c.Env[validMethods]
		if !ok {
			ch <- []string{}
			return
		}
		ch <- methods.([]string)
	})

	rt.Get("/hello/carl", http.NotFound)
	rt.Post("/hello/carl", http.NotFound)
	rt.Head("/hello/bob", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Put("/hello/:name", http.NotFound)
	rt.Patch("/hello/:name", http.NotFound)
	rt.Get("/:greet/carl", http.NotFound)
	rt.Put("/:greet/carl", http.NotFound)
	rt.Delete("/:greet/:anyone", http.NotFound)

	for path, eMethods := range validMethodsTable {
		r, _ := http.NewRequest("BOGUS", path, nil)
		rt.route(C{}, httptest.NewRecorder(), r)
		aMethods := <-ch
		if !reflect.DeepEqual(eMethods, aMethods) {
			t.Errorf("For %q, expected %v, got %v", path, eMethods,
				aMethods)
		}
	}
}
<commit_msg>Fix router tests<commit_after>package web

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"regexp"
	"testing"
	"time"
)

// These tests can probably be DRY'd up a bunch

func makeRouter() *router {
	return &router{
		routes:   make([]route, 0),
		notFound: parseHandler(http.NotFound),
	}
}

func chHandler(ch chan string, s string) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- s
	})
}

var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH",
	"POST", "PUT", "TRACE", "OTHER"}

func TestMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Connect("/", chHandler(ch, "CONNECT"))
	rt.Delete("/", chHandler(ch, "DELETE"))
	rt.Head("/", chHandler(ch, "HEAD"))
	rt.Get("/", chHandler(ch, "GET"))
	rt.Options("/", chHandler(ch, "OPTIONS"))
	rt.Patch("/", chHandler(ch, "PATCH"))
	rt.Post("/", chHandler(ch, "POST"))
	rt.Put("/", chHandler(ch, "PUT"))
	rt.Trace("/", chHandler(ch, "TRACE"))
	rt.Handle("/", chHandler(ch, "OTHER"))

	for _, method := range methods {
		r, _ := http.NewRequest(method, "/", nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case val := <-ch:
			if val != method {
				t.Errorf("Got %q, expected %q", val, method)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for method %q", method)
		}
	}
}

type testPattern struct{}

func (t testPattern) Prefix() string {
	return ""
}

func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool {
	return true
}

var _ Pattern = testPattern{}

func TestPatternTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	rt.Get("/hello/carl", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Get(regexp.MustCompile(`^/hello/(?P<name>.+)$`), http.NotFound)
	rt.Get(testPattern{}, http.NotFound)
}

type testHandler chan string

func (t testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	t <- "http"
}
func (t testHandler) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {
	t <- "httpc"
}

var testHandlerTable = map[string]string{
	"/a": "http fn",
	"/b": "http handler",
	"/c": "web fn",
	"/d": "web handler",
	"/e": "httpc",
}

func TestHandlerTypes(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Get("/a", func(w http.ResponseWriter, r *http.Request) {
		ch <- "http fn"
	})
	rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ch <- "http handler"
	}))
	rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web fn"
	})
	rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) {
		ch <- "web handler"
	}))
	rt.Get("/e", testHandler(ch))

	for route, response := range testHandlerTable {
		r, _ := http.NewRequest("GET", route, nil)
		w := httptest.NewRecorder()
		rt.route(C{}, w, r)
		select {
		case resp := <-ch:
			if resp != response {
				t.Errorf("Got %q, expected %q", resp, response)
			}
		case <-time.After(5 * time.Millisecond):
			t.Errorf("Timeout waiting for path %q", route)
		}

	}
}

func TestNotFound(t *testing.T) {
	t.Parallel()
	rt := makeRouter()

	r, _ := http.NewRequest("post", "/", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != 404 {
		t.Errorf("Expected 404, got %d", w.Code)
	}

	rt.NotFound(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "I'm a teapot!", http.StatusTeapot)
	})

	r, _ = http.NewRequest("POST", "/", nil)
	w = httptest.NewRecorder()
	rt.route(C{}, w, r)
	if w.Code != http.StatusTeapot {
		t.Errorf("Expected a teapot, got %d", w.Code)
	}
}

func TestSub(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan string, 1)

	rt.Sub("/hello", func(w http.ResponseWriter, r *http.Request) {
		ch <- r.URL.Path
	})

	r, _ := http.NewRequest("GET", "/hello/world", nil)
	w := httptest.NewRecorder()
	rt.route(C{}, w, r)
	select {
	case val := <-ch:
		if val != "/hello/world" {
			t.Error("Got %q, expected /hello/world", val)
		}
	case <-time.After(5 * time.Millisecond):
		t.Errorf("Timeout waiting for hello")
	}
}

var validMethodsTable = map[string][]string{
	"/hello/carl":       {"DELETE", "GET", "HEAD", "PATCH", "POST", "PUT"},
	"/hello/bob":        {"DELETE", "GET", "HEAD", "PATCH", "PUT"},
	"/hola/carl":        {"DELETE", "GET", "HEAD", "PUT"},
	"/hola/bob":         {"DELETE"},
	"/does/not/compute": {},
}

func TestValidMethods(t *testing.T) {
	t.Parallel()
	rt := makeRouter()
	ch := make(chan []string, 1)

	rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) {
		if c.Env == nil {
			ch <- []string{}
			return
		}
		methods, ok := c.Env[ValidMethodsKey]
		if !ok {
			ch <- []string{}
			return
		}
		ch <- methods.([]string)
	})

	rt.Get("/hello/carl", http.NotFound)
	rt.Post("/hello/carl", http.NotFound)
	rt.Head("/hello/bob", http.NotFound)
	rt.Get("/hello/:name", http.NotFound)
	rt.Put("/hello/:name", http.NotFound)
	rt.Patch("/hello/:name", http.NotFound)
	rt.Get("/:greet/carl", http.NotFound)
	rt.Put("/:greet/carl", http.NotFound)
	rt.Delete("/:greet/:anyone", http.NotFound)

	for path, eMethods := range validMethodsTable {
		r, _ := http.NewRequest("BOGUS", path, nil)
		rt.route(C{}, httptest.NewRecorder(), r)
		aMethods := <-ch
		if !reflect.DeepEqual(eMethods, aMethods) {
			t.Errorf("For %q, expected %v, got %v", path, eMethods,
				aMethods)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	// "encoding/hex"
	"fmt"
	"os/exec"
	"bytes"
	// "time"
	// "math/big"
	// "github.com/PointCoin/btcutil"
	// "github.com/PointCoin/btcwire"
	// "github.com/PointCoin/btcrpcclient"
	// "github.com/PointCoin/btcjson"
	"strconv"
	"strings"
	// "regexp"
	// "math/rand"
	"log"
)

func main() {

	s := "\"Hello\""
	fmt.Println(s)

	o,_ := strconv.Unquote(s)
	fmt.Println(o)

	// command = pointctl getrawtransaction d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548
	txid := "d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548"
	cmd := exec.Command("pointctl", "getrawtransaction", txid)

	var out bytes.Buffer
    cmd.Stdout = &out
    err := cmd.Run()
    if err != nil {
    	log.Fatal(err)
    }
    // fmt.Printf("result: %s\n", out)
    fmt.Println(out.String())

	cmd2 := exec.Command("xargs", "pointctl", "decoderawtransaction")
	cmd2.Stdin = strings.NewReader(out.String())
	var out2 bytes.Buffer
    cmd2.Stdout = &out2
    err2 := cmd2.Run()
    if err2 != nil {
    	log.Fatal(err2)
    }

    fmt.Println(out2)

}<commit_msg>successfully gets transaction info<commit_after>package main

import (
	// "encoding/hex"
	"fmt"
	"os/exec"
	"bytes"
	// "time"
	// "math/big"
	// "github.com/PointCoin/btcutil"
	// "github.com/PointCoin/btcwire"
	// "github.com/PointCoin/btcrpcclient"
	// "github.com/PointCoin/btcjson"
	"strconv"
	"strings"
	// "regexp"
	// "math/rand"
	"log"
)

func main() {

	s := "\"Hello\""
	fmt.Println(s)

	o,_ := strconv.Unquote(s)
	fmt.Println(o)

	// command = pointctl getrawtransaction d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548
	txid := "d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548"
	cmd := exec.Command("pointctl", "getrawtransaction", txid)

	var out bytes.Buffer
    cmd.Stdout = &out
    err := cmd.Run()
    if err != nil {
    	log.Fatal(err)
    }
    // fmt.Printf("result: %s\n", out)
    fmt.Println(out.String())

	cmd2 := exec.Command("xargs", "pointctl", "decoderawtransaction")
	cmd2.Stdin = strings.NewReader(out.String())
	var out2 bytes.Buffer
    cmd2.Stdout = &out2
    err2 := cmd2.Run()
    if err2 != nil {
    	log.Fatal(err2)
    }

    fmt.Println(out2.String())

}<|endoftext|>


<commit_before>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle/logverifier"
	"github.com/google/trillian/merkle/rfc6962/hasher"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/types"
	"github.com/google/trillian/util/clock"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns       monitoring.Counter
	masterCancels    monitoring.Counter
	controllerStarts monitoring.Counter
	isMaster         monitoring.Gauge
	entriesFetched   monitoring.Counter
	entriesSeen      monitoring.Counter
	entriesStored    monitoring.Counter
	sthTimestamp     monitoring.Gauge
	sthTreeSize      monitoring.Gauge
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:       mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:    mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			controllerStarts: mf.NewCounter("controller_starts", "Number of Controller (re-)starts.", treeID),
			isMaster:         mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched:   mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:      mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:    mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
			sthTimestamp:     mf.NewGauge("sth_timestamp", "Timestamp of the last seen STH.", treeID),
			sthTreeSize:      mf.NewGauge("sth_tree_size", "Tree size of the last seen STH.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
	StartDelay         time.Duration
	StopAfter          time.Duration
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
type Controller struct {
	opts     Options
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.treeID, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMasterWithRestarts calls RunWhenMaster, and, if the migration is
// configured with continuous mode, restarts it whenever it returns.
func (c *Controller) RunWhenMasterWithRestarts(ctx context.Context) {
	uri := c.ctClient.BaseURI()
	treeID := c.plClient.treeID
	for run := true; run; run = c.opts.Continuous && ctx.Err() == nil {
		glog.Infof("Starting migration Controller (%d<-%q)", treeID, uri)
		if err := c.RunWhenMaster(ctx); err != nil {
			glog.Errorf("Controller.RunWhenMaster(%d<-%q): %v", treeID, uri, err)
			continue
		}
		glog.Infof("Controller stopped (%d<-%q)", treeID, uri)
	}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	// Avoid thundering herd when starting multiple tasks on the same tree.
	if err := sleepRandom(ctx, 0, c.opts.StartDelay); err != nil {
		return err // The context has been canceled.
	}

	el, err := c.ef.NewElection(ctx, c.label)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", c.label, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", c.label)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.runWithRestarts(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so try to resign and emit the real error.
			if rerr := el.Resign(ctx); rerr != nil {
				glog.Errorf("%s: Election.Resign(): %v", c.label, rerr)
			}
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// runWithRestarts calls Run until it succeeds or the context is done, in
// continuous mode. For non-continuous mode it is simply equivalent to Run.
func (c *Controller) runWithRestarts(ctx context.Context) error {
	err := c.Run(ctx)
	if !c.opts.Continuous {
		return err
	}
	for err != nil && ctx.Err() == nil {
		glog.Errorf("%s: Controller.Run: %v", c.label, err)
		sleepRandom(ctx, 0, c.opts.StartDelay)
		err = c.Run(ctx)
	}
	return ctx.Err()
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	metrics.controllerStarts.Inc(c.label)
	stopAfter := randDuration(c.opts.StopAfter, c.opts.StopAfter)
	start := time.Now()

	// Note: Non-continuous runs are not affected by StopAfter.
	pos, err := c.fetchTail(ctx, 0)
	if err != nil {
		return err
	}
	if !c.opts.Continuous {
		return nil
	}

	for stopAfter == 0 || time.Since(start) < stopAfter {
		// TODO(pavelkalinnikov): Integrate runWithRestarts here.
		next, err := c.fetchTail(ctx, pos)
		if err != nil {
			return err
		}
		if next == pos {
			// TODO(pavelkalinnikov): Pause with accordance to the rate of growth.
			// TODO(pavelkalinnikov): Make the duration configurable.
			if err := clock.SleepContext(ctx, 30*time.Second); err != nil {
				return err
			}
		}
		pos = next
	}

	return nil
}

// fetchTail transfers entries within the range specified in FetcherConfig,
// with respect to the passed in minimal position to start from, and the
// current tree size obtained from an STH.
func (c *Controller) fetchTail(ctx context.Context, begin uint64) (uint64, error) {
	root, err := c.plClient.getRoot(ctx)
	if err != nil {
		return 0, err
	}

	fo := c.opts.FetcherOptions
	if fo.Continuous { // Ignore range parameters in continuous mode.
		fo.StartIndex, fo.EndIndex = int64(root.TreeSize), 0
		// Use non-continuous Fetcher, as we implement continuity in Controller.
		// TODO(pavelkalinnikov): Don't overload Fetcher's Continuous flag.
		fo.Continuous = false
	} else if fo.StartIndex < 0 {
		fo.StartIndex = int64(root.TreeSize)
	}
	if int64(begin) > fo.StartIndex {
		fo.StartIndex = int64(begin)
	}
	glog.Infof("%s: fetching range [%d, %d)", c.label, fo.StartIndex, fo.EndIndex)

	fetcher := scanner.NewFetcher(c.ctClient, &fo)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return 0, err
	}
	metrics.sthTimestamp.Set(float64(sth.Timestamp), c.label)
	metrics.sthTreeSize.Set(float64(sth.TreeSize), c.label)
	if sth.TreeSize <= begin {
		return begin, nil
	}

	if err := c.verifyConsistency(ctx, root, sth); err != nil {
		return 0, err
	}

	var wg sync.WaitGroup
	batches := make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx, batches); err != nil {
				glog.Errorf("%s: Stopping due to submitter error: %v", c.label, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		select {
		case batches <- b:
		case <-cctx.Done(): // Avoid deadlock when shutting down.
		}
	}

	err = fetcher.Run(cctx, handler)
	close(batches)
	wg.Wait()
	if err != nil {
		return 0, err
	}
	return sth.TreeSize, nil
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, root *types.LogRootV1, sth *ct.SignedTreeHead) error {
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}
	proof, err := c.ctClient.GetSTHConsistency(ctx, root.TreeSize, sth.TreeSize)
	if err != nil {
		return err
	}
	return logverifier.New(hasher.DefaultHasher).VerifyConsistencyProof(
		int64(root.TreeSize), int64(sth.TreeSize),
		root.RootHash, sth.SHA256RootHash[:], proof)
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context, batches <-chan scanner.EntryBatch) error {
	for b := range batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%s: added batch [%d, %d)", c.label, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}

// sleepRandom sleeps for random duration in [base, base+spread).
func sleepRandom(ctx context.Context, base, spread time.Duration) error {
	d := randDuration(base, spread)
	if d == 0 {
		return nil
	}
	return clock.SleepContext(ctx, d)
}

// randDuration returns a random duration in [base, base+spread).
func randDuration(base, spread time.Duration) time.Duration {
	d := base
	if spread != 0 {
		d += time.Duration(rand.Int63n(int64(spread)))
	}
	return d
}
<commit_msg>Avoid using LogRootV1 in verifyConsistency.<commit_after>// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package core provides transport-agnostic implementation of Migrillian tool.
package core

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/trillian/migrillian/configpb"

	"github.com/google/trillian/merkle/logverifier"
	"github.com/google/trillian/merkle/rfc6962/hasher"
	"github.com/google/trillian/monitoring"
	"github.com/google/trillian/util/clock"
	"github.com/google/trillian/util/election2"
)

var (
	metrics     treeMetrics
	metricsOnce sync.Once
)

// treeMetrics holds metrics keyed by Tree ID.
type treeMetrics struct {
	masterRuns       monitoring.Counter
	masterCancels    monitoring.Counter
	controllerStarts monitoring.Counter
	isMaster         monitoring.Gauge
	entriesFetched   monitoring.Counter
	entriesSeen      monitoring.Counter
	entriesStored    monitoring.Counter
	sthTimestamp     monitoring.Gauge
	sthTreeSize      monitoring.Gauge
}

// initMetrics creates metrics using the factory, if not yet created.
func initMetrics(mf monitoring.MetricFactory) {
	const treeID = "tree_id"
	metricsOnce.Do(func() {
		metrics = treeMetrics{
			masterRuns:       mf.NewCounter("master_runs", "Number of mastership runs.", treeID),
			masterCancels:    mf.NewCounter("master_cancels", "Number of unexpected mastership cancelations.", treeID),
			controllerStarts: mf.NewCounter("controller_starts", "Number of Controller (re-)starts.", treeID),
			isMaster:         mf.NewGauge("is_master", "The instance is currently the master.", treeID),
			entriesFetched:   mf.NewCounter("entries_fetched", "Entries fetched from the source log.", treeID),
			entriesSeen:      mf.NewCounter("entries_seen", "Entries seen by the submitters.", treeID),
			entriesStored:    mf.NewCounter("entries_stored", "Entries successfully submitted to Trillian.", treeID),
			sthTimestamp:     mf.NewGauge("sth_timestamp", "Timestamp of the last seen STH.", treeID),
			sthTreeSize:      mf.NewGauge("sth_tree_size", "Tree size of the last seen STH.", treeID),
		}
	})
}

// Options holds configuration for a Controller.
type Options struct {
	scanner.FetcherOptions
	Submitters         int
	ChannelSize        int
	NoConsistencyCheck bool
	StartDelay         time.Duration
	StopAfter          time.Duration
}

// OptionsFromConfig returns Options created from the passed in config.
func OptionsFromConfig(cfg *configpb.MigrationConfig) Options {
	opts := Options{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     int(cfg.BatchSize),
			ParallelFetch: int(cfg.NumFetchers),
			StartIndex:    cfg.StartIndex,
			EndIndex:      cfg.EndIndex,
			Continuous:    cfg.IsContinuous,
		},
		Submitters:         int(cfg.NumSubmitters),
		ChannelSize:        int(cfg.ChannelSize),
		NoConsistencyCheck: cfg.NoConsistencyCheck,
	}
	if cfg.NumFetchers == 0 {
		opts.ParallelFetch = 1
	}
	if cfg.NumSubmitters == 0 {
		opts.Submitters = 1
	}
	return opts
}

// Controller coordinates migration from a CT log to a Trillian tree.
type Controller struct {
	opts     Options
	ctClient *client.LogClient
	plClient *PreorderedLogClient
	ef       election2.Factory
	label    string
}

// NewController creates a Controller configured by the passed in options, CT
// and Trillian clients, and a master election factory.
//
// The passed in MetricFactory is used to create per-tree metrics, and it
// should be the same for all instances. However, it is used only once.
func NewController(
	opts Options,
	ctClient *client.LogClient,
	plClient *PreorderedLogClient,
	ef election2.Factory,
	mf monitoring.MetricFactory,
) *Controller {
	initMetrics(mf)
	l := strconv.FormatInt(plClient.treeID, 10)
	return &Controller{opts: opts, ctClient: ctClient, plClient: plClient, ef: ef, label: l}
}

// RunWhenMasterWithRestarts calls RunWhenMaster, and, if the migration is
// configured with continuous mode, restarts it whenever it returns.
func (c *Controller) RunWhenMasterWithRestarts(ctx context.Context) {
	uri := c.ctClient.BaseURI()
	treeID := c.plClient.treeID
	for run := true; run; run = c.opts.Continuous && ctx.Err() == nil {
		glog.Infof("Starting migration Controller (%d<-%q)", treeID, uri)
		if err := c.RunWhenMaster(ctx); err != nil {
			glog.Errorf("Controller.RunWhenMaster(%d<-%q): %v", treeID, uri, err)
			continue
		}
		glog.Infof("Controller stopped (%d<-%q)", treeID, uri)
	}
}

// RunWhenMaster is a master-elected version of Run method. It executes Run
// whenever this instance captures mastership of the tree ID. As soon as the
// instance stops being the master, Run is canceled. The method returns if a
// severe error occurs, the passed in context is canceled, or fetching is
// completed (in non-Continuous mode). Releases mastership when terminates.
func (c *Controller) RunWhenMaster(ctx context.Context) error {
	// Avoid thundering herd when starting multiple tasks on the same tree.
	if err := sleepRandom(ctx, 0, c.opts.StartDelay); err != nil {
		return err // The context has been canceled.
	}

	el, err := c.ef.NewElection(ctx, c.label)
	if err != nil {
		return err
	}
	defer func(ctx context.Context) {
		metrics.isMaster.Set(0, c.label)
		if err := el.Close(ctx); err != nil {
			glog.Warningf("%s: Election.Close(): %v", c.label, err)
		}
	}(ctx)

	for {
		if err := el.Await(ctx); err != nil {
			return err
		}
		metrics.isMaster.Set(1, c.label)

		mctx, err := el.WithMastership(ctx)
		if err != nil {
			return err
		} else if err := mctx.Err(); err != nil {
			return err
		}

		glog.Infof("%s: running as master", c.label)
		metrics.masterRuns.Inc(c.label)

		// Run while still master (or until an error).
		err = c.runWithRestarts(mctx)
		if ctx.Err() != nil {
			// We have been externally canceled, so return the current error (which
			// could be nil or a cancelation-related error).
			return err
		} else if mctx.Err() == nil {
			// We are still the master, so try to resign and emit the real error.
			if rerr := el.Resign(ctx); rerr != nil {
				glog.Errorf("%s: Election.Resign(): %v", c.label, rerr)
			}
			return err
		}

		// Otherwise the mastership has been canceled, retry.
		metrics.isMaster.Set(0, c.label)
		metrics.masterCancels.Inc(c.label)
	}
}

// runWithRestarts calls Run until it succeeds or the context is done, in
// continuous mode. For non-continuous mode it is simply equivalent to Run.
func (c *Controller) runWithRestarts(ctx context.Context) error {
	err := c.Run(ctx)
	if !c.opts.Continuous {
		return err
	}
	for err != nil && ctx.Err() == nil {
		glog.Errorf("%s: Controller.Run: %v", c.label, err)
		sleepRandom(ctx, 0, c.opts.StartDelay)
		err = c.Run(ctx)
	}
	return ctx.Err()
}

// Run transfers CT log entries obtained via the CT log client to a Trillian
// pre-ordered log via Trillian client. If Options.Continuous is true then the
// migration process runs continuously trying to keep up with the target CT
// log. Returns if an error occurs, the context is canceled, or all the entries
// have been transferred (in non-Continuous mode).
func (c *Controller) Run(ctx context.Context) error {
	metrics.controllerStarts.Inc(c.label)
	stopAfter := randDuration(c.opts.StopAfter, c.opts.StopAfter)
	start := time.Now()

	// Note: Non-continuous runs are not affected by StopAfter.
	pos, err := c.fetchTail(ctx, 0)
	if err != nil {
		return err
	}
	if !c.opts.Continuous {
		return nil
	}

	for stopAfter == 0 || time.Since(start) < stopAfter {
		// TODO(pavelkalinnikov): Integrate runWithRestarts here.
		next, err := c.fetchTail(ctx, pos)
		if err != nil {
			return err
		}
		if next == pos {
			// TODO(pavelkalinnikov): Pause with accordance to the rate of growth.
			// TODO(pavelkalinnikov): Make the duration configurable.
			if err := clock.SleepContext(ctx, 30*time.Second); err != nil {
				return err
			}
		}
		pos = next
	}

	return nil
}

// fetchTail transfers entries within the range specified in FetcherConfig,
// with respect to the passed in minimal position to start from, and the
// current tree size obtained from an STH.
func (c *Controller) fetchTail(ctx context.Context, begin uint64) (uint64, error) {
	root, err := c.plClient.getRoot(ctx)
	if err != nil {
		return 0, err
	}

	fo := c.opts.FetcherOptions
	if fo.Continuous { // Ignore range parameters in continuous mode.
		fo.StartIndex, fo.EndIndex = int64(root.TreeSize), 0
		// Use non-continuous Fetcher, as we implement continuity in Controller.
		// TODO(pavelkalinnikov): Don't overload Fetcher's Continuous flag.
		fo.Continuous = false
	} else if fo.StartIndex < 0 {
		fo.StartIndex = int64(root.TreeSize)
	}
	if int64(begin) > fo.StartIndex {
		fo.StartIndex = int64(begin)
	}
	glog.Infof("%s: fetching range [%d, %d)", c.label, fo.StartIndex, fo.EndIndex)

	fetcher := scanner.NewFetcher(c.ctClient, &fo)
	sth, err := fetcher.Prepare(ctx)
	if err != nil {
		return 0, err
	}
	metrics.sthTimestamp.Set(float64(sth.Timestamp), c.label)
	metrics.sthTreeSize.Set(float64(sth.TreeSize), c.label)
	if sth.TreeSize <= begin {
		return begin, nil
	}

	if err := c.verifyConsistency(ctx, root.TreeSize, root.RootHash, sth); err != nil {
		return 0, err
	}

	var wg sync.WaitGroup
	batches := make(chan scanner.EntryBatch, c.opts.ChannelSize)
	cctx, cancel := context.WithCancel(ctx)
	defer cancel()

	for w, cnt := 0, c.opts.Submitters; w < cnt; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := c.runSubmitter(cctx, batches); err != nil {
				glog.Errorf("%s: Stopping due to submitter error: %v", c.label, err)
				cancel() // Stop the other submitters and the Fetcher.
			}
		}()
	}

	handler := func(b scanner.EntryBatch) {
		metrics.entriesFetched.Add(float64(len(b.Entries)), c.label)
		select {
		case batches <- b:
		case <-cctx.Done(): // Avoid deadlock when shutting down.
		}
	}

	err = fetcher.Run(cctx, handler)
	close(batches)
	wg.Wait()
	if err != nil {
		return 0, err
	}
	return sth.TreeSize, nil
}

// verifyConsistency checks that the provided verified Trillian root is
// consistent with the CT log's STH.
func (c *Controller) verifyConsistency(ctx context.Context, treeSize uint64, rootHash []byte, sth *ct.SignedTreeHead) error {
	if c.opts.NoConsistencyCheck {
		glog.Warningf("%s: skipping consistency check", c.label)
		return nil
	}
	proof, err := c.ctClient.GetSTHConsistency(ctx, treeSize, sth.TreeSize)
	if err != nil {
		return err
	}
	return logverifier.New(hasher.DefaultHasher).VerifyConsistencyProof(
		int64(treeSize), int64(sth.TreeSize),
		rootHash, sth.SHA256RootHash[:], proof)
}

// runSubmitter obtains CT log entry batches from the controller's channel and
// submits them through Trillian client. Returns when the channel is closed, or
// the client returns a non-recoverable error (an example of a recoverable
// error is when Trillian write quota is exceeded).
func (c *Controller) runSubmitter(ctx context.Context, batches <-chan scanner.EntryBatch) error {
	for b := range batches {
		entries := float64(len(b.Entries))
		metrics.entriesSeen.Add(entries, c.label)

		end := b.Start + int64(len(b.Entries))
		if err := c.plClient.addSequencedLeaves(ctx, &b); err != nil {
			// addSequencedLeaves failed to submit entries despite retries. At this
			// point there is not much we can do. Seemingly the best strategy is to
			// shut down the Controller.
			return fmt.Errorf("failed to add batch [%d, %d): %v", b.Start, end, err)
		}
		glog.Infof("%s: added batch [%d, %d)", c.label, b.Start, end)
		metrics.entriesStored.Add(entries, c.label)
	}
	return nil
}

// sleepRandom sleeps for random duration in [base, base+spread).
func sleepRandom(ctx context.Context, base, spread time.Duration) error {
	d := randDuration(base, spread)
	if d == 0 {
		return nil
	}
	return clock.SleepContext(ctx, d)
}

// randDuration returns a random duration in [base, base+spread).
func randDuration(base, spread time.Duration) time.Duration {
	d := base
	if spread != 0 {
		d += time.Duration(rand.Int63n(int64(spread)))
	}
	return d
}
<|endoftext|>


<commit_before>package torrent

import (
	"strconv"
	"strings"

	"github.com/anacrolix/missinggo/pubsub"
	"github.com/anacrolix/sync"

	"github.com/anacrolix/torrent/metainfo"
)

// The Torrent's infohash. This is fixed and cannot change. It uniquely identifies a torrent.
func (t *Torrent) InfoHash() metainfo.Hash {
	return t.infoHash
}

// Returns a channel that is closed when the info (.Info()) for the torrent has become available.
func (t *Torrent) GotInfo() (ret <-chan struct{}) {
	// TODO: We shouldn't need to lock to take a channel here, if the event is only ever set.
	t.nameMu.RLock()
	ret = t.gotMetainfoC
	t.nameMu.RUnlock()
	return
}

// Returns the metainfo info dictionary, or nil if it's not yet available.
func (t *Torrent) Info() (info *metainfo.Info) {
	t.nameMu.RLock()
	info = t.info
	t.nameMu.RUnlock()
	return
}

// Returns a Reader bound to the torrent's data. All read calls block until the data requested is
// actually available. Note that you probably want to ensure the Torrent Info is available first.
func (t *Torrent) NewReader() Reader {
	return t.newReader(0, *t.length)
}

func (t *Torrent) newReader(offset, length int64) Reader {
	r := reader{
		mu:     t.cl.locker(),
		t:      t,
		offset: offset,
		length: length,
	}
	r.readaheadFunc = r.defaultReadaheadFunc
	t.addReader(&r)
	return &r
}

type PieceStateRuns []PieceStateRun

func (me PieceStateRuns) String() (s string) {
	if len(me) > 0 {
		var sb strings.Builder
		sb.WriteString(me[0].String())
		for i := 1; i < len(me); i += 1 {
			sb.WriteByte(' ')
			sb.WriteString(me[i].String())
		}
		return sb.String()
	}
	return
}

// Returns the state of pieces of the torrent. They are grouped into runs of same state. The sum of
// the state run-lengths is the number of pieces in the torrent.
func (t *Torrent) PieceStateRuns() (runs PieceStateRuns) {
	t.cl.rLock()
	runs = t.pieceStateRuns()
	t.cl.rUnlock()
	return
}

func (t *Torrent) PieceState(piece pieceIndex) (ps PieceState) {
	t.cl.rLock()
	ps = t.pieceState(piece)
	t.cl.rUnlock()
	return
}

// The number of pieces in the torrent. This requires that the info has been
// obtained first.
func (t *Torrent) NumPieces() pieceIndex {
	return t.numPieces()
}

// Get missing bytes count for specific piece.
func (t *Torrent) PieceBytesMissing(piece int) int64 {
	t.cl.lock()
	defer t.cl.unlock()

	return int64(t.pieces[piece].bytesLeft())
}

// Drop the torrent from the client, and close it. It's always safe to do
// this. No data corruption can, or should occur to either the torrent's data,
// or connected peers.
func (t *Torrent) Drop() {
	var wg sync.WaitGroup
	defer wg.Wait()
	t.cl.lock()
	defer t.cl.unlock()
	t.cl.dropTorrent(t.infoHash, &wg)
}

// Number of bytes of the entire torrent we have completed. This is the sum of
// completed pieces, and dirtied chunks of incomplete pieces. Do not use this
// for download rate, as it can go down when pieces are lost or fail checks.
// Sample Torrent.Stats.DataBytesRead for actual file data download rate.
func (t *Torrent) BytesCompleted() int64 {
	t.cl.rLock()
	defer t.cl.rUnlock()
	return t.bytesCompleted()
}

// The subscription emits as (int) the index of pieces as their state changes.
// A state change is when the PieceState for a piece alters in value.
func (t *Torrent) SubscribePieceStateChanges() *pubsub.Subscription {
	return t.pieceStateChanges.Subscribe()
}

// Returns true if the torrent is currently being seeded. This occurs when the
// client is willing to upload without wanting anything in return.
func (t *Torrent) Seeding() bool {
	t.cl.lock()
	defer t.cl.unlock()
	return t.seeding()
}

// Clobbers the torrent display name. The display name is used as the torrent
// name if the metainfo is not available.
func (t *Torrent) SetDisplayName(dn string) {
	t.nameMu.Lock()
	defer t.nameMu.Unlock()
	if t.haveInfo() {
		return
	}
	t.displayName = dn
}

// The current working name for the torrent. Either the name in the info dict,
// or a display name given such as by the dn value in a magnet link, or "".
func (t *Torrent) Name() string {
	return t.name()
}

// The completed length of all the torrent data, in all its files. This is
// derived from the torrent info, when it is available.
func (t *Torrent) Length() int64 {
	return *t.length
}

// Returns a run-time generated metainfo for the torrent that includes the
// info bytes and announce-list as currently known to the client.
func (t *Torrent) Metainfo() metainfo.MetaInfo {
	t.cl.lock()
	defer t.cl.unlock()
	return t.newMetaInfo()
}

func (t *Torrent) addReader(r *reader) {
	t.cl.lock()
	defer t.cl.unlock()
	if t.readers == nil {
		t.readers = make(map[*reader]struct{})
	}
	t.readers[r] = struct{}{}
	r.posChanged()
}

func (t *Torrent) deleteReader(r *reader) {
	delete(t.readers, r)
	t.readersChanged()
}

// Raise the priorities of pieces in the range [begin, end) to at least Normal
// priority. Piece indexes are not the same as bytes. Requires that the info
// has been obtained, see Torrent.Info and Torrent.GotInfo.
func (t *Torrent) DownloadPieces(begin, end pieceIndex) {
	t.cl.lock()
	t.downloadPiecesLocked(begin, end)
	t.cl.unlock()
}

func (t *Torrent) downloadPiecesLocked(begin, end pieceIndex) {
	for i := begin; i < end; i++ {
		if t.pieces[i].priority.Raise(PiecePriorityNormal) {
			t.updatePiecePriority(i)
		}
	}
}

func (t *Torrent) CancelPieces(begin, end pieceIndex) {
	t.cl.lock()
	t.cancelPiecesLocked(begin, end)
	t.cl.unlock()
}

func (t *Torrent) cancelPiecesLocked(begin, end pieceIndex) {
	for i := begin; i < end; i++ {
		p := &t.pieces[i]
		if p.priority == PiecePriorityNone {
			continue
		}
		p.priority = PiecePriorityNone
		t.updatePiecePriority(i)
	}
}

func (t *Torrent) initFiles() {
	var offset int64
	t.files = new([]*File)
	for _, fi := range t.info.UpvertedFiles() {
		var path []string
		if len(fi.PathUTF8) != 0 {
			path = fi.PathUTF8
		} else {
			path = fi.Path
		}
		dp := t.info.Name
		if len(fi.Path) != 0 {
			dp = strings.Join(fi.Path, "/")
		}
		*t.files = append(*t.files, &File{
			t,
			strings.Join(append([]string{t.info.Name}, path...), "/"),
			offset,
			fi.Length,
			fi,
			dp,
			PiecePriorityNone,
		})
		offset += fi.Length
	}

}

// Returns handles to the files in the torrent. This requires that the Info is
// available first.
func (t *Torrent) Files() []*File {
	return *t.files
}

func (t *Torrent) AddPeers(pp []PeerInfo) (n int) {
	t.cl.lock()
	n = t.addPeers(pp)
	t.cl.unlock()
	return
}

// Marks the entire torrent for download. Requires the info first, see
// GotInfo. Sets piece priorities for historical reasons.
func (t *Torrent) DownloadAll() {
	t.DownloadPieces(0, t.numPieces())
}

func (t *Torrent) String() string {
	s := t.name()
	if s == "" {
		return t.infoHash.HexString()
	} else {
		return strconv.Quote(s)
	}
}

func (t *Torrent) AddTrackers(announceList [][]string) {
	t.cl.lock()
	defer t.cl.unlock()
	t.addTrackers(announceList)
}

func (t *Torrent) Piece(i pieceIndex) *Piece {
	return t.piece(i)
}

func (t *Torrent) PeerConns() []*PeerConn {
	t.cl.rLock()
	defer t.cl.rUnlock()
	ret := make([]*PeerConn, 0, len(t.conns))
	for c := range t.conns {
		ret = append(ret, c)
	}
	return ret
}
<commit_msg>Inlineable `(*Torrent).SetDisplayName()` (#625)<commit_after>package torrent

import (
	"strconv"
	"strings"

	"github.com/anacrolix/missinggo/pubsub"
	"github.com/anacrolix/sync"

	"github.com/anacrolix/torrent/metainfo"
)

// The Torrent's infohash. This is fixed and cannot change. It uniquely identifies a torrent.
func (t *Torrent) InfoHash() metainfo.Hash {
	return t.infoHash
}

// Returns a channel that is closed when the info (.Info()) for the torrent has become available.
func (t *Torrent) GotInfo() (ret <-chan struct{}) {
	// TODO: We shouldn't need to lock to take a channel here, if the event is only ever set.
	t.nameMu.RLock()
	ret = t.gotMetainfoC
	t.nameMu.RUnlock()
	return
}

// Returns the metainfo info dictionary, or nil if it's not yet available.
func (t *Torrent) Info() (info *metainfo.Info) {
	t.nameMu.RLock()
	info = t.info
	t.nameMu.RUnlock()
	return
}

// Returns a Reader bound to the torrent's data. All read calls block until the data requested is
// actually available. Note that you probably want to ensure the Torrent Info is available first.
func (t *Torrent) NewReader() Reader {
	return t.newReader(0, *t.length)
}

func (t *Torrent) newReader(offset, length int64) Reader {
	r := reader{
		mu:     t.cl.locker(),
		t:      t,
		offset: offset,
		length: length,
	}
	r.readaheadFunc = r.defaultReadaheadFunc
	t.addReader(&r)
	return &r
}

type PieceStateRuns []PieceStateRun

func (me PieceStateRuns) String() (s string) {
	if len(me) > 0 {
		var sb strings.Builder
		sb.WriteString(me[0].String())
		for i := 1; i < len(me); i += 1 {
			sb.WriteByte(' ')
			sb.WriteString(me[i].String())
		}
		return sb.String()
	}
	return
}

// Returns the state of pieces of the torrent. They are grouped into runs of same state. The sum of
// the state run-lengths is the number of pieces in the torrent.
func (t *Torrent) PieceStateRuns() (runs PieceStateRuns) {
	t.cl.rLock()
	runs = t.pieceStateRuns()
	t.cl.rUnlock()
	return
}

func (t *Torrent) PieceState(piece pieceIndex) (ps PieceState) {
	t.cl.rLock()
	ps = t.pieceState(piece)
	t.cl.rUnlock()
	return
}

// The number of pieces in the torrent. This requires that the info has been
// obtained first.
func (t *Torrent) NumPieces() pieceIndex {
	return t.numPieces()
}

// Get missing bytes count for specific piece.
func (t *Torrent) PieceBytesMissing(piece int) int64 {
	t.cl.lock()
	defer t.cl.unlock()

	return int64(t.pieces[piece].bytesLeft())
}

// Drop the torrent from the client, and close it. It's always safe to do
// this. No data corruption can, or should occur to either the torrent's data,
// or connected peers.
func (t *Torrent) Drop() {
	var wg sync.WaitGroup
	defer wg.Wait()
	t.cl.lock()
	defer t.cl.unlock()
	t.cl.dropTorrent(t.infoHash, &wg)
}

// Number of bytes of the entire torrent we have completed. This is the sum of
// completed pieces, and dirtied chunks of incomplete pieces. Do not use this
// for download rate, as it can go down when pieces are lost or fail checks.
// Sample Torrent.Stats.DataBytesRead for actual file data download rate.
func (t *Torrent) BytesCompleted() int64 {
	t.cl.rLock()
	defer t.cl.rUnlock()
	return t.bytesCompleted()
}

// The subscription emits as (int) the index of pieces as their state changes.
// A state change is when the PieceState for a piece alters in value.
func (t *Torrent) SubscribePieceStateChanges() *pubsub.Subscription {
	return t.pieceStateChanges.Subscribe()
}

// Returns true if the torrent is currently being seeded. This occurs when the
// client is willing to upload without wanting anything in return.
func (t *Torrent) Seeding() bool {
	t.cl.lock()
	defer t.cl.unlock()
	return t.seeding()
}

// Clobbers the torrent display name if metainfo is unavailable.
// The display name is used as the torrent name while the metainfo is unavailable.
func (t *Torrent) SetDisplayName(dn string) {
	t.nameMu.Lock()
	if !t.haveInfo() {
		t.displayName = dn
	}
	t.nameMu.Unlock()
}

// The current working name for the torrent. Either the name in the info dict,
// or a display name given such as by the dn value in a magnet link, or "".
func (t *Torrent) Name() string {
	return t.name()
}

// The completed length of all the torrent data, in all its files. This is
// derived from the torrent info, when it is available.
func (t *Torrent) Length() int64 {
	return *t.length
}

// Returns a run-time generated metainfo for the torrent that includes the
// info bytes and announce-list as currently known to the client.
func (t *Torrent) Metainfo() metainfo.MetaInfo {
	t.cl.lock()
	defer t.cl.unlock()
	return t.newMetaInfo()
}

func (t *Torrent) addReader(r *reader) {
	t.cl.lock()
	defer t.cl.unlock()
	if t.readers == nil {
		t.readers = make(map[*reader]struct{})
	}
	t.readers[r] = struct{}{}
	r.posChanged()
}

func (t *Torrent) deleteReader(r *reader) {
	delete(t.readers, r)
	t.readersChanged()
}

// Raise the priorities of pieces in the range [begin, end) to at least Normal
// priority. Piece indexes are not the same as bytes. Requires that the info
// has been obtained, see Torrent.Info and Torrent.GotInfo.
func (t *Torrent) DownloadPieces(begin, end pieceIndex) {
	t.cl.lock()
	t.downloadPiecesLocked(begin, end)
	t.cl.unlock()
}

func (t *Torrent) downloadPiecesLocked(begin, end pieceIndex) {
	for i := begin; i < end; i++ {
		if t.pieces[i].priority.Raise(PiecePriorityNormal) {
			t.updatePiecePriority(i)
		}
	}
}

func (t *Torrent) CancelPieces(begin, end pieceIndex) {
	t.cl.lock()
	t.cancelPiecesLocked(begin, end)
	t.cl.unlock()
}

func (t *Torrent) cancelPiecesLocked(begin, end pieceIndex) {
	for i := begin; i < end; i++ {
		p := &t.pieces[i]
		if p.priority == PiecePriorityNone {
			continue
		}
		p.priority = PiecePriorityNone
		t.updatePiecePriority(i)
	}
}

func (t *Torrent) initFiles() {
	var offset int64
	t.files = new([]*File)
	for _, fi := range t.info.UpvertedFiles() {
		var path []string
		if len(fi.PathUTF8) != 0 {
			path = fi.PathUTF8
		} else {
			path = fi.Path
		}
		dp := t.info.Name
		if len(fi.Path) != 0 {
			dp = strings.Join(fi.Path, "/")
		}
		*t.files = append(*t.files, &File{
			t,
			strings.Join(append([]string{t.info.Name}, path...), "/"),
			offset,
			fi.Length,
			fi,
			dp,
			PiecePriorityNone,
		})
		offset += fi.Length
	}

}

// Returns handles to the files in the torrent. This requires that the Info is
// available first.
func (t *Torrent) Files() []*File {
	return *t.files
}

func (t *Torrent) AddPeers(pp []PeerInfo) (n int) {
	t.cl.lock()
	n = t.addPeers(pp)
	t.cl.unlock()
	return
}

// Marks the entire torrent for download. Requires the info first, see
// GotInfo. Sets piece priorities for historical reasons.
func (t *Torrent) DownloadAll() {
	t.DownloadPieces(0, t.numPieces())
}

func (t *Torrent) String() string {
	s := t.name()
	if s == "" {
		return t.infoHash.HexString()
	} else {
		return strconv.Quote(s)
	}
}

func (t *Torrent) AddTrackers(announceList [][]string) {
	t.cl.lock()
	defer t.cl.unlock()
	t.addTrackers(announceList)
}

func (t *Torrent) Piece(i pieceIndex) *Piece {
	return t.piece(i)
}

func (t *Torrent) PeerConns() []*PeerConn {
	t.cl.rLock()
	defer t.cl.rUnlock()
	ret := make([]*PeerConn, 0, len(t.conns))
	for c := range t.conns {
		ret = append(ret, c)
	}
	return ret
}
<|endoftext|>


<commit_before>package mgr

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/json-iterator/go"
	"github.com/qiniu/pandora-go-sdk/base/reqerr"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/parser/grok"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/router"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/transforms"
	. "github.com/qiniu/logkit/utils/models"
)

const DefaultTryTimes = 3

// RawData 从 reader 模块中根据 type 获取字符串形式的样例日志
func RawData(readerConfig conf.MapConf) (string, error) {
	if readerConfig == nil {
		return "", fmt.Errorf("reader config cannot be empty")
	}

	rd, err := reader.NewReader(readerConfig, true)
	if err != nil {
		return "", err
	}
	defer rd.Close()

	var rawData string

	// DataReader 设定超时，普通 Reader 设定重试
	if dr, ok := rd.(reader.DataReader); ok {
		type dataErr struct {
			data string
			err  error
		}
		// Note: 添加一位缓冲保证 goroutine 在 runner 已经超时的情况下能够正常退出，避免资源泄露
		readChan := make(chan dataErr, 1)
		go func() {
			data, _, err := dr.ReadData()
			if err != nil && err != io.EOF {
				readChan <- dataErr{"", err}
				return
			}

			p, err := jsoniter.MarshalIndent(data, "", "  ")
			readChan <- dataErr{string(p), err}
		}()

		timeout := time.NewTimer(time.Minute)
		select {
		case de := <-readChan:
			rawData, err = de.data, de.err
			if err != nil {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}

		case <-timeout.C:
			return "", fmt.Errorf("reader %q read data timeout, no data received", rd.Name())
		}

	} else {
		tryCount := DefaultTryTimes
		for {
			if tryCount <= 0 {
				return "", fmt.Errorf("reader %q read line timeout, no data received", rd.Name())
			}
			tryCount--

			rawData, err = rd.ReadLine()
			if err != nil && err != io.EOF {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}
			if err == io.EOF {
				return rawData, nil
			}

			if len(rawData) > 0 {
				break
			}
		}
	}

	if len(rawData) >= DefaultMaxBatchSize {
		err = errors.New("data size large than 2M and will be discard")
		return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
	}
	return rawData, nil
}

//parse模块中各种type的日志都能获取解析后的数据
func ParseData(parserConfig conf.MapConf) (parsedData []Data, err error) {
	parserConfig = parser.ConvertWebParserConfig(parserConfig)
	if parserConfig == nil {
		err = fmt.Errorf("parser config was empty after web config convet")
		return
	}

	sampleData, err := getSampleData(parserConfig)
	if err != nil {
		return nil, err
	}
	parserRegistry := parser.NewRegistry()
	logParser, err := parserRegistry.NewLogParser(parserConfig)
	if err != nil {
		return nil, err
	}
	sampleData, err = checkSampleData(sampleData, logParser)
	if err != nil {
		return nil, err
	}

	parsedData, err = logParser.Parse(sampleData)
	err = checkErr(err, logParser.Name())
	if err != nil {
		return nil, err
	}

	return
}

func TransformData(transformerConfig map[string]interface{}) ([]Data, error) {
	if transformerConfig == nil {
		err := fmt.Errorf("transformer config cannot be empty")
		return nil, err
	}

	create, err := getTransformerCreator(transformerConfig)
	if err != nil {
		return nil, err
	}

	data, err := getDataFromTransformConfig(transformerConfig)
	if err != nil {
		return nil, err
	}

	transformer, err := getTransformer(transformerConfig, create)
	if err != nil {
		return nil, err
	}

	// Transform data
	transformedData, transErr := transformer.Transform(data)
	se, ok := transErr.(*StatsError)
	if ok {
		transErr = se.ErrorDetail
	}
	if transErr != nil {
		se, ok := transErr.(*StatsError)
		if ok {
			transErr = se.ErrorDetail
		}
		err := fmt.Errorf("transform processing error %v", transErr)
		return nil, err
	}
	return transformedData, nil
}

func SendData(senderConfig map[string]interface{}) error {
	if senderConfig == nil {
		return fmt.Errorf("sender config cannot be empty")
	}

	sendersConf, err := getSendersConfig(senderConfig)
	if err != nil {
		return err
	}

	senders, err := getSenders(sendersConf)
	if err != nil {
		return err
	}

	router, err := getRouter(senderConfig, len(senders))
	if err != nil {
		return err
	}

	datas, err := getDataFromSenderConfig(senderConfig)
	if err != nil {
		return err
	}

	senderDataList := classifySenderData(senders, datas, router)
	for index, s := range senders {
		if err := trySend(s, senderDataList[index], DefaultTryTimes); err != nil {
			return err
		}
	}

	return nil
}

func getSendersConfig(senderConfig map[string]interface{}) ([]conf.MapConf, error) {
	config := senderConfig[KeySendConfig]
	byteSendersConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return nil, err
	}
	var sendersConf []conf.MapConf
	if jsonErr := jsoniter.Unmarshal(byteSendersConfig, &sendersConf); jsonErr != nil {
		return nil, jsonErr
	}
	if sendersConf == nil {
		return nil, fmt.Errorf("sender config cannot be empty")
	}
	return sendersConf, nil
}

func getDataFromSenderConfig(senderConfig map[string]interface{}) ([]Data, error) {
	var datas = []Data{}
	rawData := senderConfig[KeySampleLog]
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("sender fetched empty sample log")
		return nil, err
	}

	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &datas); jsonErr != nil {
		return nil, jsonErr
	}
	return datas, nil
}

func getSenders(sendersConf []conf.MapConf) ([]sender.Sender, error) {
	senders := make([]sender.Sender, 0)
	sr := sender.NewRegistry()
	for i, senderConfig := range sendersConf {
		senderConfig[sender.KeyFaultTolerant] = "false"
		s, err := sr.NewSender(senderConfig, "")
		if err != nil {
			return nil, err
		}
		senders = append(senders, s)
		delete(sendersConf[i], sender.InnerUserAgent)
	}
	return senders, nil
}

func getRouter(senderConfig map[string]interface{}, senderCnt int) (*router.Router, error) {
	routerConf, err := getRouterConfig(senderConfig)
	if err != nil {
		return nil, err
	}

	router, err := router.NewSenderRouter(routerConf, senderCnt)
	if err != nil {
		return nil, fmt.Errorf("sender router error, %v", err)
	}
	return router, nil
}

func getRouterConfig(senderConfig map[string]interface{}) (router.RouterConfig, error) {
	config := senderConfig[KeyRouterConfig]
	byteRouterConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return router.RouterConfig{}, err
	}

	var routerConfig router.RouterConfig
	if jsonErr := jsoniter.Unmarshal(byteRouterConfig, &routerConfig); jsonErr != nil {
		return router.RouterConfig{}, jsonErr
	}
	return routerConfig, nil
}

// trySend 尝试发送数据，如果此时runner退出返回false，其他情况无论是达到最大重试次数还是发送成功，都返回true
func trySend(s sender.Sender, datas []Data, times int) (err error) {
	if times <= 0 {
		times = DefaultTryTimes
	}
	if len(datas) <= 0 {
		return nil
	}
	cnt := 0
	for {
		if cnt >= times {
			break
		}
		err = s.Send(datas)
		if se, ok := err.(*StatsError); ok {
			err = se.ErrorDetail
		}

		if err != nil {
			cnt++
			se, ok := err.(*reqerr.SendError)
			if ok {
				datas = sender.ConvertDatas(se.GetFailDatas())
				continue
			}
			if cnt < times {
				continue
			}
			err = fmt.Errorf("retry send %v times, but still error %v, discard datas %v ... total %v lines", cnt, err, datas, len(datas))
			return err
		}
		break
	}
	return
}

func getSampleData(parserConfig conf.MapConf) ([]string, error) {
	parserType, _ := parserConfig.GetString(parser.KeyParserType)
	rawData, _ := parserConfig.GetStringOr(KeySampleLog, "")
	var sampleData []string

	switch parserType {
	case parser.TypeCSV, parser.TypeJSON, parser.TypeRaw, parser.TypeNginx, parser.TypeEmpty, parser.TypeKafkaRest, parser.TypeLogv1:
		sampleData = append(sampleData, rawData)
	case parser.TypeSyslog:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeMySQL:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeGrok:
		grokMode, _ := parserConfig.GetString(parser.KeyGrokMode)
		if grokMode != grok.ModeMulti {
			sampleData = append(sampleData, rawData)
		} else {
			sampleData = []string{rawData}
		}
	default:
		sampleData = append(sampleData, rawData)
	}

	return sampleData, nil
}

func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, error) {
	if len(sampleData) <= 0 {
		_, ok := logParser.(parser.Flushable)
		if ok {
			sampleData = []string{parser.PandoraParseFlushSignal}
		} else {
			err := fmt.Errorf("parser [%v] fetched 0 lines", logParser.Name())
			return nil, err
		}
	}
	return sampleData, nil
}

func checkErr(err error, parserName string) error {
	se, ok := err.(*StatsError)
	var errorCnt int64
	if ok {
		errorCnt = se.Errors
		err = se.ErrorDetail
	} else if err != nil {
		errorCnt = 1
	}
	if err != nil {
		errMsg := fmt.Sprintf("parser %s, error %v ", parserName, err.Error())
		err = fmt.Errorf("%v parse line errors occured, same as %v", errorCnt, errors.New(errMsg))
	}

	return err
}

func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) {
	transformKeyType, ok := transformerConfig[transforms.KeyType]
	if !ok {
		err := fmt.Errorf("missing param %s", transforms.KeyType)
		return nil, err
	}
	transformKeyTypeStr, ok := transformKeyType.(string)
	if !ok {
		err := fmt.Errorf("param %s must be of type string", transforms.KeyType)
		return nil, err
	}

	create, ok := transforms.Transformers[transformKeyTypeStr]
	if !ok {
		err := fmt.Errorf("transformer of type %v not exist", transformKeyTypeStr)
		return nil, err
	}
	return create, nil
}

func getDataFromTransformConfig(transformerConfig map[string]interface{}) ([]Data, error) {
	rawData, ok := transformerConfig[KeySampleLog]
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("transformer fetched empty sample log")
		return nil, err
	}

	var data = []Data{}
	var singleData Data
	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &singleData); jsonErr != nil {
		if jsonErr = jsoniter.Unmarshal([]byte(rawDataStr), &data); jsonErr != nil {
			return nil, jsonErr
		}
	} else {
		// is single log, and method transformer.transform(data []sender.Data) accept a param of slice type
		data = append(data, singleData)
	}
	return data, nil
}

func getTransformer(transConfig map[string]interface{}, create transforms.Creator) (transforms.Transformer, error) {
	trans := create()
	transformerConfig := convertWebTransformerConfig(transConfig)
	delete(transformerConfig, KeySampleLog)
	bts, jsonErr := jsoniter.Marshal(transformerConfig)
	if jsonErr != nil {
		return nil, jsonErr
	}
	if jsonErr = jsoniter.Unmarshal(bts, trans); jsonErr != nil {
		return nil, jsonErr
	}

	if trans, ok := trans.(transforms.Initialize); ok {
		if err := trans.Init(); err != nil {
			return nil, err
		}
	}
	return trans, nil
}
<commit_msg>raw data with mock meta path<commit_after>package mgr

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/json-iterator/go"

	"github.com/qiniu/log"
	"github.com/qiniu/pandora-go-sdk/base/reqerr"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/parser/grok"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/router"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/transforms"
	. "github.com/qiniu/logkit/utils/models"
)

const (
	DefaultTryTimes = 3
	MetaTmp         = "meta_tmp/"
)

// RawData 从 reader 模块中根据 type 获取字符串形式的样例日志
func RawData(readerConfig conf.MapConf) (string, error) {
	if readerConfig == nil {
		return "", fmt.Errorf("reader config cannot be empty")
	}

	runnerName, _ := readerConfig.GetString(GlobalKeyName)
	configMetaPath := runnerName + "_" + Hash(strconv.FormatInt(time.Now().Unix(), 10))
	metaPath := path.Join(MetaTmp, configMetaPath)
	log.Debugf("Runner[%v] Using %s as default metaPath", runnerName, metaPath)
	readerConfig[reader.KeyMetaPath] = metaPath

	rd, err := reader.NewReader(readerConfig, true)
	if err != nil {
		return "", err
	}
	defer func() {
		rd.Close()
		os.RemoveAll(metaPath)
	}()

	var rawData string

	// DataReader 设定超时，普通 Reader 设定重试
	if dr, ok := rd.(reader.DataReader); ok {
		type dataErr struct {
			data string
			err  error
		}
		// Note: 添加一位缓冲保证 goroutine 在 runner 已经超时的情况下能够正常退出，避免资源泄露
		readChan := make(chan dataErr, 1)
		go func() {
			data, _, err := dr.ReadData()
			if err != nil && err != io.EOF {
				readChan <- dataErr{"", err}
				return
			}

			p, err := jsoniter.MarshalIndent(data, "", "  ")
			readChan <- dataErr{string(p), err}
		}()

		timeout := time.NewTimer(time.Minute)
		select {
		case de := <-readChan:
			rawData, err = de.data, de.err
			if err != nil {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}

		case <-timeout.C:
			return "", fmt.Errorf("reader %q read data timeout, no data received", rd.Name())
		}

	} else {
		tryCount := DefaultTryTimes
		for {
			if tryCount <= 0 {
				return "", fmt.Errorf("reader %q read line timeout, no data received", rd.Name())
			}
			tryCount--

			rawData, err = rd.ReadLine()
			if err != nil && err != io.EOF {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}
			if err == io.EOF {
				return rawData, nil
			}

			if len(rawData) > 0 {
				break
			}
		}
	}

	if len(rawData) >= DefaultMaxBatchSize {
		err = errors.New("data size large than 2M and will be discard")
		return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
	}
	return rawData, nil
}

//parse模块中各种type的日志都能获取解析后的数据
func ParseData(parserConfig conf.MapConf) (parsedData []Data, err error) {
	parserConfig = parser.ConvertWebParserConfig(parserConfig)
	if parserConfig == nil {
		err = fmt.Errorf("parser config was empty after web config convet")
		return
	}

	sampleData, err := getSampleData(parserConfig)
	if err != nil {
		return nil, err
	}
	parserRegistry := parser.NewRegistry()
	logParser, err := parserRegistry.NewLogParser(parserConfig)
	if err != nil {
		return nil, err
	}
	sampleData, err = checkSampleData(sampleData, logParser)
	if err != nil {
		return nil, err
	}

	parsedData, err = logParser.Parse(sampleData)
	err = checkErr(err, logParser.Name())
	if err != nil {
		return nil, err
	}

	return
}

func TransformData(transformerConfig map[string]interface{}) ([]Data, error) {
	if transformerConfig == nil {
		err := fmt.Errorf("transformer config cannot be empty")
		return nil, err
	}

	create, err := getTransformerCreator(transformerConfig)
	if err != nil {
		return nil, err
	}

	data, err := getDataFromTransformConfig(transformerConfig)
	if err != nil {
		return nil, err
	}

	transformer, err := getTransformer(transformerConfig, create)
	if err != nil {
		return nil, err
	}

	// Transform data
	transformedData, transErr := transformer.Transform(data)
	se, ok := transErr.(*StatsError)
	if ok {
		transErr = se.ErrorDetail
	}
	if transErr != nil {
		se, ok := transErr.(*StatsError)
		if ok {
			transErr = se.ErrorDetail
		}
		err := fmt.Errorf("transform processing error %v", transErr)
		return nil, err
	}
	return transformedData, nil
}

func SendData(senderConfig map[string]interface{}) error {
	if senderConfig == nil {
		return fmt.Errorf("sender config cannot be empty")
	}

	sendersConf, err := getSendersConfig(senderConfig)
	if err != nil {
		return err
	}

	senders, err := getSenders(sendersConf)
	if err != nil {
		return err
	}

	router, err := getRouter(senderConfig, len(senders))
	if err != nil {
		return err
	}

	datas, err := getDataFromSenderConfig(senderConfig)
	if err != nil {
		return err
	}

	senderDataList := classifySenderData(senders, datas, router)
	for index, s := range senders {
		if err := trySend(s, senderDataList[index], DefaultTryTimes); err != nil {
			return err
		}
	}

	return nil
}

func getSendersConfig(senderConfig map[string]interface{}) ([]conf.MapConf, error) {
	config := senderConfig[KeySendConfig]
	byteSendersConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return nil, err
	}
	var sendersConf []conf.MapConf
	if jsonErr := jsoniter.Unmarshal(byteSendersConfig, &sendersConf); jsonErr != nil {
		return nil, jsonErr
	}
	if sendersConf == nil {
		return nil, fmt.Errorf("sender config cannot be empty")
	}
	return sendersConf, nil
}

func getDataFromSenderConfig(senderConfig map[string]interface{}) ([]Data, error) {
	var datas = []Data{}
	rawData := senderConfig[KeySampleLog]
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("sender fetched empty sample log")
		return nil, err
	}

	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &datas); jsonErr != nil {
		return nil, jsonErr
	}
	return datas, nil
}

func getSenders(sendersConf []conf.MapConf) ([]sender.Sender, error) {
	senders := make([]sender.Sender, 0)
	sr := sender.NewRegistry()
	for i, senderConfig := range sendersConf {
		senderConfig[sender.KeyFaultTolerant] = "false"
		s, err := sr.NewSender(senderConfig, "")
		if err != nil {
			return nil, err
		}
		senders = append(senders, s)
		delete(sendersConf[i], sender.InnerUserAgent)
	}
	return senders, nil
}

func getRouter(senderConfig map[string]interface{}, senderCnt int) (*router.Router, error) {
	routerConf, err := getRouterConfig(senderConfig)
	if err != nil {
		return nil, err
	}

	router, err := router.NewSenderRouter(routerConf, senderCnt)
	if err != nil {
		return nil, fmt.Errorf("sender router error, %v", err)
	}
	return router, nil
}

func getRouterConfig(senderConfig map[string]interface{}) (router.RouterConfig, error) {
	config := senderConfig[KeyRouterConfig]
	byteRouterConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return router.RouterConfig{}, err
	}

	var routerConfig router.RouterConfig
	if jsonErr := jsoniter.Unmarshal(byteRouterConfig, &routerConfig); jsonErr != nil {
		return router.RouterConfig{}, jsonErr
	}
	return routerConfig, nil
}

// trySend 尝试发送数据，如果此时runner退出返回false，其他情况无论是达到最大重试次数还是发送成功，都返回true
func trySend(s sender.Sender, datas []Data, times int) (err error) {
	if times <= 0 {
		times = DefaultTryTimes
	}
	if len(datas) <= 0 {
		return nil
	}
	cnt := 0
	for {
		if cnt >= times {
			break
		}
		err = s.Send(datas)
		if se, ok := err.(*StatsError); ok {
			err = se.ErrorDetail
		}

		if err != nil {
			cnt++
			se, ok := err.(*reqerr.SendError)
			if ok {
				datas = sender.ConvertDatas(se.GetFailDatas())
				continue
			}
			if cnt < times {
				continue
			}
			err = fmt.Errorf("retry send %v times, but still error %v, discard datas %v ... total %v lines", cnt, err, datas, len(datas))
			return err
		}
		break
	}
	return
}

func getSampleData(parserConfig conf.MapConf) ([]string, error) {
	parserType, _ := parserConfig.GetString(parser.KeyParserType)
	rawData, _ := parserConfig.GetStringOr(KeySampleLog, "")
	var sampleData []string

	switch parserType {
	case parser.TypeCSV, parser.TypeJSON, parser.TypeRaw, parser.TypeNginx, parser.TypeEmpty, parser.TypeKafkaRest, parser.TypeLogv1:
		sampleData = append(sampleData, rawData)
	case parser.TypeSyslog:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeMySQL:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeGrok:
		grokMode, _ := parserConfig.GetString(parser.KeyGrokMode)
		if grokMode != grok.ModeMulti {
			sampleData = append(sampleData, rawData)
		} else {
			sampleData = []string{rawData}
		}
	default:
		sampleData = append(sampleData, rawData)
	}

	return sampleData, nil
}

func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, error) {
	if len(sampleData) <= 0 {
		_, ok := logParser.(parser.Flushable)
		if ok {
			sampleData = []string{parser.PandoraParseFlushSignal}
		} else {
			err := fmt.Errorf("parser [%v] fetched 0 lines", logParser.Name())
			return nil, err
		}
	}
	return sampleData, nil
}

func checkErr(err error, parserName string) error {
	se, ok := err.(*StatsError)
	var errorCnt int64
	if ok {
		errorCnt = se.Errors
		err = se.ErrorDetail
	} else if err != nil {
		errorCnt = 1
	}
	if err != nil {
		errMsg := fmt.Sprintf("parser %s, error %v ", parserName, err.Error())
		err = fmt.Errorf("%v parse line errors occured, same as %v", errorCnt, errors.New(errMsg))
	}

	return err
}

func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) {
	transformKeyType, ok := transformerConfig[transforms.KeyType]
	if !ok {
		err := fmt.Errorf("missing param %s", transforms.KeyType)
		return nil, err
	}
	transformKeyTypeStr, ok := transformKeyType.(string)
	if !ok {
		err := fmt.Errorf("param %s must be of type string", transforms.KeyType)
		return nil, err
	}

	create, ok := transforms.Transformers[transformKeyTypeStr]
	if !ok {
		err := fmt.Errorf("transformer of type %v not exist", transformKeyTypeStr)
		return nil, err
	}
	return create, nil
}

func getDataFromTransformConfig(transformerConfig map[string]interface{}) ([]Data, error) {
	rawData, ok := transformerConfig[KeySampleLog]
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("transformer fetched empty sample log")
		return nil, err
	}

	var data = []Data{}
	var singleData Data
	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &singleData); jsonErr != nil {
		if jsonErr = jsoniter.Unmarshal([]byte(rawDataStr), &data); jsonErr != nil {
			return nil, jsonErr
		}
	} else {
		// is single log, and method transformer.transform(data []sender.Data) accept a param of slice type
		data = append(data, singleData)
	}
	return data, nil
}

func getTransformer(transConfig map[string]interface{}, create transforms.Creator) (transforms.Transformer, error) {
	trans := create()
	transformerConfig := convertWebTransformerConfig(transConfig)
	delete(transformerConfig, KeySampleLog)
	bts, jsonErr := jsoniter.Marshal(transformerConfig)
	if jsonErr != nil {
		return nil, jsonErr
	}
	if jsonErr = jsoniter.Unmarshal(bts, trans); jsonErr != nil {
		return nil, jsonErr
	}

	if trans, ok := trans.(transforms.Initialize); ok {
		if err := trans.Init(); err != nil {
			return nil, err
		}
	}
	return trans, nil
}
<|endoftext|>