<commit_before>package xero

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/gorilla/pat"
	"github.com/markbates/goth"
	"github.com/mrjones/oauth"
	"github.com/stretchr/testify/assert"
)

func Test_New(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()
	a.Equal(provider.ClientKey, os.Getenv("XERO_KEY"))
	a.Equal(provider.Secret, os.Getenv("XERO_SECRET"))
	a.Equal(provider.CallbackURL, "/foo")
}

func Test_Implements_Provider(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	a.Implements((*goth.Provider)(nil), xeroProvider())
}

func Test_BeginAuth(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	mockXero(func(ts *httptest.Server) {
		provider := xeroProvider()
		session, err := provider.BeginAuth("state")
		if err != nil {
			a.Error(err, nil)
		}
		s := session.(*Session)
		a.NoError(err)
		a.Contains(s.AuthURL, "Authorize")
		a.Equal("TOKEN", s.RequestToken.Token)
		a.Equal("SECRET", s.RequestToken.Secret)
	})
}

func Test_FetchUser(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	mockXero(func(ts *httptest.Server) {
		provider := xeroProvider()
		session := Session{AccessToken: &oauth.AccessToken{Token: "TOKEN", Secret: "SECRET"}}

		user, err := provider.FetchUser(&session)
		if err != nil {
			a.Error(err, nil)
		}

		a.NoError(err)

		a.Equal("Vanderlay Industries", user.Name)
		a.Equal("Vanderlay Industries", user.NickName)
		a.Equal("COMPANY", user.Description)
		a.Equal("111-11", user.UserID)
		a.Equal("NZ", user.Location)
		a.Empty(user.Email)
	})
}

func Test_SessionFromJSON(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()

	s, err := provider.UnmarshalSession(`{"AuthURL":"http://com/auth_url","AccessToken":{"Token":"1234567890","Secret":"secret!!","AdditionalData":{}},"RequestToken":{"Token":"0987654321","Secret":"!!secret"}}`)
	a.NoError(err)
	session := s.(*Session)
	a.Equal(session.AuthURL, "http://com/auth_url")
	a.Equal(session.AccessToken.Token, "1234567890")
	a.Equal(session.AccessToken.Secret, "secret!!")
	a.Equal(session.RequestToken.Token, "0987654321")
	a.Equal(session.RequestToken.Secret, "!!secret")
}

func xeroProvider() *Provider {
	return New(os.Getenv("XERO_KEY"), os.Getenv("XERO_SECRET"), "/foo")
}

func mockXero(f func(*httptest.Server)) {
	p := pat.New()
	p.Get("/oauth/RequestToken", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "oauth_token=TOKEN&oauth_token_secret=SECRET")
	})
	p.Get("/oauth/Authorize", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "DO NOT USE THIS ENDPOINT")
	})
	p.Get("/oauth/AccessToken", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "oauth_token=TOKEN&oauth_token_secret=SECRET")
	})
	p.Get("/api.xro/2.0/Organisation", func(res http.ResponseWriter, req *http.Request) {
		apiResponse := APIResponse{
			Organisations: []Organisation{
				{"Vanderlay Industries", "Vanderlay Industries", "COMPANY", "NZ", "111-11"},
			},
		}

		js, err := json.Marshal(apiResponse)
		if err != nil {
			fmt.Fprint(res, "Json did not Marshal")
		}

		res.Write(js)
	})

	ts := httptest.NewServer(p)
	defer ts.Close()

	originalRequestURL := requestURL
	originalEndpointProfile := endpointProfile
	originalAuthorizeURL := authorizeURL
	originalAccessTokenURL := tokenURL

	requestURL = ts.URL + "/oauth/RequestToken"
	endpointProfile = ts.URL + "/api.xro/2.0/"
	authorizeURL = ts.URL + "/oauth/Authorize"
	tokenURL = ts.URL + "/oauth/AccessToken"

	f(ts)

	requestURL = originalRequestURL
	endpointProfile = originalEndpointProfile
	authorizeURL = originalAuthorizeURL
	tokenURL = originalAccessTokenURL
}
<commit_msg>test: use single xero testserver<commit_after>package xero

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/gorilla/pat"
	"github.com/markbates/goth"
	"github.com/mrjones/oauth"
	"github.com/stretchr/testify/assert"
)

func Test_New(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()
	a.Equal(provider.ClientKey, os.Getenv("XERO_KEY"))
	a.Equal(provider.Secret, os.Getenv("XERO_SECRET"))
	a.Equal(provider.CallbackURL, "/foo")
}

func Test_Implements_Provider(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	a.Implements((*goth.Provider)(nil), xeroProvider())
}

func Test_BeginAuth(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()
	session, err := provider.BeginAuth("state")
	if err != nil {
		a.Error(err, nil)
	}
	s := session.(*Session)
	a.NoError(err)
	a.Contains(s.AuthURL, "Authorize")
	a.Equal("TOKEN", s.RequestToken.Token)
	a.Equal("SECRET", s.RequestToken.Secret)
}

func Test_FetchUser(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()
	session := Session{AccessToken: &oauth.AccessToken{Token: "TOKEN", Secret: "SECRET"}}

	user, err := provider.FetchUser(&session)
	if err != nil {
		a.Error(err, nil)
	}

	a.NoError(err)

	a.Equal("Vanderlay Industries", user.Name)
	a.Equal("Vanderlay Industries", user.NickName)
	a.Equal("COMPANY", user.Description)
	a.Equal("111-11", user.UserID)
	a.Equal("NZ", user.Location)
	a.Empty(user.Email)
}

func Test_SessionFromJSON(t *testing.T) {
	t.Parallel()
	a := assert.New(t)

	provider := xeroProvider()

	s, err := provider.UnmarshalSession(`{"AuthURL":"http://com/auth_url","AccessToken":{"Token":"1234567890","Secret":"secret!!","AdditionalData":{}},"RequestToken":{"Token":"0987654321","Secret":"!!secret"}}`)
	a.NoError(err)
	session := s.(*Session)
	a.Equal(session.AuthURL, "http://com/auth_url")
	a.Equal(session.AccessToken.Token, "1234567890")
	a.Equal(session.AccessToken.Secret, "secret!!")
	a.Equal(session.RequestToken.Token, "0987654321")
	a.Equal(session.RequestToken.Secret, "!!secret")
}

func xeroProvider() *Provider {
	return New(os.Getenv("XERO_KEY"), os.Getenv("XERO_SECRET"), "/foo")
}

func init() {
	p := pat.New()
	p.Get("/oauth/RequestToken", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "oauth_token=TOKEN&oauth_token_secret=SECRET")
	})
	p.Get("/oauth/Authorize", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "DO NOT USE THIS ENDPOINT")
	})
	p.Get("/oauth/AccessToken", func(res http.ResponseWriter, req *http.Request) {
		fmt.Fprint(res, "oauth_token=TOKEN&oauth_token_secret=SECRET")
	})
	p.Get("/api.xro/2.0/Organisation", func(res http.ResponseWriter, req *http.Request) {
		apiResponse := APIResponse{
			Organisations: []Organisation{
				{"Vanderlay Industries", "Vanderlay Industries", "COMPANY", "NZ", "111-11"},
			},
		}

		js, err := json.Marshal(apiResponse)
		if err != nil {
			fmt.Fprint(res, "Json did not Marshal")
		}

		res.Write(js)
	})

	ts := httptest.NewServer(p)

	requestURL = ts.URL + "/oauth/RequestToken"
	endpointProfile = ts.URL + "/api.xro/2.0/"
	authorizeURL = ts.URL + "/oauth/Authorize"
	tokenURL = ts.URL + "/oauth/AccessToken"
}
<|endoftext|>


<commit_before>// Very simple sudoku solver to show the use of depth-first
// to solve a problem with plain simple backtracking
package main

import (
	"fmt"
	"github.com/bertbaron/solve"
)

type sudoku struct {
	values   [9][9]int
	position int
}

func newSudoku(values [9][9]int) sudoku {
	return sudoku{values, 0}
}

func (s sudoku) Cost(solve.Context) float64 {
	return float64(s.position)
}

func (s sudoku) Heuristic(solve.Context) float64 {
	return 0
}

func (s sudoku) IsGoal(solve.Context) bool {
	return s.position == 9*9
}

func (s sudoku) Expand(solve.Context) []solve.State {
	var children []solve.State
	for v := 1; v < 10; v++ {
		if child := s.withValue(v); child != nil {
			children = append(children, *child)
		}
	}
	return children
}

// Returns a new sudoku with the value set at the current position,
// or nil if that would be invalid
func (s sudoku) withValue(value int) *sudoku {
	row := s.position / 9
	col := s.position % 9
	current := s.values[row][col]
	copy := s
	copy.position++
	if current == value {
		return &copy // value already filled
	}
	if current != 0 {
		return nil // other value at position
	}
	for i := 0; i < 9; i++ {
		if s.values[row][i] == value || s.values[i][col] == value || s.values[row/3*3+i/3][col/3*3+i%3] == value {
			return nil // row, column or block conflict
		}
	}
	copy.values[row][col] = value
	return &copy
}

func (s sudoku) Print() {
	for _, row := range s.values {
		for _, value := range row {
			if value == 0 {
				fmt.Print("  ")
			} else {
				fmt.Printf(" %d", value)
			}
		}
		fmt.Println()
	}
}

func main() {
	s := newSudoku([9][9]int{
		{0, 0, 6, 9, 0, 5, 0, 0, 2},
		{0, 4, 0, 0, 0, 0, 8, 0, 0},
		{0, 0, 0, 0, 1, 0, 0, 4, 5},
		{0, 8, 0, 6, 0, 4, 7, 0, 0},
		{0, 0, 0, 0, 2, 0, 0, 0, 9},
		{7, 0, 0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 0, 0},
		{0, 0, 1, 0, 0, 7, 0, 6, 0},
		{0, 3, 0, 4, 6, 0, 0, 0, 1}})
	fmt.Println("Solving:")
	s.Print()
	result := solve.NewSolver(s).
		Algorithm(solve.DepthFirst).
		Solve()
	n := len(result.Solution)
	if n == 0 {
		fmt.Println("No solution found")
	} else {
		fmt.Println("Solution:")
		result.Solution[n-1].(sudoku).Print()
	}
}
<commit_msg>Sudoku example<commit_after>// Very simple sudoku solver to show the use of depth-first
// to solve a problem with plain simple backtracking
package main

import (
	"fmt"
	"github.com/bertbaron/solve"
)

type sudoku struct {
	values   [9][9]int
	position int
}

func newSudoku(values [9][9]int) sudoku {
	return sudoku{values, 0}
}

func (s sudoku) Cost(solve.Context) float64 {
	return 0
}

func (s sudoku) Heuristic(solve.Context) float64 {
	return 0
}

func (s sudoku) IsGoal(solve.Context) bool {
	return s.position == 9*9
}

func (s sudoku) Expand(solve.Context) []solve.State {
	var children []solve.State
	for v := 1; v <= 9; v++ {
		if child := s.withValue(v); child != nil {
			children = append(children, *child)
		}
	}
	return children
}

// Returns a new sudoku with the value set at the current position,
// or nil if that would be invalid
func (s sudoku) withValue(value int) *sudoku {
	row := s.position / 9
	col := s.position % 9
	current := s.values[row][col]
	copy := s
	copy.position++
	if current == value {
		return &copy // value already filled
	}
	if current != 0 {
		return nil // other value at position
	}
	for i := 0; i < 9; i++ {
		if s.values[row][i] == value || s.values[i][col] == value || s.values[row/3*3+i/3][col/3*3+i%3] == value {
			return nil // row, column or block conflict
		}
	}
	copy.values[row][col] = value
	return &copy
}

func (s sudoku) Print() {
	for _, row := range s.values {
		for _, value := range row {
			if value == 0 {
				fmt.Print("  ")
			} else {
				fmt.Printf(" %d", value)
			}
		}
		fmt.Println()
	}
}

func main() {
	s := newSudoku([9][9]int{
		{0, 0, 6, 9, 0, 5, 0, 0, 2},
		{0, 4, 0, 0, 0, 0, 8, 0, 0},
		{0, 0, 0, 0, 1, 0, 0, 4, 5},
		{0, 8, 0, 6, 0, 4, 7, 0, 0},
		{0, 0, 0, 0, 2, 0, 0, 0, 9},
		{7, 0, 0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 0, 0},
		{0, 0, 1, 0, 0, 7, 0, 6, 0},
		{0, 3, 0, 4, 6, 0, 0, 0, 1}})
	fmt.Println("Solving:")
	s.Print()
	result := solve.NewSolver(s).
		Algorithm(solve.DepthFirst).
		Solve()
	n := len(result.Solution)
	if n == 0 {
		fmt.Println("No solution found")
	} else {
		fmt.Println("Solution:")
		result.Solution[n-1].(sudoku).Print()
	}
}
<|endoftext|>


<commit_before>// +build !confonly

package shadowsocks

import (
	"context"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol"
	udp_proto "v2ray.com/core/common/protocol/udp"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/common/task"
	"v2ray.com/core/features/policy"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/internet/udp"
)

type Server struct {
	config        ServerConfig
	user          *protocol.MemoryUser
	policyManager policy.Manager
}

// NewServer create a new Shadowsocks server.
func NewServer(ctx context.Context, config *ServerConfig) (*Server, error) {
	if config.GetUser() == nil {
		return nil, newError("user is not specified")
	}

	mUser, err := config.User.ToMemoryUser()
	if err != nil {
		return nil, newError("failed to parse user account").Base(err)
	}

	v := core.MustFromContext(ctx)
	s := &Server{
		config:        *config,
		user:          mUser,
		policyManager: v.GetFeature(policy.ManagerType()).(policy.Manager),
	}

	return s, nil
}

func (s *Server) Network() []net.Network {
	list := s.config.Network
	if len(list) == 0 {
		list = append(list, net.Network_TCP)
	}
	if s.config.UdpEnabled {
		list = append(list, net.Network_UDP)
	}
	return list
}

func (s *Server) Process(ctx context.Context, network net.Network, conn internet.Connection, dispatcher routing.Dispatcher) error {
	switch network {
	case net.Network_TCP:
		return s.handleConnection(ctx, conn, dispatcher)
	case net.Network_UDP:
		return s.handlerUDPPayload(ctx, conn, dispatcher)
	default:
		return newError("unknown network: ", network)
	}
}

func (s *Server) handlerUDPPayload(ctx context.Context, conn internet.Connection, dispatcher routing.Dispatcher) error {
	udpServer := udp.NewDispatcher(dispatcher, func(ctx context.Context, packet *udp_proto.Packet) {
		request := protocol.RequestHeaderFromContext(ctx)
		if request == nil {
			return
		}

		payload := packet.Payload
		data, err := EncodeUDPPacket(request, payload.Bytes())
		payload.Release()
		if err != nil {
			newError("failed to encode UDP packet").Base(err).AtWarning().WriteToLog(session.ExportIDToError(ctx))
			return
		}
		defer data.Release()

		conn.Write(data.Bytes())
	})

	account := s.user.Account.(*MemoryAccount)
	inbound := session.InboundFromContext(ctx)
	if inbound == nil {
		panic("no inbound metadata")
	}
	inbound.User = s.user

	reader := buf.NewPacketReader(conn)
	for {
		mpayload, err := reader.ReadMultiBuffer()
		if err != nil {
			break
		}

		for _, payload := range mpayload {
			request, data, err := DecodeUDPPacket(s.user, payload)
			if err != nil {
				if inbound := session.InboundFromContext(ctx); inbound != nil && inbound.Source.IsValid() {
					newError("dropping invalid UDP packet from: ", inbound.Source).Base(err).WriteToLog(session.ExportIDToError(ctx))
					log.Record(&log.AccessMessage{
						From:   inbound.Source,
						To:     "",
						Status: log.AccessRejected,
						Reason: err,
					})
				}
				payload.Release()
				continue
			}

			if request.Option.Has(RequestOptionOneTimeAuth) && account.OneTimeAuth == Account_Disabled {
				newError("client payload enables OTA but server doesn't allow it").WriteToLog(session.ExportIDToError(ctx))
				payload.Release()
				continue
			}

			if !request.Option.Has(RequestOptionOneTimeAuth) && account.OneTimeAuth == Account_Enabled {
				newError("client payload disables OTA but server forces it").WriteToLog(session.ExportIDToError(ctx))
				payload.Release()
				continue
			}

			dest := request.Destination()
			if inbound.Source.IsValid() {
				log.Record(&log.AccessMessage{
					From:   inbound.Source,
					To:     dest,
					Status: log.AccessAccepted,
					Reason: "",
					Email:  request.User.Email,
				})
			}
			newError("tunnelling request to ", dest).WriteToLog(session.ExportIDToError(ctx))

			ctx = protocol.ContextWithRequestHeader(ctx, request)
			udpServer.Dispatch(ctx, dest, data)
		}
	}

	return nil
}

func (s *Server) handleConnection(ctx context.Context, conn internet.Connection, dispatcher routing.Dispatcher) error {
	sessionPolicy := s.policyManager.ForLevel(s.user.Level)
	conn.SetReadDeadline(time.Now().Add(sessionPolicy.Timeouts.Handshake))

	bufferedReader := buf.BufferedReader{Reader: buf.NewReader(conn)}
	request, bodyReader, err := ReadTCPSession(s.user, &bufferedReader)
	if err != nil {
		log.Record(&log.AccessMessage{
			From:   conn.RemoteAddr(),
			To:     "",
			Status: log.AccessRejected,
			Reason: err,
			Email:  request.User.Email,
		})
		return newError("failed to create request from: ", conn.RemoteAddr()).Base(err)
	}
	conn.SetReadDeadline(time.Time{})

	inbound := session.InboundFromContext(ctx)
	if inbound == nil {
		panic("no inbound metadata")
	}
	inbound.User = s.user

	dest := request.Destination()
	log.Record(&log.AccessMessage{
		From:   conn.RemoteAddr(),
		To:     dest,
		Status: log.AccessAccepted,
		Reason: "",
		Email:  request.User.Email,
	})
	newError("tunnelling request to ", dest).WriteToLog(session.ExportIDToError(ctx))

	ctx, cancel := context.WithCancel(ctx)
	timer := signal.CancelAfterInactivity(ctx, cancel, sessionPolicy.Timeouts.ConnectionIdle)

	ctx = policy.ContextWithBufferPolicy(ctx, sessionPolicy.Buffer)
	link, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}

	responseDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.UplinkOnly)

		bufferedWriter := buf.NewBufferedWriter(buf.NewWriter(conn))
		responseWriter, err := WriteTCPResponse(request, bufferedWriter)
		if err != nil {
			return newError("failed to write response").Base(err)
		}

		{
			payload, err := link.Reader.ReadMultiBuffer()
			if err != nil {
				return err
			}
			if err := responseWriter.WriteMultiBuffer(payload); err != nil {
				return err
			}
		}

		if err := bufferedWriter.SetBuffered(false); err != nil {
			return err
		}

		if err := buf.Copy(link.Reader, responseWriter, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transport all TCP response").Base(err)
		}

		return nil
	}

	requestDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.DownlinkOnly)

		if err := buf.Copy(bodyReader, link.Writer, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transport all TCP request").Base(err)
		}

		return nil
	}

	var requestDoneAndCloseWriter = task.OnSuccess(requestDone, task.Close(link.Writer))
	if err := task.Run(ctx, requestDoneAndCloseWriter, responseDone); err != nil {
		common.Interrupt(link.Reader)
		common.Interrupt(link.Writer)
		return newError("connection ends").Base(err)
	}

	return nil
}

func init() {
	common.Must(common.RegisterConfig((*ServerConfig)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return NewServer(ctx, config.(*ServerConfig))
	}))
}
<commit_msg>fix a crash problem of fetching user from a nil request which returned by shadowsocks failed to read IV<commit_after>// +build !confonly

package shadowsocks

import (
	"context"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol"
	udp_proto "v2ray.com/core/common/protocol/udp"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/common/task"
	"v2ray.com/core/features/policy"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/internet/udp"
)

type Server struct {
	config        ServerConfig
	user          *protocol.MemoryUser
	policyManager policy.Manager
}

// NewServer create a new Shadowsocks server.
func NewServer(ctx context.Context, config *ServerConfig) (*Server, error) {
	if config.GetUser() == nil {
		return nil, newError("user is not specified")
	}

	mUser, err := config.User.ToMemoryUser()
	if err != nil {
		return nil, newError("failed to parse user account").Base(err)
	}

	v := core.MustFromContext(ctx)
	s := &Server{
		config:        *config,
		user:          mUser,
		policyManager: v.GetFeature(policy.ManagerType()).(policy.Manager),
	}

	return s, nil
}

func (s *Server) Network() []net.Network {
	list := s.config.Network
	if len(list) == 0 {
		list = append(list, net.Network_TCP)
	}
	if s.config.UdpEnabled {
		list = append(list, net.Network_UDP)
	}
	return list
}

func (s *Server) Process(ctx context.Context, network net.Network, conn internet.Connection, dispatcher routing.Dispatcher) error {
	switch network {
	case net.Network_TCP:
		return s.handleConnection(ctx, conn, dispatcher)
	case net.Network_UDP:
		return s.handlerUDPPayload(ctx, conn, dispatcher)
	default:
		return newError("unknown network: ", network)
	}
}

func (s *Server) handlerUDPPayload(ctx context.Context, conn internet.Connection, dispatcher routing.Dispatcher) error {
	udpServer := udp.NewDispatcher(dispatcher, func(ctx context.Context, packet *udp_proto.Packet) {
		request := protocol.RequestHeaderFromContext(ctx)
		if request == nil {
			return
		}

		payload := packet.Payload
		data, err := EncodeUDPPacket(request, payload.Bytes())
		payload.Release()
		if err != nil {
			newError("failed to encode UDP packet").Base(err).AtWarning().WriteToLog(session.ExportIDToError(ctx))
			return
		}
		defer data.Release()

		conn.Write(data.Bytes())
	})

	account := s.user.Account.(*MemoryAccount)
	inbound := session.InboundFromContext(ctx)
	if inbound == nil {
		panic("no inbound metadata")
	}
	inbound.User = s.user

	reader := buf.NewPacketReader(conn)
	for {
		mpayload, err := reader.ReadMultiBuffer()
		if err != nil {
			break
		}

		for _, payload := range mpayload {
			request, data, err := DecodeUDPPacket(s.user, payload)
			if err != nil {
				if inbound := session.InboundFromContext(ctx); inbound != nil && inbound.Source.IsValid() {
					newError("dropping invalid UDP packet from: ", inbound.Source).Base(err).WriteToLog(session.ExportIDToError(ctx))
					log.Record(&log.AccessMessage{
						From:   inbound.Source,
						To:     "",
						Status: log.AccessRejected,
						Reason: err,
					})
				}
				payload.Release()
				continue
			}

			if request.Option.Has(RequestOptionOneTimeAuth) && account.OneTimeAuth == Account_Disabled {
				newError("client payload enables OTA but server doesn't allow it").WriteToLog(session.ExportIDToError(ctx))
				payload.Release()
				continue
			}

			if !request.Option.Has(RequestOptionOneTimeAuth) && account.OneTimeAuth == Account_Enabled {
				newError("client payload disables OTA but server forces it").WriteToLog(session.ExportIDToError(ctx))
				payload.Release()
				continue
			}

			dest := request.Destination()
			if inbound.Source.IsValid() {
				log.Record(&log.AccessMessage{
					From:   inbound.Source,
					To:     dest,
					Status: log.AccessAccepted,
					Reason: "",
					Email:  request.User.Email,
				})
			}
			newError("tunnelling request to ", dest).WriteToLog(session.ExportIDToError(ctx))

			ctx = protocol.ContextWithRequestHeader(ctx, request)
			udpServer.Dispatch(ctx, dest, data)
		}
	}

	return nil
}

func (s *Server) handleConnection(ctx context.Context, conn internet.Connection, dispatcher routing.Dispatcher) error {
	sessionPolicy := s.policyManager.ForLevel(s.user.Level)
	conn.SetReadDeadline(time.Now().Add(sessionPolicy.Timeouts.Handshake))

	bufferedReader := buf.BufferedReader{Reader: buf.NewReader(conn)}
	request, bodyReader, err := ReadTCPSession(s.user, &bufferedReader)
	if err != nil {
		log.Record(&log.AccessMessage{
			From:   conn.RemoteAddr(),
			To:     "",
			Status: log.AccessRejected,
			Reason: err,
		})
		return newError("failed to create request from: ", conn.RemoteAddr()).Base(err)
	}
	conn.SetReadDeadline(time.Time{})

	inbound := session.InboundFromContext(ctx)
	if inbound == nil {
		panic("no inbound metadata")
	}
	inbound.User = s.user

	dest := request.Destination()
	log.Record(&log.AccessMessage{
		From:   conn.RemoteAddr(),
		To:     dest,
		Status: log.AccessAccepted,
		Reason: "",
		Email:  request.User.Email,
	})
	newError("tunnelling request to ", dest).WriteToLog(session.ExportIDToError(ctx))

	ctx, cancel := context.WithCancel(ctx)
	timer := signal.CancelAfterInactivity(ctx, cancel, sessionPolicy.Timeouts.ConnectionIdle)

	ctx = policy.ContextWithBufferPolicy(ctx, sessionPolicy.Buffer)
	link, err := dispatcher.Dispatch(ctx, dest)
	if err != nil {
		return err
	}

	responseDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.UplinkOnly)

		bufferedWriter := buf.NewBufferedWriter(buf.NewWriter(conn))
		responseWriter, err := WriteTCPResponse(request, bufferedWriter)
		if err != nil {
			return newError("failed to write response").Base(err)
		}

		{
			payload, err := link.Reader.ReadMultiBuffer()
			if err != nil {
				return err
			}
			if err := responseWriter.WriteMultiBuffer(payload); err != nil {
				return err
			}
		}

		if err := bufferedWriter.SetBuffered(false); err != nil {
			return err
		}

		if err := buf.Copy(link.Reader, responseWriter, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transport all TCP response").Base(err)
		}

		return nil
	}

	requestDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.DownlinkOnly)

		if err := buf.Copy(bodyReader, link.Writer, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transport all TCP request").Base(err)
		}

		return nil
	}

	var requestDoneAndCloseWriter = task.OnSuccess(requestDone, task.Close(link.Writer))
	if err := task.Run(ctx, requestDoneAndCloseWriter, responseDone); err != nil {
		common.Interrupt(link.Reader)
		common.Interrupt(link.Writer)
		return newError("connection ends").Base(err)
	}

	return nil
}

func init() {
	common.Must(common.RegisterConfig((*ServerConfig)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return NewServer(ctx, config.(*ServerConfig))
	}))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/gookit/color"
)

// First message when beginning the AES Installation process
func (i *Installer) ShowFirstInstalling() {
	i.show.Println()
	i.show.Println(color.Bold.Sprintf("Installing the Ambassador Edge Stack"))
}

func (i *Installer) ShowScoutDisabled() {
	i.show.Printf("INFO: phone-home is disabled by environment variable")
}

func (i *Installer) ShowRequestEmail() {
	i.show.Println()
	i.ShowWrapped("Please enter an email address for us to notify you before your TLS certificate and domain name expire. In order to acquire the TLS certificate, we share this email with Let’s Encrypt.")
}

func (i *Installer) ShowACMEFailed(reason string) {
	i.show.Println()
	i.show.Println(color.Bold.Sprintf("Acquiring TLS certificate via ACME has failed: %s", reason))
}

func (i *Installer) ShowBeginAESInstallation() {
	i.show.Println()
	i.show.Println("========================================================================")
	i.show.Println("Beginning Ambassador Edge Stack Installation")
}

func (i *Installer) ShowFailedWhenLookingForExistingVersion() {
	i.show.Println("-> Failed when looking for an existing installation")
}

func (i *Installer) ShowAESVersionBeingInstalled() {
	i.show.Println(fmt.Sprintf("-> Installing the Ambassador Edge Stack %s.", i.version))
}

func (i *Installer) ShowAESExistingVersion(versionName string, method string) {
	i.show.Println(fmt.Sprintf("-> Ambassador Edge Stack %s was already installed using %s", versionName, method))
}

func (i *Installer) FindingRepositoriesAndVersions() {
	i.show.Println("-> Finding repositories and chart versions")
}

func (i *Installer) ShowOverridingChartVersion(aesChartVersion string, version string) {
	i.show.Println(fmt.Sprintf("   Overriding chart version rule using %s = %s", aesChartVersion, version))
}

func (i *Installer) ShowOverridingImageRepo(aesImageRepo string, repo string) {
	i.show.Println(fmt.Sprintf("   Overriding image repo using %s = %s", aesImageRepo, repo))
}

func (i *Installer) ShowOverridingImageTag(aesImageTag string, tag string) {
	i.show.Println(fmt.Sprintf("   Overriding image tag using %s = %s", aesImageTag, tag))
}

func (i *Installer) ShowOverridingHelmRepo(aesHelmRepo string, repo string) {
	i.show.Println(fmt.Sprintf("   Overriding Helm repo using %s = %s", aesHelmRepo, repo))
}

func (i *Installer) ShowAESCRDsButNoAESInstallation() {
	i.show.Println("-> Found Ambassador CRDs in your cluster, but no Edge Stack installation.")
}

func (i *Installer) ShowDownloadingImages() {
	i.show.Println("-> Downloading latest version")
}

func (i *Installer) ShowInstalling(version string) {
	i.show.Println(fmt.Sprintf("-> Installing Ambassador Edge Stack %s", version))
}

func (i *Installer) ShowCheckingAESPodDeployment() {
	i.show.Println("-> Checking the AES pod deployment")
}

func (i *Installer) ShowLocalClusterDetected() {
	i.show.Println("-> Local cluster detected. Not configuring automatic TLS.")
}

func (i *Installer) ShowProvisioningLoadBalancer() {
	i.show.Println("-> Provisioning a cloud load balancer")
}

func (i *Installer) ShowAESInstallAddress(address string) {
	i.show.Println("-> Your Ambassador Edge Stack installation's address is", color.Bold.Sprintf(address))
}

func (i *Installer) ShowAESRespondingToACME() {
	i.show.Println("-> Checking that Ambassador Edge Stack is responding to ACME challenge")
}

func (i *Installer) ShowWaiting(what string) {
	i.show.Printf("   Still waiting for %s. (This may take a minute.)", what)
}

func (i *Installer) ShowTimedOut(what string) {
	i.show.Printf("   Timed out waiting for %s (or interrupted)", what)
}

func (i *Installer) ShowAESConfiguringTLS() {
	i.show.Println("-> Automatically configuring TLS")
}

func (i *Installer) ShowFailedToCreateDNSName(message string) {
	i.show.Println("   Failed to create a DNS name:", message)
}

func (i *Installer) ShowAcquiringDNSName(hostname string) {
	i.show.Println("-> Acquiring DNS name", color.Bold.Sprintf(hostname))
}

func (i *Installer) ShowObtainingTLSCertificate() {
	i.show.Println("-> Obtaining a TLS certificate from Let's Encrypt")
}

func (i *Installer) ShowTLSConfiguredSuccessfully() {
	i.show.Println("-> TLS configured successfully")
}

// AES installation complete, but no DNS.
func (i *Installer) ShowAESInstallationCompleteNoDNS() {
	i.show.Println()
	i.show.Println("Ambassador Edge Stack Installation Complete!")
	i.show.Println("========================================================================")

	// Show congratulations message
	i.show.Println()
	message := "<bold>Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically.</>\n\n"
	message += "If the IP address is reachable from your computer, you can access your installation without a DNS name. The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.\n"
	message += "<bold>$ edgectl login -n ambassador {{ .address }}</>\n\n"
	i.ShowTemplated(message)
	i.show.Println()
}

// AES installation complete!
func (i *Installer) ShowAESInstallationComplete() {
	i.show.Println()
	i.show.Println("Ambassador Edge Stack Installation Complete!")
	i.show.Println("========================================================================")

	// Show congratulations message
	i.show.Println()
	message := color.Bold.Sprintf("Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. You can find it at your custom URL: https://{{.hostname}}/")
	i.ShowTemplated(message)
	i.show.Println()
}
<commit_msg>checkpoint<commit_after>package main

import (
	"fmt"

	"github.com/gookit/color"
)

// First message when beginning the AES Installation process
func (i *Installer) ShowFirstInstalling() {
	i.show.Println()
	i.show.Println(color.Bold.Sprintf("Installing the Ambassador Edge Stack"))
}

func (i *Installer) ShowScoutDisabled() {
	i.show.Printf("INFO: phone-home is disabled by environment variable")
}

func (i *Installer) ShowRequestEmail() {
	i.show.Println()
	i.ShowWrapped("Please enter an email address for us to notify you before your TLS certificate and domain name expire. In order to acquire the TLS certificate, we share this email with Let’s Encrypt.")
}

func (i *Installer) ShowACMEFailed(reason string) {
	i.show.Println()
	i.show.Println(color.Bold.Sprintf("Acquiring TLS certificate via ACME has failed: %s", reason))
}

func (i *Installer) ShowBeginAESInstallation() {
	i.show.Println()
	i.show.Println("========================================================================")
	i.show.Println("Beginning Ambassador Edge Stack Installation")
}

func (i *Installer) ShowFailedWhenLookingForExistingVersion() {
	i.show.Println("-> Failed when looking for an existing installation")
}

func (i *Installer) ShowAESVersionBeingInstalled() {
	i.show.Println(fmt.Sprintf("-> Installing the Ambassador Edge Stack %s.", i.version))
}

func (i *Installer) ShowAESExistingVersion(versionName string, method string) {
	i.show.Println(fmt.Sprintf("-> Ambassador Edge Stack %s was already installed using %s", versionName, method))
}

func (i *Installer) FindingRepositoriesAndVersions() {
	i.show.Println("-> Finding repositories and chart versions")
}

func (i *Installer) ShowOverridingChartVersion(aesChartVersion string, version string) {
	i.show.Println(fmt.Sprintf("   Overriding chart version rule using %s = %s", aesChartVersion, version))
}

func (i *Installer) ShowOverridingImageRepo(aesImageRepo string, repo string) {
	i.show.Println(fmt.Sprintf("   Overriding image repo using %s = %s", aesImageRepo, repo))
}

func (i *Installer) ShowOverridingImageTag(aesImageTag string, tag string) {
	i.show.Println(fmt.Sprintf("   Overriding image tag using %s = %s", aesImageTag, tag))
}

func (i *Installer) ShowOverridingHelmRepo(aesHelmRepo string, repo string) {
	i.show.Println(fmt.Sprintf("   Overriding Helm repo using %s = %s", aesHelmRepo, repo))
}

func (i *Installer) ShowAESCRDsButNoAESInstallation() {
	i.show.Println("-> Found Ambassador CRDs in your cluster, but no Edge Stack installation.")
}

func (i *Installer) ShowDownloadingImages() {
	i.show.Println("-> Downloading latest version")
}

func (i *Installer) ShowInstalling(version string) {
	i.show.Println(fmt.Sprintf("-> Installing Ambassador Edge Stack %s", version))
}

func (i *Installer) ShowCheckingAESPodDeployment() {
	i.show.Println("-> Checking the AES pod deployment")
}

func (i *Installer) ShowLocalClusterDetected() {
	i.show.Println("-> Local cluster detected. Not configuring automatic TLS.")
}

func (i *Installer) ShowProvisioningLoadBalancer() {
	i.show.Println("-> Provisioning a cloud load balancer")
}

func (i *Installer) ShowAESInstallAddress(address string) {
	i.show.Println("-> Your Ambassador Edge Stack installation's address is", color.Bold.Sprintf(address))
}

func (i *Installer) ShowAESRespondingToACME() {
	i.show.Println("-> Checking that Ambassador Edge Stack is responding to ACME challenge")
}

func (i *Installer) ShowWaiting(what string) {
	i.show.Printf("   Still waiting for %s. (This may take a minute.)", what)
}

func (i *Installer) ShowTimedOut(what string) {
	i.show.Printf("   Timed out waiting for %s (or interrupted)", what)
}

func (i *Installer) ShowAESConfiguringTLS() {
	i.show.Println("-> Automatically configuring TLS")
}

func (i *Installer) ShowFailedToCreateDNSName(message string) {
	i.show.Println("   Failed to create a DNS name:", message)
}

func (i *Installer) ShowAcquiringDNSName(hostname string) {
	i.show.Println("-> Acquiring DNS name", color.Bold.Sprintf(hostname))
}

func (i *Installer) ShowObtainingTLSCertificate() {
	i.show.Println("-> Obtaining a TLS certificate from Let's Encrypt")
}

func (i *Installer) ShowTLSConfiguredSuccessfully() {
	i.show.Println("-> TLS configured successfully")
}

// AES installation partially complete -- other instructions follow.
func (i *Installer) ShowAESInstallationPartiallyComplete() {
	i.show.Println()
	i.show.Println("Ambassador Edge Stack Installation Complete!")
	i.show.Println("========================================================================")
}

// AES installation complete, but no DNS.
func (i *Installer) ShowAESInstallationCompleteNoDNS() {
	i.show.Println()
	i.show.Println("Ambassador Edge Stack Installation Complete!")
	i.show.Println("========================================================================")

	// Show congratulations message
	i.show.Println()
	message := "<bold>Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically.</>\n\n"
	message += "If the IP address is reachable from your computer, you can access your installation without a DNS name. The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.\n"
	message += "<bold>$ edgectl login -n ambassador {{ .address }}</>\n\n"
	i.ShowTemplated(message)
	i.show.Println()
}

// AES installation complete!
func (i *Installer) ShowAESInstallationComplete() {
	i.show.Println()
	i.show.Println("Ambassador Edge Stack Installation Complete!")
	i.show.Println("========================================================================")

	// Show congratulations message
	i.show.Println()
	message := color.Bold.Sprintf("Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. You can find it at your custom URL: https://{{.hostname}}/")
	i.ShowTemplated(message)
	i.show.Println()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type ARIN_OrgNets struct {
	Nets struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		LimitExceeded       struct {
			Limit string `json:"@limit"`
			Value string `json:"$"`
		} `json:"limitExceeded"`
		NetRef []struct {
			EndAddress   string `json:"@endAddress"`
			StartAddress string `json:"@startAddress"`
			Handle       string `json:"@handle"`
			Name         string `json:"@name"`
			Value        string `json:"$"`
		} `json:"netRef"`
	} `json:"nets"`
}

type ARIN_OrgNet struct {
	Nets struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		LimitExceeded       struct {
			Limit string `json:"@limit"`
			Value string `json:"$"`
		} `json:"limitExceeded"`
		NetRef struct {
			EndAddress   string `json:"@endAddress"`
			StartAddress string `json:"@startAddress"`
			Handle       string `json:"@handle"`
			Name         string `json:"@name"`
			Value        string `json:"$"`
		} `json:"netRef"`
	} `json:"nets"`
}

type ARIN_Nets struct {
	Net struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		RegistrationDate    struct {
			Value string `json:"$"`
		} `json:"registrationDate"`
		Ref struct {
			Value string `json:"$"`
		} `json:"ref"`
		EndAddress struct {
			Value string `json:"$"`
		} `json:"endAddress"`
		Handle struct {
			Value string `json:"$"`
		} `json:"handle"`
		Name struct {
			Value string `json:"$"`
		} `json:"name"`
		NetBlocks []struct {
			NetBlock struct {
				CidrLength struct {
					Value string `json:"$"`
				} `json:"cidrLength"`
				EndAddress struct {
					Value string `json:"$"`
				} `json:"endAddress"`
				Description struct {
					Value string `json:"$"`
				} `json:"description"`
				Type struct {
					Value string `json:"$"`
				} `json:"type"`
				StartAddress struct {
					Value string `json:"$"`
				} `json:"startAddress"`
			} `json:"netBlock"`
		} `json:"netBlocks"`
		Resources struct {
			InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
			TermsOfUse          string `json:"@termsOfUse"`
			LimitExceeded       struct {
				Limit string `json:"@limit"`
				Value string `json:"$"`
			} `json:"limitExceeded"`
		} `json:"resources"`
		OrgRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"orgRef"`
		ParentNetRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"parentNetRef"`
		StartAddress struct {
			Value string `json:"$"`
		} `json:"startAddress"`
		UpdateDate struct {
			Value string `json:"$"`
		} `json:"updateDate"`
		Version struct {
			Value string `json:"$"`
		} `json:"version"`
	} `json:"net"`
}

type ARIN_Net struct {
	Net struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		RegistrationDate    struct {
			Value string `json:"$"`
		} `json:"registrationDate"`
		Ref struct {
			Value string `json:"$"`
		} `json:"ref"`
		EndAddress struct {
			Value string `json:"$"`
		} `json:"endAddress"`
		Handle struct {
			Value string `json:"$"`
		} `json:"handle"`
		Name struct {
			Value string `json:"$"`
		} `json:"name"`
		NetBlocks struct {
			NetBlock struct {
				CidrLength struct {
					Value string `json:"$"`
				} `json:"cidrLength"`
				EndAddress struct {
					Value string `json:"$"`
				} `json:"endAddress"`
				Description struct {
					Value string `json:"$"`
				} `json:"description"`
				Type struct {
					Value string `json:"$"`
				} `json:"type"`
				StartAddress struct {
					Value string `json:"$"`
				} `json:"startAddress"`
			} `json:"netBlock"`
		} `json:"netBlocks"`
		Resources struct {
			InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
			TermsOfUse          string `json:"@termsOfUse"`
			LimitExceeded       struct {
				Limit string `json:"@limit"`
				Value string `json:"$"`
			} `json:"limitExceeded"`
		} `json:"resources"`
		OrgRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"orgRef"`
		ParentNetRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"parentNetRef"`
		StartAddress struct {
			Value string `json:"$"`
		} `json:"startAddress"`
		UpdateDate struct {
			Value string `json:"$"`
		} `json:"updateDate"`
		Version struct {
			Value string `json:"$"`
		} `json:"version"`
	} `json:"net"`
}

func LookupOrgNets(org string) ([]string, error) {
	handles := []string{}

	safe_org := url.QueryEscape(org)
	u := fmt.Sprintf("http://whois.arin.net/rest/org/%s/nets", safe_org)

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return handles, err
	}
	req.Header.Set("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return handles, err
	}

	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return handles, err
	}

	var nets ARIN_OrgNets

	if err := json.Unmarshal(content, &nets); err == nil {
		for i := range nets.Nets.NetRef {
			handles = append(handles, nets.Nets.NetRef[i].Handle)
		}
	} else {
		// Try to decode as a single-net organization
		var net ARIN_OrgNet
		if err := json.Unmarshal(content, &net); err != nil {
			return handles, err
		}

		handles = append(handles, net.Nets.NetRef.Handle)
	}

	return handles, nil
}

func LookupNetCidrs(handle string) ([]string, error) {
	cidrs := []string{}

	safe_handle := url.QueryEscape(handle)
	u := fmt.Sprintf("http://whois.arin.net/rest/net/%s", safe_handle)

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return cidrs, err
	}
	req.Header.Set("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return cidrs, err
	}

	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return cidrs, err
	}

	var nets ARIN_Nets

	if err := json.Unmarshal(content, &nets); err == nil {
		for i := range nets.Net.NetBlocks {
			cidrs = append(cidrs, fmt.Sprintf("%s/%s", nets.Net.NetBlocks[i].NetBlock.StartAddress.Value, nets.Net.NetBlocks[i].NetBlock.CidrLength.Value))
		}
	} else {
		// Try to decode as a single-block network
		var net ARIN_Net
		if err := json.Unmarshal(content, &net); err != nil {
			return cidrs, err
		}
		cidrs = append(cidrs, fmt.Sprintf("%s/%s", net.Net.NetBlocks.NetBlock.StartAddress.Value, net.Net.NetBlocks.NetBlock.CidrLength.Value))
	}

	return cidrs, nil
}

func main() {

	if len(os.Args) != 2 {
		fmt.Println("Usage: inetdata-arin-org2nets <org-handle>\n")
		os.Exit(1)
	}

	org := os.Args[1]

	handles, he := LookupOrgNets(org)
	if he != nil {
		log.Fatal("Error: ", he)
		os.Exit(1)
	}

	for i := range handles {
		cidrs, e := LookupNetCidrs(handles[i])
		if e != nil {
			log.Fatal("Error: ", e)
			os.Exit(1)
		}
		fmt.Println(strings.Join(cidrs, "\n"))
	}
}
<commit_msg>Handle orgs without network handles gracefully<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type ARIN_OrgNets struct {
	Nets struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		LimitExceeded       struct {
			Limit string `json:"@limit"`
			Value string `json:"$"`
		} `json:"limitExceeded"`
		NetRef []struct {
			EndAddress   string `json:"@endAddress"`
			StartAddress string `json:"@startAddress"`
			Handle       string `json:"@handle"`
			Name         string `json:"@name"`
			Value        string `json:"$"`
		} `json:"netRef"`
	} `json:"nets"`
}

type ARIN_OrgNet struct {
	Nets struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		LimitExceeded       struct {
			Limit string `json:"@limit"`
			Value string `json:"$"`
		} `json:"limitExceeded"`
		NetRef struct {
			EndAddress   string `json:"@endAddress"`
			StartAddress string `json:"@startAddress"`
			Handle       string `json:"@handle"`
			Name         string `json:"@name"`
			Value        string `json:"$"`
		} `json:"netRef"`
	} `json:"nets"`
}

type ARIN_Nets struct {
	Net struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		RegistrationDate    struct {
			Value string `json:"$"`
		} `json:"registrationDate"`
		Ref struct {
			Value string `json:"$"`
		} `json:"ref"`
		EndAddress struct {
			Value string `json:"$"`
		} `json:"endAddress"`
		Handle struct {
			Value string `json:"$"`
		} `json:"handle"`
		Name struct {
			Value string `json:"$"`
		} `json:"name"`
		NetBlocks []struct {
			NetBlock struct {
				CidrLength struct {
					Value string `json:"$"`
				} `json:"cidrLength"`
				EndAddress struct {
					Value string `json:"$"`
				} `json:"endAddress"`
				Description struct {
					Value string `json:"$"`
				} `json:"description"`
				Type struct {
					Value string `json:"$"`
				} `json:"type"`
				StartAddress struct {
					Value string `json:"$"`
				} `json:"startAddress"`
			} `json:"netBlock"`
		} `json:"netBlocks"`
		Resources struct {
			InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
			TermsOfUse          string `json:"@termsOfUse"`
			LimitExceeded       struct {
				Limit string `json:"@limit"`
				Value string `json:"$"`
			} `json:"limitExceeded"`
		} `json:"resources"`
		OrgRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"orgRef"`
		ParentNetRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"parentNetRef"`
		StartAddress struct {
			Value string `json:"$"`
		} `json:"startAddress"`
		UpdateDate struct {
			Value string `json:"$"`
		} `json:"updateDate"`
		Version struct {
			Value string `json:"$"`
		} `json:"version"`
	} `json:"net"`
}

type ARIN_Net struct {
	Net struct {
		InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
		TermsOfUse          string `json:"@termsOfUse"`
		RegistrationDate    struct {
			Value string `json:"$"`
		} `json:"registrationDate"`
		Ref struct {
			Value string `json:"$"`
		} `json:"ref"`
		EndAddress struct {
			Value string `json:"$"`
		} `json:"endAddress"`
		Handle struct {
			Value string `json:"$"`
		} `json:"handle"`
		Name struct {
			Value string `json:"$"`
		} `json:"name"`
		NetBlocks struct {
			NetBlock struct {
				CidrLength struct {
					Value string `json:"$"`
				} `json:"cidrLength"`
				EndAddress struct {
					Value string `json:"$"`
				} `json:"endAddress"`
				Description struct {
					Value string `json:"$"`
				} `json:"description"`
				Type struct {
					Value string `json:"$"`
				} `json:"type"`
				StartAddress struct {
					Value string `json:"$"`
				} `json:"startAddress"`
			} `json:"netBlock"`
		} `json:"netBlocks"`
		Resources struct {
			InaccuracyReportURL string `json:"@inaccuracyReportUrl"`
			TermsOfUse          string `json:"@termsOfUse"`
			LimitExceeded       struct {
				Limit string `json:"@limit"`
				Value string `json:"$"`
			} `json:"limitExceeded"`
		} `json:"resources"`
		OrgRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"orgRef"`
		ParentNetRef struct {
			Handle string `json:"@handle"`
			Name   string `json:"@name"`
			Value  string `json:"$"`
		} `json:"parentNetRef"`
		StartAddress struct {
			Value string `json:"$"`
		} `json:"startAddress"`
		UpdateDate struct {
			Value string `json:"$"`
		} `json:"updateDate"`
		Version struct {
			Value string `json:"$"`
		} `json:"version"`
	} `json:"net"`
}

func LookupOrgNets(org string) ([]string, error) {
	handles := []string{}

	safe_org := url.QueryEscape(org)
	u := fmt.Sprintf("http://whois.arin.net/rest/org/%s/nets", safe_org)

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return handles, err
	}
	req.Header.Set("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return handles, err
	}

	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return handles, err
	}

	// No network handles associated with this organization
	if strings.Contains(string(content), "No related resources were found for the handle provided") {
		return handles, nil
	}

	var nets ARIN_OrgNets

	if err := json.Unmarshal(content, &nets); err == nil {
		for i := range nets.Nets.NetRef {
			handles = append(handles, nets.Nets.NetRef[i].Handle)
		}
	} else {
		// Try to decode as a single-net organization
		var net ARIN_OrgNet
		if err := json.Unmarshal(content, &net); err != nil {
			return handles, err
		}

		handles = append(handles, net.Nets.NetRef.Handle)
	}

	return handles, nil
}

func LookupNetCidrs(handle string) ([]string, error) {
	cidrs := []string{}

	safe_handle := url.QueryEscape(handle)
	u := fmt.Sprintf("http://whois.arin.net/rest/net/%s", safe_handle)

	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return cidrs, err
	}
	req.Header.Set("Accept", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return cidrs, err
	}

	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return cidrs, err
	}

	var nets ARIN_Nets

	if err := json.Unmarshal(content, &nets); err == nil {
		for i := range nets.Net.NetBlocks {
			cidrs = append(cidrs, fmt.Sprintf("%s/%s", nets.Net.NetBlocks[i].NetBlock.StartAddress.Value, nets.Net.NetBlocks[i].NetBlock.CidrLength.Value))
		}
	} else {
		// Try to decode as a single-block network
		var net ARIN_Net
		if err := json.Unmarshal(content, &net); err != nil {
			return cidrs, err
		}
		cidrs = append(cidrs, fmt.Sprintf("%s/%s", net.Net.NetBlocks.NetBlock.StartAddress.Value, net.Net.NetBlocks.NetBlock.CidrLength.Value))
	}

	return cidrs, nil
}

func main() {

	if len(os.Args) != 2 {
		fmt.Println("Usage: inetdata-arin-org2nets <org-handle>\n")
		os.Exit(1)
	}

	org := os.Args[1]

	handles, he := LookupOrgNets(org)
	if he != nil {
		fmt.Fprintf(os.Stderr, "Could not list network handles: %s", he.Error())
		os.Exit(1)
	}

	for i := range handles {
		cidrs, e := LookupNetCidrs(handles[i])
		if e != nil {
			fmt.Fprintf(os.Stderr, "Could not list CIDRs for %s: %s", handles[i], he.Error())
			continue
		}
		fmt.Println(strings.Join(cidrs, "\n"))
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"fmt"
	"runtime"
	"time"

	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kuberuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)

func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
	client, err := kubeconfigutil.ClientSetFromFile(file)
	if err != nil {
		return nil, err
	}

	fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
	WaitForAPI(client)

	fmt.Println("[apiclient] Waiting for at least one node to register and become ready")
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		nodeList, err := client.Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
			return false, nil
		}
		if len(nodeList.Items) < 1 {
			return false, nil
		}
		n := &nodeList.Items[0]
		if !v1.IsNodeReady(n) {
			fmt.Println("[apiclient] First node has registered, but is not ready yet")
			return false, nil
		}

		fmt.Printf("[apiclient] First node is ready after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})

	if err := createAndWaitForADummyDeployment(client); err != nil {
		return nil, err
	}

	return client, nil
}

func WaitForAPI(client *clientset.Clientset) {
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: use /healthz API instead of this
		cs, err := client.ComponentStatuses().List(metav1.ListOptions{})
		if err != nil {
			if apierrs.IsForbidden(err) {
				fmt.Println("[apiclient] Waiting for API server authorization")
			}
			return false, nil
		}

		// TODO(phase2) must revisit this when we implement HA
		if len(cs.Items) < 3 {
			return false, nil
		}
		for _, item := range cs.Items {
			for _, condition := range item.Conditions {
				if condition.Type != v1.ComponentHealthy {
					fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions)
					return false, nil
				}
			}
		}

		fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})
}

func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
		Arch:            runtime.GOARCH,
	})
	if err != nil {
		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
	}

	dummyDeployment := &extensions.Deployment{}
	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
		return fmt.Errorf("unable to decode dummy deployment %v", err)
	}

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: we should check the error, as some cases may be fatal
		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
			fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		return true, nil
	})

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
		if err != nil {
			fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})

	fmt.Println("[apiclient] Test deployment succeeded")

	foreground := metav1.DeletePropagationForeground
	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{
		PropagationPolicy: &foreground,
	}); err != nil {
		fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
	}
	return nil
}
<commit_msg>don't wait for first kubelet to be ready<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package master

import (
	"fmt"
	"runtime"
	"time"

	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kuberuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
)

func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
	client, err := kubeconfigutil.ClientSetFromFile(file)
	if err != nil {
		return nil, err
	}

	fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
	WaitForAPI(client)

	fmt.Println("[apiclient] Waiting for at least one node to register")
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		nodeList, err := client.Nodes().List(metav1.ListOptions{})
		if err != nil {
			fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
			return false, nil
		}
		if len(nodeList.Items) < 1 {
			return false, nil
		}

		fmt.Printf("[apiclient] First node has registered after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})

	return client, nil
}

func WaitForAPI(client *clientset.Clientset) {
	start := time.Now()
	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: use /healthz API instead of this
		cs, err := client.ComponentStatuses().List(metav1.ListOptions{})
		if err != nil {
			if apierrs.IsForbidden(err) {
				fmt.Println("[apiclient] Waiting for API server authorization")
			}
			return false, nil
		}

		// TODO(phase2) must revisit this when we implement HA
		if len(cs.Items) < 3 {
			return false, nil
		}
		for _, item := range cs.Items {
			for _, condition := range item.Conditions {
				if condition.Type != v1.ComponentHealthy {
					fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions)
					return false, nil
				}
			}
		}

		fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
		return true, nil
	})
}

func createAndWaitForADummyDeployment(client *clientset.Clientset) error {
	dummyDeploymentBytes, err := kubeadmutil.ParseTemplate(DummyDeployment, struct{ ImageRepository, Arch string }{
		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
		Arch:            runtime.GOARCH,
	})
	if err != nil {
		return fmt.Errorf("error when parsing dummy deployment template: %v", err)
	}

	dummyDeployment := &extensions.Deployment{}
	if err := kuberuntime.DecodeInto(api.Codecs.UniversalDecoder(), dummyDeploymentBytes, dummyDeployment); err != nil {
		return fmt.Errorf("unable to decode dummy deployment %v", err)
	}

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		// TODO: we should check the error, as some cases may be fatal
		if _, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Create(dummyDeployment); err != nil {
			fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		return true, nil
	})

	wait.PollInfinite(kubeadmconstants.APICallRetryInterval, func() (bool, error) {
		d, err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Get("dummy", metav1.GetOptions{})
		if err != nil {
			fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
			return false, nil
		}
		if d.Status.AvailableReplicas < 1 {
			return false, nil
		}
		return true, nil
	})

	fmt.Println("[apiclient] Test deployment succeeded")

	foreground := metav1.DeletePropagationForeground
	if err := client.ExtensionsV1beta1().Deployments(metav1.NamespaceSystem).Delete("dummy", &metav1.DeleteOptions{
		PropagationPolicy: &foreground,
	}); err != nil {
		fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"time"

	"github.com/spf13/cobra"

	"github.com/tendermint/tendermint/node"
	"github.com/tendermint/tendermint/types"
	cmn "github.com/tendermint/tmlibs/common"
)

var runNodeCmd = &cobra.Command{
	Use:   "node",
	Short: "Run the tendermint node",
	RunE:  runNode,
}

func init() {
	AddNodeFlags(runNodeCmd)
	RootCmd.AddCommand(runNodeCmd)
}

// AddNodeFlags exposes some common configuration options on the command-line
// These are exposed for convenience of commands embedding a tendermint node
func AddNodeFlags(cmd *cobra.Command) {
	// bind flags
	cmd.Flags().String("moniker", config.Moniker, "Node Name")

	// node flags
	cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing")

	// abci flags
	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.")
	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")

	// rpc flags
	cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required")
	cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required")
	cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods")

	// p2p flags
	cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)")
	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes")
	cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration")
	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)")

	// consensus flags
	cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes")
}

// Users wishing to:
//	* Use an external signer for their validators
//	* Supply an in-proc abci app
// should import github.com/tendermint/tendermint/node and implement
// their own run_node to call node.NewNode (instead of node.NewNodeDefault)
// with their custom priv validator and/or custom proxy.ClientCreator
func runNode(cmd *cobra.Command, args []string) error {

	// Wait until the genesis doc becomes available
	// This is for Mintnet compatibility.
	// TODO: If Mintnet gets deprecated or genesis_file is
	// always available, remove.
	genDocFile := config.GenesisFile()
	for !cmn.FileExists(genDocFile) {
		logger.Info(cmn.Fmt("Waiting for genesis file %v...", genDocFile))
		time.Sleep(time.Second)
	}

	genDoc, err := types.GenesisDocFromFile(genDocFile)
	if err != nil {
		return err
	}
	config.ChainID = genDoc.ChainID

	// Create & start node
	n := node.NewNodeDefault(config, logger.With("module", "node"))
	if _, err := n.Start(); err != nil {
		return fmt.Errorf("Failed to start node: %v", err)
	} else {
		logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
	}

	// Trap signal, run forever.
	n.RunForever()

	return nil
}
<commit_msg>cmd: dont wait for genesis. closes #562<commit_after>package commands

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/tendermint/tendermint/node"
	"github.com/tendermint/tendermint/types"
)

var runNodeCmd = &cobra.Command{
	Use:   "node",
	Short: "Run the tendermint node",
	RunE:  runNode,
}

func init() {
	AddNodeFlags(runNodeCmd)
	RootCmd.AddCommand(runNodeCmd)
}

// AddNodeFlags exposes some common configuration options on the command-line
// These are exposed for convenience of commands embedding a tendermint node
func AddNodeFlags(cmd *cobra.Command) {
	// bind flags
	cmd.Flags().String("moniker", config.Moniker, "Node Name")

	// node flags
	cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing")

	// abci flags
	cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'dummy' for local testing.")
	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")

	// rpc flags
	cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required")
	cmd.Flags().String("rpc.grpc_laddr", config.RPC.GRPCListenAddress, "GRPC listen address (BroadcastTx only). Port required")
	cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods")

	// p2p flags
	cmd.Flags().String("p2p.laddr", config.P2P.ListenAddress, "Node listen address. (0.0.0.0:0 means any interface, any port)")
	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma delimited host:port seed nodes")
	cmd.Flags().Bool("p2p.skip_upnp", config.P2P.SkipUPNP, "Skip UPNP configuration")
	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable Peer-Exchange (dev feature)")

	// consensus flags
	cmd.Flags().Bool("consensus.create_empty_blocks", config.Consensus.CreateEmptyBlocks, "Set this to false to only produce blocks when there are txs or when the AppHash changes")
}

// Users wishing to:
//	* Use an external signer for their validators
//	* Supply an in-proc abci app
// should import github.com/tendermint/tendermint/node and implement
// their own run_node to call node.NewNode (instead of node.NewNodeDefault)
// with their custom priv validator and/or custom proxy.ClientCreator
func runNode(cmd *cobra.Command, args []string) error {

	genDocFile := config.GenesisFile()
	genDoc, err := types.GenesisDocFromFile(genDocFile)
	if err != nil {
		return err
	}
	config.ChainID = genDoc.ChainID

	// Create & start node
	n := node.NewNodeDefault(config, logger.With("module", "node"))
	if _, err := n.Start(); err != nil {
		return fmt.Errorf("Failed to start node: %v", err)
	} else {
		logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
	}

	// Trap signal, run forever.
	n.RunForever()

	return nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl    string
	listen     string
	dataDir    string
	lifeTime   time.Duration
	maxSizeStr string
	idSize     int
	maxSize    ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func randomId() Id {
	s := make([]byte, idSize)
	var offset int = 0
MainLoop:
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return Id(s)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) endLife() {
	err := os.Remove(id.Path())
	if err == nil {
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.endLifeAfter(2 * time.Minute)
	}
}

func (id Id) endLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.endLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, siteUrl)
			return
		case "/form":
			formTemplate.Execute(w, siteUrl)
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		defer pasteFile.Close()
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer compReader.Close()
		io.Copy(w, compReader)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id Id
		var pastePath, content string
		found := false
		for i := 0; i < 10; i++ {
			id = randomId()
			pastePath = id.Path()
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				found = true
				break
			}
		}
		if !found {
			log.Printf("Gave up trying to find an unused random id")
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		id.endLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		compWriter := zlib.NewWriter(pasteFile)
		defer compWriter.Close()
		b, err := io.WriteString(compWriter, content)
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		log.Printf("Created a new paste: %s (%s)", id, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	deathTime := fileInfo.ModTime().Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.endLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", id, lifeLeft)
	id.endLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>Join string vars<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func randomId() Id {
	s := make([]byte, idSize)
	var offset int = 0
MainLoop:
	for {
		r := rand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return Id(s)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) endLife() {
	err := os.Remove(id.Path())
	if err == nil {
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.endLifeAfter(2 * time.Minute)
	}
}

func (id Id) endLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.endLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, siteUrl)
			return
		case "/form":
			formTemplate.Execute(w, siteUrl)
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		defer pasteFile.Close()
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer compReader.Close()
		io.Copy(w, compReader)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id Id
		var pastePath, content string
		found := false
		for i := 0; i < 10; i++ {
			id = randomId()
			pastePath = id.Path()
			if _, err := os.Stat(pastePath); os.IsNotExist(err) {
				found = true
				break
			}
		}
		if !found {
			log.Printf("Gave up trying to find an unused random id")
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		id.endLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		compWriter := zlib.NewWriter(pasteFile)
		defer compWriter.Close()
		b, err := io.WriteString(compWriter, content)
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		log.Printf("Created a new paste: %s (%s)", id, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	deathTime := fileInfo.ModTime().Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.endLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", id, lifeLeft)
	id.endLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provisioner

import (
	"errors"
	"fmt"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"github.com/kubernetes-incubator/external-storage/openstack/standalone-cinder/pkg/volumeservice"
	"k8s.io/api/core/v1"
)

const rbdType = "rbd"

type rbdMapper struct {
	volumeMapper
}

func getMonitors(conn volumeservice.VolumeConnection) []string {
	if len(conn.Data.Hosts) != len(conn.Data.Ports) {
		glog.Errorf("Error parsing rbd connection info: 'hosts' and 'ports' have different lengths")
		return nil
	}
	mons := make([]string, len(conn.Data.Hosts))
	for i := range conn.Data.Hosts {
		mons[i] = fmt.Sprintf("%s:%s", conn.Data.Hosts[i], conn.Data.Ports[i])
	}
	return mons
}

func getRbdSecretName(pvc *v1.PersistentVolumeClaim) string {
	return fmt.Sprintf("%s-cephx-secret", *pvc.Spec.StorageClassName)
}

func (m *rbdMapper) BuildPVSource(conn volumeservice.VolumeConnection, options controller.VolumeOptions) (*v1.PersistentVolumeSource, error) {
	mons := getMonitors(conn)
	if mons == nil {
		return nil, errors.New("No monitors could be parsed from connection info")
	}
	splitName := strings.SplitN(conn.Data.Name, "/", 2)
	if len(splitName) != 2 {
		return nil, errors.New("Field 'name' cannot be split into pool and image")
	}

	return &v1.PersistentVolumeSource{
		RBD: &v1.RBDPersistentVolumeSource{
			CephMonitors: mons,
			RBDPool:      splitName[0],
			RBDImage:     splitName[1],
			RadosUser:    conn.Data.AuthUsername,
			SecretRef: &v1.SecretReference{
				Name: getRbdSecretName(options.PVC),
			},
		},
	}, nil
}

func (m *rbdMapper) AuthSetup(p *cinderProvisioner, options controller.VolumeOptions, conn volumeservice.VolumeConnection) error {
	return nil
}

func (m *rbdMapper) AuthTeardown(p *cinderProvisioner, pv *v1.PersistentVolume) error {
	return nil
}
<commit_msg>minor fix: getRbdSecretName() get pvc storageclass not only in pvc spec<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provisioner

import (
	"errors"
	"fmt"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"github.com/kubernetes-incubator/external-storage/openstack/standalone-cinder/pkg/volumeservice"

	"k8s.io/api/core/v1"
	"k8s.io/kubernetes/pkg/apis/core/v1/helper"
)

const rbdType = "rbd"

type rbdMapper struct {
	volumeMapper
}

func getMonitors(conn volumeservice.VolumeConnection) []string {
	if len(conn.Data.Hosts) != len(conn.Data.Ports) {
		glog.Errorf("Error parsing rbd connection info: 'hosts' and 'ports' have different lengths")
		return nil
	}
	mons := make([]string, len(conn.Data.Hosts))
	for i := range conn.Data.Hosts {
		mons[i] = fmt.Sprintf("%s:%s", conn.Data.Hosts[i], conn.Data.Ports[i])
	}
	return mons
}

func getRbdSecretName(pvc *v1.PersistentVolumeClaim) string {
	return fmt.Sprintf("%s-cephx-secret", helper.GetPersistentVolumeClaimClass(pvc))
}

func (m *rbdMapper) BuildPVSource(conn volumeservice.VolumeConnection, options controller.VolumeOptions) (*v1.PersistentVolumeSource, error) {
	mons := getMonitors(conn)
	if mons == nil {
		return nil, errors.New("No monitors could be parsed from connection info")
	}
	splitName := strings.SplitN(conn.Data.Name, "/", 2)
	if len(splitName) != 2 {
		return nil, errors.New("Field 'name' cannot be split into pool and image")
	}

	return &v1.PersistentVolumeSource{
		RBD: &v1.RBDPersistentVolumeSource{
			CephMonitors: mons,
			RBDPool:      splitName[0],
			RBDImage:     splitName[1],
			RadosUser:    conn.Data.AuthUsername,
			SecretRef: &v1.SecretReference{
				Name: getRbdSecretName(options.PVC),
			},
		},
	}, nil
}

func (m *rbdMapper) AuthSetup(p *cinderProvisioner, options controller.VolumeOptions, conn volumeservice.VolumeConnection) error {
	return nil
}

func (m *rbdMapper) AuthTeardown(p *cinderProvisioner, pv *v1.PersistentVolume) error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/GeoNet/fits/dapper/dapperlib"
	"github.com/GeoNet/fits/dapper/internal/platform/s3"
	"github.com/GeoNet/fits/dapper/internal/platform/sqs"
	"github.com/GeoNet/kit/cfg"
	"github.com/GeoNet/kit/metrics"
	_ "github.com/lib/pq"
	"log"
	"os"
	"time"
)

var (
	queueURL  = os.Getenv("SQS_QUEUE_URL")
	s3Client  s3.S3
	sqsClient sqs.SQS
	db        *sql.DB
)

type notification struct {
	s3.Event
}

func main() {
	var err error

	p, err := cfg.PostgresEnv()
	if err != nil {
		log.Fatalf("error reading DB config from the environment vars: %v", err)
	}

	db, err = sql.Open("postgres", p.Connection())
	if err != nil {
		log.Fatalf("error with DB config: %v", err)
	}

	sqsClient, err = sqs.New()
	if err != nil {
		log.Fatal(err)
	}

	s3Client, err = s3.New()
	if err != nil {
		log.Fatal(err)
	}

	var r sqs.Raw
	var n notification

	for {
		// TODO - does this visibility time out make sense?
		// we don't want the message to become visible again if there is
		// still processing happening
		r, err = sqsClient.Receive(queueURL, 120)
		if err != nil {
			log.Printf("problem receiving message, backing off: %s", err)
			time.Sleep(time.Second * 20)
			continue
		}

		err = metrics.DoProcess(&n, []byte(r.Body))
		if err != nil {
			log.Printf("problem processing message, redelivering: %s", err)
			continue
		}

		err = sqsClient.Delete(queueURL, r.ReceiptHandle)
		if err != nil {
			log.Printf("problem deleting message, continuing: %s", err)
		}
	}
}

// Process implements msg.Processor for notification.
func (n *notification) Process(msg []byte) error {
	err := json.Unmarshal(msg, n)
	if err != nil {
		log.Println(err, "not redelivering")
		return nil // not going to retry this
	}
	if n.Records == nil {
		log.Println("got nil Records pointer in notification message, not redelivering")
		return nil // not going to retry this
	}
	if len(n.Records) == 0 {
		log.Println("got zero Records in notification message, not redelivering")
		return nil // not going to retry this
	}

	tx, err := db.Begin()
	if err != nil {
		return fmt.Errorf("couldn't open db transaction: %v", err)
	}

	stmt, err := tx.Prepare(`INSERT INTO dapper.records (record_domain, record_key, field, time, value, archived, modtime) VALUES ($1, $2, $3, $4, $5, FALSE, NOW());`)
	if err != nil {
		return fmt.Errorf("failed to prepare record insert stmt: %v", err)
	}

	var br bytes.Buffer

	// read all notified raw csv files into tablesData
	// TODO: dealing with a notification contains multiple records and one of the record got error
	for _, v := range n.Records { //TODO: Parallelise
		//get the file
		log.Println("Processing", v.S3.Bucket.Name+"/"+v.S3.Object.Key)
		err = s3Client.Get(v.S3.Bucket.Name, v.S3.Object.Key, "", &br)
		if err != nil {
			tx.Rollback()
			return fmt.Errorf("couldn't get specified object: %v", err)
		}

		//convert the file into CSV
		csvr := csv.NewReader(&br)
		records, err := csvr.ReadAll()
		if err != nil {
			tx.Rollback()
			return fmt.Errorf("couldn't unpackage as CSV: %v", err)
		}

		for _, row := range records {
			rec, err := dapperlib.RecordFromCSV(row)
			if err != nil {
				tx.Rollback()
				return fmt.Errorf("couldn't parse csv line: %v", err)
			}

			_, err = stmt.Exec(rec.Domain, rec.Key, rec.Field, rec.Time, rec.Value)
			if err != nil {
				tx.Rollback()
				return fmt.Errorf("query failed: %v for record %s,%s,%s,%s", err, rec.Domain, rec.Key, rec.Field, rec.Time.Format(time.RFC3339))
			}
		}
		log.Println("Done processing", v.S3.Bucket.Name+"/"+v.S3.Object.Key)
	}

	err = tx.Commit()
	if err != nil {
		return fmt.Errorf("couldn't commit db transaction: %v", err)
	}
	log.Println("Transactions committed to database")
	return nil
}
<commit_msg>Extend retry time to 15 minutes.<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/GeoNet/fits/dapper/dapperlib"
	"github.com/GeoNet/fits/dapper/internal/platform/s3"
	"github.com/GeoNet/fits/dapper/internal/platform/sqs"
	"github.com/GeoNet/kit/cfg"
	"github.com/GeoNet/kit/metrics"
	_ "github.com/lib/pq"
	"log"
	"os"
	"time"
)

var (
	queueURL  = os.Getenv("SQS_QUEUE_URL")
	s3Client  s3.S3
	sqsClient sqs.SQS
	db        *sql.DB
)

type notification struct {
	s3.Event
}

func main() {
	var err error

	p, err := cfg.PostgresEnv()
	if err != nil {
		log.Fatalf("error reading DB config from the environment vars: %v", err)
	}

	db, err = sql.Open("postgres", p.Connection())
	if err != nil {
		log.Fatalf("error with DB config: %v", err)
	}

	sqsClient, err = sqs.New()
	if err != nil {
		log.Fatal(err)
	}

	s3Client, err = s3.New()
	if err != nil {
		log.Fatal(err)
	}

	var r sqs.Raw
	var n notification

	for {
		// TODO - does this visibility time out make sense?
		// we don't want the message to become visible again if there is
		// still processing happening
		r, err = sqsClient.Receive(queueURL, 900)
		if err != nil {
			log.Printf("problem receiving message, backing off: %s", err)
			time.Sleep(time.Second * 20)
			continue
		}

		err = metrics.DoProcess(&n, []byte(r.Body))
		if err != nil {
			log.Printf("problem processing message, redelivering: %s", err)
			continue
		}

		err = sqsClient.Delete(queueURL, r.ReceiptHandle)
		if err != nil {
			log.Printf("problem deleting message, continuing: %s", err)
		}
	}
}

// Process implements msg.Processor for notification.
func (n *notification) Process(msg []byte) error {
	err := json.Unmarshal(msg, n)
	if err != nil {
		log.Println(err, "not redelivering")
		return nil // not going to retry this
	}
	if n.Records == nil {
		log.Println("got nil Records pointer in notification message, not redelivering")
		return nil // not going to retry this
	}
	if len(n.Records) == 0 {
		log.Println("got zero Records in notification message, not redelivering")
		return nil // not going to retry this
	}

	tx, err := db.Begin()
	if err != nil {
		return fmt.Errorf("couldn't open db transaction: %v", err)
	}

	stmt, err := tx.Prepare(`INSERT INTO dapper.records (record_domain, record_key, field, time, value, archived, modtime) VALUES ($1, $2, $3, $4, $5, FALSE, NOW());`)
	if err != nil {
		return fmt.Errorf("failed to prepare record insert stmt: %v", err)
	}

	var br bytes.Buffer

	// read all notified raw csv files into tablesData
	// TODO: dealing with a notification contains multiple records and one of the record got error
	for _, v := range n.Records { //TODO: Parallelise
		//get the file
		log.Println("Processing", v.S3.Bucket.Name+"/"+v.S3.Object.Key)
		err = s3Client.Get(v.S3.Bucket.Name, v.S3.Object.Key, "", &br)
		if err != nil {
			tx.Rollback()
			return fmt.Errorf("couldn't get specified object: %v", err)
		}

		//convert the file into CSV
		csvr := csv.NewReader(&br)
		records, err := csvr.ReadAll()
		if err != nil {
			tx.Rollback()
			return fmt.Errorf("couldn't unpackage as CSV: %v", err)
		}

		for _, row := range records {
			rec, err := dapperlib.RecordFromCSV(row)
			if err != nil {
				tx.Rollback()
				return fmt.Errorf("couldn't parse csv line: %v", err)
			}

			_, err = stmt.Exec(rec.Domain, rec.Key, rec.Field, rec.Time, rec.Value)
			if err != nil {
				tx.Rollback()
				return fmt.Errorf("query failed: %v for record %s,%s,%s,%s", err, rec.Domain, rec.Key, rec.Field, rec.Time.Format(time.RFC3339))
			}
		}
		log.Println("Done processing", v.S3.Bucket.Name+"/"+v.S3.Object.Key)
	}

	err = tx.Commit()
	if err != nil {
		return fmt.Errorf("couldn't commit db transaction: %v", err)
	}
	log.Println("Transactions committed to database")
	return nil
}
<|endoftext|>