<commit_before>package request

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync/atomic"

	"github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/config"
)

const defaultUserAgent = "go-cozy-client"

type (
	// Authorizer is an interface to represent any element that can be used as a
	// token bearer.
	Authorizer interface {
		AuthHeader() string
	}

	// Headers is a map of strings used to represent HTTP headers
	Headers map[string]string

	// Options is a struct holding of the details of a request.
	//
	// The NoResponse field can be used in case the call's response if not used. In
	// such cases, the response body is automatically closed.
	Options struct {
		Domain     string
		Scheme     string
		Method     string
		Path       string
		Queries    url.Values
		Headers    Headers
		Body       io.Reader
		Authorizer Authorizer
		NoResponse bool

		DisableSecure bool
		Client        *http.Client
		UserAgent     string
		ParseError    func(res *http.Response, b []byte) error
	}

	// Error is the typical JSON-API error returned by the API
	Error struct {
		Status string `json:"status"`
		Title  string `json:"title"`
		Detail string `json:"detail"`
	}
)

// httpFallback can be used to fallback on a non-secure http request if the
// https one has failed to open. This should happend only on dev binaries.
var httpFallback int32

func (e *Error) Error() string {
	if e.Detail == "" || e.Title == e.Detail {
		return e.Title
	}
	return fmt.Sprintf("%s: %s", e.Title, e.Detail)
}

// BasicAuthorizer implements the HTTP basic auth for authorization.
type BasicAuthorizer struct {
	Username string
	Password string
}

// AuthHeader implemented the interface Authorizer.
func (b *BasicAuthorizer) AuthHeader() string {
	auth := b.Username + ":" + b.Password
	return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
}

// BearerAuthorizer implements a placeholder authorizer if the token is already
// known.
type BearerAuthorizer struct {
	Token string
}

// AuthHeader implemented the interface Authorizer.
func (b *BearerAuthorizer) AuthHeader() string {
	return "Bearer " + b.Token
}

// Req performs a request with the specified request options.
func Req(opts *Options) (*http.Response, error) {
	scheme := opts.Scheme
	if scheme == "" {
		if atomic.LoadInt32(&httpFallback) == 1 {
			scheme = "http"
		} else {
			scheme = "https"
		}
	}
	u := url.URL{
		Scheme: scheme,
		Host:   opts.Domain,
		Path:   opts.Path,
	}
	if opts.Queries != nil {
		u.RawQuery = opts.Queries.Encode()
	}

	req, err := http.NewRequest(opts.Method, u.String(), opts.Body)
	if err != nil {
		return nil, err
	}

	if opts.Headers != nil {
		for k, v := range opts.Headers {
			if k == "Content-Length" {
				var contentLength int64
				contentLength, err = strconv.ParseInt(v, 10, 64)
				if err != nil {
					return nil, fmt.Errorf("Invalid Content-Length value")
				}
				req.ContentLength = contentLength
			} else {
				req.Header.Add(k, v)
			}
		}
	}

	if opts.Authorizer != nil {
		req.Header.Add("Authorization", opts.Authorizer.AuthHeader())
	}

	ua := opts.UserAgent
	if ua == "" {
		ua = defaultUserAgent
	}

	req.Header.Add("User-Agent", ua)

	client := opts.Client
	if client == nil {
		client = http.DefaultClient
	}

	res, err := client.Do(req)
	if err != nil {
		// for a development release, we fallback on http mode if the https request
		// is not successful.
		// TODO: better identify the error (cross-platform)
		if config.IsDevRelease() && scheme == "https" {
			logrus.Debug("[request] fallback on http transport since https request has failed", err)
			atomic.StoreInt32(&httpFallback, 1)
			return Req(opts)
		}
		return nil, err
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, parseError(opts, res)
	}

	if opts.NoResponse {
		err = res.Body.Close()
		if err != nil {
			return nil, err
		}
	}

	return res, nil
}

func parseError(opts *Options, res *http.Response) (err error) {
	defer checkClose(res.Body, &err)
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: err.Error(),
		}
	}
	if opts.ParseError == nil {
		return &Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: string(b),
		}
	}
	return opts.ParseError(res, b)
}

// ErrSSEParse is used when an error occured while parsing the SSE stream.
var ErrSSEParse = errors.New("could not parse event stream")

// SSEEvent holds the data of a single SSE event.
type SSEEvent struct {
	Name  string
	Data  []byte
	Error error
}

// ReadSSE reads and parse a SSE source from a bufio.Reader into a channel of
// SSEEvent.
func ReadSSE(r io.ReadCloser, ch chan *SSEEvent) {
	var err error
	defer func() {
		if err != nil {
			ch <- &SSEEvent{Error: err}
		}
		if errc := r.Close(); errc != nil && err == nil {
			ch <- &SSEEvent{Error: errc}
		}
		close(ch)
	}()
	rb := bufio.NewReader(r)
	var ev *SSEEvent
	for {
		var bs []byte
		bs, err = rb.ReadBytes('\n')
		if err == io.EOF {
			err = nil
			return
		}
		if err != nil {
			return
		}
		if bytes.Equal(bs, []byte("\r\n")) {
			ev = nil
			continue
		}
		spl := bytes.Split(bs, []byte(": "))
		if len(spl) != 2 {
			err = ErrSSEParse
			return
		}
		k, v := string(spl[0]), bytes.TrimSpace(spl[1])
		switch k {
		case "event":
			ev = &SSEEvent{Name: string(v)}
		case "data":
			if ev == nil {
				err = ErrSSEParse
				return
			}
			ev.Data = v
			ch <- ev
		default:
			err = ErrSSEParse
			return
		}
	}
}

// ReadJSON reads the content of the specified ReadCloser and closes it.
func ReadJSON(r io.ReadCloser, data interface{}) (err error) {
	defer checkClose(r, &err)
	return json.NewDecoder(r).Decode(&data)
}

// WriteJSON returns an io.Reader from which a JSON encoded data can be read.
func WriteJSON(data interface{}) (io.Reader, error) {
	buf, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(buf), nil
}

func checkClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil && cerr != nil {
		*err = cerr
	}
}
<commit_msg>Cross platform err checking for connection refused error<commit_after>package request

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"sync/atomic"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/config"
)

const defaultUserAgent = "go-cozy-client"

type (
	// Authorizer is an interface to represent any element that can be used as a
	// token bearer.
	Authorizer interface {
		AuthHeader() string
	}

	// Headers is a map of strings used to represent HTTP headers
	Headers map[string]string

	// Options is a struct holding of the details of a request.
	//
	// The NoResponse field can be used in case the call's response if not used. In
	// such cases, the response body is automatically closed.
	Options struct {
		Domain     string
		Scheme     string
		Method     string
		Path       string
		Queries    url.Values
		Headers    Headers
		Body       io.Reader
		Authorizer Authorizer
		NoResponse bool

		DisableSecure bool
		Client        *http.Client
		UserAgent     string
		ParseError    func(res *http.Response, b []byte) error
	}

	// Error is the typical JSON-API error returned by the API
	Error struct {
		Status string `json:"status"`
		Title  string `json:"title"`
		Detail string `json:"detail"`
	}
)

// httpFallback can be used to fallback on a non-secure http request if the
// https one has failed to open. This should happend only on dev binaries.
var httpFallback int32

func (e *Error) Error() string {
	if e.Detail == "" || e.Title == e.Detail {
		return e.Title
	}
	return fmt.Sprintf("%s: %s", e.Title, e.Detail)
}

// BasicAuthorizer implements the HTTP basic auth for authorization.
type BasicAuthorizer struct {
	Username string
	Password string
}

// AuthHeader implemented the interface Authorizer.
func (b *BasicAuthorizer) AuthHeader() string {
	auth := b.Username + ":" + b.Password
	return "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
}

// BearerAuthorizer implements a placeholder authorizer if the token is already
// known.
type BearerAuthorizer struct {
	Token string
}

// AuthHeader implemented the interface Authorizer.
func (b *BearerAuthorizer) AuthHeader() string {
	return "Bearer " + b.Token
}

// Req performs a request with the specified request options.
func Req(opts *Options) (*http.Response, error) {
	scheme := opts.Scheme
	if scheme == "" {
		if atomic.LoadInt32(&httpFallback) == 1 {
			scheme = "http"
		} else {
			scheme = "https"
		}
	}
	u := url.URL{
		Scheme: scheme,
		Host:   opts.Domain,
		Path:   opts.Path,
	}
	if opts.Queries != nil {
		u.RawQuery = opts.Queries.Encode()
	}

	req, err := http.NewRequest(opts.Method, u.String(), opts.Body)
	if err != nil {
		return nil, err
	}

	if opts.Headers != nil {
		for k, v := range opts.Headers {
			if k == "Content-Length" {
				var contentLength int64
				contentLength, err = strconv.ParseInt(v, 10, 64)
				if err != nil {
					return nil, fmt.Errorf("Invalid Content-Length value")
				}
				req.ContentLength = contentLength
			} else {
				req.Header.Add(k, v)
			}
		}
	}

	if opts.Authorizer != nil {
		req.Header.Add("Authorization", opts.Authorizer.AuthHeader())
	}

	ua := opts.UserAgent
	if ua == "" {
		ua = defaultUserAgent
	}

	req.Header.Add("User-Agent", ua)

	client := opts.Client
	if client == nil {
		client = http.DefaultClient
	}

	res, err := client.Do(req)
	if err != nil {
		// for a development release, we fallback on http mode if the https request
		// is not successful.
		if config.IsDevRelease() && scheme == "https" {
			var isConnRefused bool
			switch t := err.(type) {
			case *net.OpError:
				if t.Op == "read" {
					isConnRefused = true
				}
			case syscall.Errno:
				if t == syscall.ECONNREFUSED {
					isConnRefused = true
				}
			}
			if isConnRefused {
				logrus.Debug("[request] fallback on http transport since https request has failed", err)
				atomic.StoreInt32(&httpFallback, 1)
				return Req(opts)
			}
		}
		return nil, err
	}

	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, parseError(opts, res)
	}

	if opts.NoResponse {
		err = res.Body.Close()
		if err != nil {
			return nil, err
		}
	}

	return res, nil
}

func parseError(opts *Options, res *http.Response) (err error) {
	defer checkClose(res.Body, &err)
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return &Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: err.Error(),
		}
	}
	if opts.ParseError == nil {
		return &Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: string(b),
		}
	}
	return opts.ParseError(res, b)
}

// ErrSSEParse is used when an error occured while parsing the SSE stream.
var ErrSSEParse = errors.New("could not parse event stream")

// SSEEvent holds the data of a single SSE event.
type SSEEvent struct {
	Name  string
	Data  []byte
	Error error
}

// ReadSSE reads and parse a SSE source from a bufio.Reader into a channel of
// SSEEvent.
func ReadSSE(r io.ReadCloser, ch chan *SSEEvent) {
	var err error
	defer func() {
		if err != nil {
			ch <- &SSEEvent{Error: err}
		}
		if errc := r.Close(); errc != nil && err == nil {
			ch <- &SSEEvent{Error: errc}
		}
		close(ch)
	}()
	rb := bufio.NewReader(r)
	var ev *SSEEvent
	for {
		var bs []byte
		bs, err = rb.ReadBytes('\n')
		if err == io.EOF {
			err = nil
			return
		}
		if err != nil {
			return
		}
		if bytes.Equal(bs, []byte("\r\n")) {
			ev = nil
			continue
		}
		spl := bytes.Split(bs, []byte(": "))
		if len(spl) != 2 {
			err = ErrSSEParse
			return
		}
		k, v := string(spl[0]), bytes.TrimSpace(spl[1])
		switch k {
		case "event":
			ev = &SSEEvent{Name: string(v)}
		case "data":
			if ev == nil {
				err = ErrSSEParse
				return
			}
			ev.Data = v
			ch <- ev
		default:
			err = ErrSSEParse
			return
		}
	}
}

// ReadJSON reads the content of the specified ReadCloser and closes it.
func ReadJSON(r io.ReadCloser, data interface{}) (err error) {
	defer checkClose(r, &err)
	return json.NewDecoder(r).Decode(&data)
}

// WriteJSON returns an io.Reader from which a JSON encoded data can be read.
func WriteJSON(data interface{}) (io.Reader, error) {
	buf, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(buf), nil
}

func checkClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil && cerr != nil {
		*err = cerr
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"math"
	"strconv"
	"strings"
	"time"
)

type OrderLaw int

const (
	// to correct template compilation it cannot starts from zero
	FZ44 OrderLaw = iota + 1
	FZ223
	FZ94
)

func ParseLow(str string) (OrderLaw, error) {
	switch {
	case strings.Contains(str, "44"):
		return FZ44, nil
	case strings.Contains(str, "223"):
		return FZ223, nil
	case strings.Contains(str, "94"):
		return FZ94, nil
	}
	return -1, errors.New("Invalid or unknown law id")
}

type Price float64

func ParsePrice(str string) (Price, error) {
	price, err := strconv.ParseFloat(str, 64)
	if err != nil {
		return 0, err
	}
	// round order price with kopeika
	if price < 0 {
		price = -price
	}
	price = price * 100
	if math.Mod(price, 1) >= 0.5 {
		price = math.Ceil(price)
	} else {
		price = math.Floor(price)
	}
	return Price(price / 100), nil
}

const (
	_FIELD_LAW_ID int = iota
	_FIELD_ORDER_ID
	_FIELD_ORDER_TYPE
	_FIELD_ORDER_NAME
	_FIELD_EXHIBITION_NUMBER
	_FIELD_EXHIBITION_NAME
	_FIELD_START_ORDER_PRICE
	_FIELD_CURRENCY_ID
	_FIELD_OKDP
	_FIELD_OKPD
	_FIELD_ORGANISATION_NAME
	_FIELD_PUB_DATE
	_FIELD_LAST_EVENT_DATE
	_FIELD_ORDER_STAGE
	_FIELD_FEATURES
	_FIELD_START_FILING_DATE
	_FIELD_FINISH_FILING_DATE
	_ORDER_COLUMN_COUNT // result column count
)

type Order struct {
	LawId            OrderLaw  // Номер ФЗ
	OrderId          string    // Реестровый номер закупки
	OrderType        string    // Способ размещения закупки
	OrderName        string    // Наименование закупки
	ExhibitionNumber int       // Номер лота
	ExhibitionName   string    // Наименование лота
	StartOrderPrice  Price     // Начальная (максимальная) цена
	CurrencyId       string    // Код валюты
	OKDP             string    // Классификация по ОКДП
	OKPD             string    // Классификация по ОКПД
	OrganisationName string    // Организация, размещающая заказ
	PubDate          time.Time // Дата публикации
	LastEventDate    time.Time // Дата последнего события
	OrderStage       string    // Этап закупки (размещения заказа)
	Features         string    // Особенности размещения заказа
	StartFilingDate  time.Time // Дата начала подачи заявок
	FinishFilingDate time.Time // Дата окончания подачи заявок
	Errors           []error   // Ошибки при анализе закупки
}

func ParseOrder(rowBytes []byte) (*Order, error) {
	if len(rowBytes) == 0 {
		return nil, errors.New("Passed empty bytes row")
	}

	var (
		// Array of order fields
		row   [_ORDER_COLUMN_COUNT]string
		field int // index of current order field

		block  []byte // block
		quoted bool   // true if current block is quoted

		// Tokenes:
		//     i is index of `;`
		//     j is index of `"`
		//     k is index of `""`
		i, j, k int

		l int // parser position in rowBytes
	)

	for {
		i = bytes.IndexByte(rowBytes[l:], ';')
		j = bytes.IndexByte(rowBytes[l:], '"')
		k = bytes.Index(rowBytes[l:], []byte{'"', '"'})

		if (i > -1) && (i < j || j < 0) && (i < k || k < 0) {
			// next token is `;`
			if quoted {
				// `;` is part of block
				block = append(block, rowBytes[l:i+1]...)
			} else {
				// `;` is separator
				block = append(block, rowBytes[l:i]...)
				// write block
				row[field] = string(block)
				field++
			}
			l += i + 1

		} else if (j > -1) && (j < i || i < 0) && (j < k || k < 0) {
			// next token is `"`
			if quoted {
				block = append(block, rowBytes[l:j]...)
				row[field] = string(block)
				field++
			}
			l += j + 1
			quoted = !quoted

		} else if quoted && (k > -1) && (k < i || i < 0) && k == j {
			// next token is `""`
			block = append(block, rowBytes[l:k]...)
			// append only one double quote
			block = append(block, '"')
			l += k + 2
			row[field] = string(block)
			field++

		} else if i < 0 && j < 0 && k < 0 {
			block = append(block, rowBytes[l:])

		}
	}

	return NewOrder(row), nil
}

func ParseOrder2(rowBytes []byte) {
	if len(rowBytes) == 0 {
		fmt.Println("empty")
	}

	var (
		// Array of order fields
		// row   [_ORDER_COLUMN_COUNT]string
		// field int // index of current order field

		block  []byte // block
		quoted bool   // true if current block is quoted
		inside = true

		l int // parser cursor

		// Tokenes:
		//     l+i is current index of `;`
		//     l+j is current index of `"`
		//     l+k is current index of `""`
		i, j, k int
	)

	for l < len(rowBytes) {
		i = bytes.IndexByte(rowBytes[l:], ';')
		j = bytes.IndexByte(rowBytes[l:], '"')
		k = bytes.Index(rowBytes[l:], []byte{'"', '"'})

		if (i > -1) && (i < j || j < 0) && (i < k || k < 0) {
			// next token is `;`
			if quoted {
				// `;` is part of block
				block = append(block, rowBytes[l:l+i+1]...)
			} else {
				// `;` is separator
				block = append(block, rowBytes[l:l+i]...)
				// close block
				inside = false
			}
			l += i + 1

		} else if (j > -1) && (j < i || i < 0) && (j < k || k < 0 || !quoted) {
			// next token is `"`
			// if field are quoted
			if quoted {
				block = append(block, rowBytes[l:l+j]...)
				// ignore all bytes before next seporator `;`
				if i > j {
					j = i
				}
				// close block
				inside = false // ????????
			}
			l += j + 1
			quoted = !quoted

		} else if quoted && (k > -1) && (k < i || i < 0) && k == j {
			// next token is `""`
			// append only one double quote
			block = append(block, rowBytes[l:l+k+1]...)
			// but skip two
			l += k + 2
			continue

		} else if i < 0 && j < 0 && k < 0 {
			if quoted {
				fmt.Println("error")
				break
			}
			if inside {
				block = append(block, rowBytes[l:]...)
				inside = false
			} else {
				fmt.Println("err 2")
			}
			//fmt.Println("4:", string())
			// block = append(block, rowBytes[l:])
			l = len(rowBytes)
		}

		if !inside {
			// fmt.Println(inside)
			fmt.Printf("%q\n", block)
			block = block[:0]
			inside = true
		}
		// fmt.Println(inside)
	}
}

func NewOrder(row [_ORDER_COLUMN_COUNT]string) (order *Order) {
	order = &Order{
		OrderId:          strings.TrimLeft(row[_FIELD_ORDER_ID], "№"),
		OrderType:        row[_FIELD_ORDER_TYPE],
		OrderName:        row[_FIELD_ORDER_NAME],
		ExhibitionName:   row[_FIELD_EXHIBITION_NAME],
		CurrencyId:       row[_FIELD_CURRENCY_ID],
		OKDP:             row[_FIELD_OKDP],
		OKPD:             row[_FIELD_OKPD],
		OrganisationName: row[_FIELD_ORGANISATION_NAME],
		OrderStage:       row[_FIELD_ORDER_STAGE],
		Features:         row[_FIELD_FEATURES],
	}
	var err error
	order.LawId, err = ParseLow(row[_FIELD_LAW_ID])
	if err != nil {
		order.PushError(err)
	}
	if len(row[_FIELD_EXHIBITION_NUMBER]) > 0 {
		// Только для многолотовых закупок по ФЗ 223
		order.ExhibitionNumber, err =
			strconv.Atoi(row[_FIELD_EXHIBITION_NUMBER])
		if err != nil {
			order.PushError(errors.New("Invalid exhibition number: " +
				err.Error()))
		}
	}
	order.StartOrderPrice, err =
		ParsePrice(row[_FIELD_START_ORDER_PRICE])
	if err != nil {
		order.PushError(errors.New("Invalid order price: " +
			err.Error()))
	}
	if len(row[_FIELD_CURRENCY_ID]) == 0 {
		order.PushError(errors.New("Unknown currency"))
	}
	order.PubDate, err = ParseRusFormatDate(row[_FIELD_PUB_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown publish date: " +
			err.Error()))
	}
	order.LastEventDate, err =
		ParseRusFormatDate(row[_FIELD_LAST_EVENT_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown last event date: " +
			err.Error()))
	}
	order.StartFilingDate, err =
		ParseRusFormatDate(row[_FIELD_START_FILING_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown start filing date: " +
			err.Error()))
	}
	order.FinishFilingDate, err =
		ParseRusFormatDate(row[_FIELD_FINISH_FILING_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown finish filing date: " +
			err.Error()))
	}
	return
}

func (order *Order) PushError(err error) {
	if err != nil {
		order.Errors = append(order.Errors, err)
	}
}
<commit_msg>-m "must be tested"<commit_after>package main

import (
	"bytes"
	"errors"
	"math"
	"strconv"
	"strings"
	"time"
)

type OrderLaw int

const (
	// to correct template compilation it cannot starts from zero
	FZ44 OrderLaw = iota + 1
	FZ223
	FZ94
)

func ParseLow(str string) (OrderLaw, error) {
	switch {
	case strings.Contains(str, "44"):
		return FZ44, nil
	case strings.Contains(str, "223"):
		return FZ223, nil
	case strings.Contains(str, "94"):
		return FZ94, nil
	}
	return -1, errors.New("Invalid or unknown law id")
}

type Price float64

func ParsePrice(str string) (Price, error) {
	price, err := strconv.ParseFloat(str, 64)
	if err != nil {
		return 0, err
	}
	// round order price with kopeika
	if price < 0 {
		price = -price
	}
	price = price * 100
	if math.Mod(price, 1) >= 0.5 {
		price = math.Ceil(price)
	} else {
		price = math.Floor(price)
	}
	return Price(price / 100), nil
}

const (
	_FIELD_LAW_ID int = iota
	_FIELD_ORDER_ID
	_FIELD_ORDER_TYPE
	_FIELD_ORDER_NAME
	_FIELD_EXHIBITION_NUMBER
	_FIELD_EXHIBITION_NAME
	_FIELD_START_ORDER_PRICE
	_FIELD_CURRENCY_ID
	_FIELD_OKDP
	_FIELD_OKPD
	_FIELD_ORGANISATION_NAME
	_FIELD_PUB_DATE
	_FIELD_LAST_EVENT_DATE
	_FIELD_ORDER_STAGE
	_FIELD_FEATURES
	_FIELD_START_FILING_DATE
	_FIELD_FINISH_FILING_DATE
	_ORDER_COLUMN_COUNT // result column count
)

type Order struct {
	LawId            OrderLaw  // Номер ФЗ
	OrderId          string    // Реестровый номер закупки
	OrderType        string    // Способ размещения закупки
	OrderName        string    // Наименование закупки
	ExhibitionNumber int       // Номер лота
	ExhibitionName   string    // Наименование лота
	StartOrderPrice  Price     // Начальная (максимальная) цена
	CurrencyId       string    // Код валюты
	OKDP             string    // Классификация по ОКДП
	OKPD             string    // Классификация по ОКПД
	OrganisationName string    // Организация, размещающая заказ
	PubDate          time.Time // Дата публикации
	LastEventDate    time.Time // Дата последнего события
	OrderStage       string    // Этап закупки (размещения заказа)
	Features         string    // Особенности размещения заказа
	StartFilingDate  time.Time // Дата начала подачи заявок
	FinishFilingDate time.Time // Дата окончания подачи заявок
	Errors           []error   // Ошибки при анализе закупки
}

const _CSV_FIELD_SEPARATOR = ';'

func ParseOrder(rowBytes []byte) (*Order, error) {
	if len(rowbyte) == 0 {
		return nil, errors.New("passed empty rowbyte")
	}
	rowbyte = append(rowbyte, ';')

	var (
		row   [_ORDER_COLUMN_COUNT]string // array of order fields
		field int                         // index of current field

		buff   []byte // current block
		allow  = true // true if character is expected
		quoted bool   // true if current block is quoted
	)

	for i := 0; i < len(rowbyte) && field < _ORDER_COLUMN_COUNT; i++ {
		if rowbyte[i] == _CSV_FIELD_SEPARATOR {
			if quoted {
				// quoted separator is part of field
				buff = append(buff, rowbyte[i])
			} else {
				// separator is end of string
				// save
				row[field] = string(buff)
				field++
				// remove
				buff = buff[:0]
				allow = true
			}
		} else if allow {
			if rowbyte[i] == '"' {
				if quoted {
					if i+1 < len(rowbyte) && rowbyte[i+1] == '"' {
						// append only one double quote
						buff = append(buff, '"')
						// skip second quote
						i += 1
					} else {
						// closing quote
						quoted = false
						allow = false
					}
				} else if len(buff) == 0 {
					// new quoted field
					// opening quote
					quoted = true
				} else {
					// quote inside unquoted field
					return nil, errors.New("unexpected quote")
				}
			} else {
				// append eny char
				buff = append(buff, rowbyte[i])
			}
		} else {
			// block is closed
			return nil, errors.New("unexpected character")
		}
	}
	if field < _ORDER_COLUMN_COUNT-1 {
		return nil, errors.New("bad field count")
	}
	return NewOrder(row), nil
}

func NewOrder(row [_ORDER_COLUMN_COUNT]string) (order *Order) {
	order = &Order{
		OrderId:          strings.TrimLeft(row[_FIELD_ORDER_ID], "№"),
		OrderType:        row[_FIELD_ORDER_TYPE],
		OrderName:        row[_FIELD_ORDER_NAME],
		ExhibitionName:   row[_FIELD_EXHIBITION_NAME],
		CurrencyId:       row[_FIELD_CURRENCY_ID],
		OKDP:             row[_FIELD_OKDP],
		OKPD:             row[_FIELD_OKPD],
		OrganisationName: row[_FIELD_ORGANISATION_NAME],
		OrderStage:       row[_FIELD_ORDER_STAGE],
		Features:         row[_FIELD_FEATURES],
	}
	var err error
	order.LawId, err = ParseLow(row[_FIELD_LAW_ID])
	if err != nil {
		order.PushError(err)
	}
	if len(row[_FIELD_EXHIBITION_NUMBER]) > 0 {
		// Только для многолотовых закупок по ФЗ 223
		order.ExhibitionNumber, err =
			strconv.Atoi(row[_FIELD_EXHIBITION_NUMBER])
		if err != nil {
			order.PushError(errors.New("Invalid exhibition number: " +
				err.Error()))
		}
	}
	order.StartOrderPrice, err =
		ParsePrice(row[_FIELD_START_ORDER_PRICE])
	if err != nil {
		order.PushError(errors.New("Invalid order price: " +
			err.Error()))
	}
	if len(row[_FIELD_CURRENCY_ID]) == 0 {
		order.PushError(errors.New("Unknown currency"))
	}
	order.PubDate, err = ParseRusFormatDate(row[_FIELD_PUB_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown publish date: " +
			err.Error()))
	}
	order.LastEventDate, err =
		ParseRusFormatDate(row[_FIELD_LAST_EVENT_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown last event date: " +
			err.Error()))
	}
	order.StartFilingDate, err =
		ParseRusFormatDate(row[_FIELD_START_FILING_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown start filing date: " +
			err.Error()))
	}
	order.FinishFilingDate, err =
		ParseRusFormatDate(row[_FIELD_FINISH_FILING_DATE])
	if err != nil {
		order.PushError(errors.New("Unknown finish filing date: " +
			err.Error()))
	}
	return
}

func (order *Order) PushError(err error) {
	if err != nil {
		order.Errors = append(order.Errors, err)
	}
}
<|endoftext|>


<commit_before>package operation

import (
	"bytes"
	"encoding/json"
	"errors"
	_ "fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
)

type UploadResult struct {
	Size  int
	Error string
}

func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	file_writer, err := body_writer.CreateFormFile("file", filename)
	if err != nil {
		log.Println("error creating form file", err)
		return nil, err
	}
	if _, err = io.Copy(file_writer, reader); err != nil {
		log.Println("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err = body_writer.Close(); err != nil {
		log.Println("error closing body", err)
		return nil, err
	}
	resp, err := http.Post(uploadUrl, content_type, body_buf)
	if err != nil {
		log.Println("failing to upload to", uploadUrl)
		return nil, err
	}
	defer resp.Body.Close()
	resp_body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var ret UploadResult
	err = json.Unmarshal(resp_body, &ret)
	if err != nil {
		log.Println("failing to read upload resonse", uploadUrl, resp_body)
		return nil, err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	return &ret, nil
}
<commit_msg>Issue 28:	[HELP NEEDED] weed upload does not send correct mime type to weedfs<commit_after>package operation

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
)

type UploadResult struct {
	Size  int
	Error string
}

func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
	h.Set("Content-Type", mime.TypeByExtension(filepath.Ext(filename)))
	file_writer, err := body_writer.CreatePart(h)
	if err != nil {
		log.Println("error creating form file", err)
		return nil, err
	}
	if _, err = io.Copy(file_writer, reader); err != nil {
		log.Println("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err = body_writer.Close(); err != nil {
		log.Println("error closing body", err)
		return nil, err
	}
	resp, err := http.Post(uploadUrl, content_type, body_buf)
	if err != nil {
		log.Println("failing to upload to", uploadUrl)
		return nil, err
	}
	defer resp.Body.Close()
	resp_body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var ret UploadResult
	err = json.Unmarshal(resp_body, &ret)
	if err != nil {
		log.Println("failing to read upload resonse", uploadUrl, resp_body)
		return nil, err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	return &ret, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

func main() {
	utils.Startup("broker", false)
	utils.RunStatusLogger()

	utils.AmqpAutoReconnect("broker", func(consumeConn, publishConn *amqp.Connection) {
		consumeChannel := utils.CreateAmqpChannel(consumeConn)
		defer consumeChannel.Close()

		routeMap := make(map[string]([]chan<- interface{}))
		var routeMapMutex sync.RWMutex

		service := sockjs.NewService("http://localhost/sockjs.js", true, false, 10*time.Minute, 0, func(receiveChan <-chan interface{}, sendChan chan<- interface{}) {
			defer log.RecoverAndLog()

			r := make([]byte, 128/8)
			rand.Read(r)
			socketId := base64.StdEncoding.EncodeToString(r)

			utils.ChangeNumClients <- 1
			log.Debug("Client connected: " + socketId)
			defer func() {
				utils.ChangeNumClients <- -1
				log.Debug("Client disconnected: " + socketId)
			}()

			addToRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], sendChan)
				routeMapMutex.Unlock()
			}
			removeFromRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				channels := routeMap[routingKeyPrefix]
				for i, channel := range channels {
					if channel == sendChan {
						channels[i] = channels[len(channels)-1]
						routeMap[routingKeyPrefix] = channels[:len(channels)-1]
						break
					}
				}
				routeMapMutex.Unlock()
			}

			subscriptions := make(map[string]bool)

			controlChannel := utils.CreateAmqpChannel(publishConn)
			defer func() { controlChannel.Close() }() // controlChannel is replaced on error

			err := controlChannel.Publish("auth", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
			if err != nil {
				panic(err)
			}

			defer func() {
				for routingKeyPrefix := range subscriptions {
					removeFromRouteMap(routingKeyPrefix)
				}
				for {
					err := controlChannel.Publish("auth", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
					if err != nil {
						log.LogError(err)
						controlChannel.Close()
						controlChannel = utils.CreateAmqpChannel(publishConn)
					} else {
						break
					}
				}
			}()

			for data := range receiveChan {
				func() {
					defer log.RecoverAndLog()

					message := data.(map[string]interface{})
					log.Debug(message)

					action := message["action"]
					switch action {
					case "subscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						addToRouteMap(routingKeyPrefix)
						subscriptions[routingKeyPrefix] = true

						body, err := json.Marshal(map[string]string{"routingKey": "broker.subscribed", "payload": routingKeyPrefix})
						if err != nil {
							panic(err)
						}
						sendChan <- string(body)

					case "unsubscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						removeFromRouteMap(routingKeyPrefix)
						delete(subscriptions, routingKeyPrefix)

					case "publish":
						exchange := message["exchange"].(string)
						routingKey := message["routingKey"].(string)
						if strings.HasPrefix(routingKey, "client.") {
							for {
								err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
								if err != nil {
									log.LogError(err)
									controlChannel.Close()
									controlChannel = utils.CreateAmqpChannel(publishConn)
								} else {
									break
								}
							}
						} else {
							log.Warn(fmt.Sprintf("Invalid routing key: %v", message))
						}

					default:
						log.Warn(fmt.Sprintf("Invalid action: %v", message))

					}
				}()
			}
		})
		defer service.Close()

		server := &http.Server{Handler: &sockjs.Mux{
			Services: map[string]*sockjs.Service{
				"/subscribe": service,
			},
		}}
		listener, err := net.Listen("tcp", ":8008")
		if err != nil {
			panic(err)
		}
		defer listener.Close()

		go func() {
			err = server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
			}
			consumeConn.Close()
		}()

		stream := utils.DeclareBindConsumeAmqpQueue(consumeChannel, "topic", "broker", "#")
		for message := range stream {
			routingKey := message.RoutingKey
			body, err := json.Marshal(map[string]string{"routingKey": routingKey, "payload": string(message.Body)})
			if err != nil {
				panic(err)
			}
			bodyStr := string(body)

			pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
			for {
				index := strings.IndexRune(routingKey[pos+1:], '.')
				if index == -1 {
					break
				}
				pos += 1 + index
				prefix := routingKey[:pos]
				routeMapMutex.RLock()
				channels := routeMap[prefix]
				routeMapMutex.RUnlock()
				for _, channel := range channels {
					select {
					case channel <- bodyStr:
						// successful
					default:
						log.Warn("Dropped message")
					}
				}
			}
		}
	})
}
<commit_msg>broker: Also match whole routingKey.<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

func main() {
	utils.Startup("broker", false)
	utils.RunStatusLogger()

	utils.AmqpAutoReconnect("broker", func(consumeConn, publishConn *amqp.Connection) {
		consumeChannel := utils.CreateAmqpChannel(consumeConn)
		defer consumeChannel.Close()

		routeMap := make(map[string]([]chan<- interface{}))
		var routeMapMutex sync.RWMutex

		service := sockjs.NewService("http://localhost/sockjs.js", true, false, 10*time.Minute, 0, func(receiveChan <-chan interface{}, sendChan chan<- interface{}) {
			defer log.RecoverAndLog()

			r := make([]byte, 128/8)
			rand.Read(r)
			socketId := base64.StdEncoding.EncodeToString(r)

			utils.ChangeNumClients <- 1
			log.Debug("Client connected: " + socketId)
			defer func() {
				utils.ChangeNumClients <- -1
				log.Debug("Client disconnected: " + socketId)
			}()

			addToRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], sendChan)
				routeMapMutex.Unlock()
			}
			removeFromRouteMap := func(routingKeyPrefix string) {
				routeMapMutex.Lock()
				channels := routeMap[routingKeyPrefix]
				for i, channel := range channels {
					if channel == sendChan {
						channels[i] = channels[len(channels)-1]
						routeMap[routingKeyPrefix] = channels[:len(channels)-1]
						break
					}
				}
				routeMapMutex.Unlock()
			}

			subscriptions := make(map[string]bool)

			controlChannel := utils.CreateAmqpChannel(publishConn)
			defer func() { controlChannel.Close() }() // controlChannel is replaced on error

			err := controlChannel.Publish("auth", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
			if err != nil {
				panic(err)
			}

			defer func() {
				for routingKeyPrefix := range subscriptions {
					removeFromRouteMap(routingKeyPrefix)
				}
				for {
					err := controlChannel.Publish("auth", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
					if err != nil {
						log.LogError(err)
						controlChannel.Close()
						controlChannel = utils.CreateAmqpChannel(publishConn)
					} else {
						break
					}
				}
			}()

			for data := range receiveChan {
				func() {
					defer log.RecoverAndLog()

					message := data.(map[string]interface{})
					log.Debug(message)

					action := message["action"]
					switch action {
					case "subscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						addToRouteMap(routingKeyPrefix)
						subscriptions[routingKeyPrefix] = true

						body, err := json.Marshal(map[string]string{"routingKey": "broker.subscribed", "payload": routingKeyPrefix})
						if err != nil {
							panic(err)
						}
						sendChan <- string(body)

					case "unsubscribe":
						routingKeyPrefix := message["routingKeyPrefix"].(string)
						removeFromRouteMap(routingKeyPrefix)
						delete(subscriptions, routingKeyPrefix)

					case "publish":
						exchange := message["exchange"].(string)
						routingKey := message["routingKey"].(string)
						if strings.HasPrefix(routingKey, "client.") {
							for {
								err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
								if err != nil {
									log.LogError(err)
									controlChannel.Close()
									controlChannel = utils.CreateAmqpChannel(publishConn)
								} else {
									break
								}
							}
						} else {
							log.Warn(fmt.Sprintf("Invalid routing key: %v", message))
						}

					default:
						log.Warn(fmt.Sprintf("Invalid action: %v", message))

					}
				}()
			}
		})
		defer service.Close()

		server := &http.Server{Handler: &sockjs.Mux{
			Services: map[string]*sockjs.Service{
				"/subscribe": service,
			},
		}}
		listener, err := net.Listen("tcp", ":8008")
		if err != nil {
			panic(err)
		}
		defer listener.Close()

		go func() {
			err = server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
			}
			consumeConn.Close()
		}()

		stream := utils.DeclareBindConsumeAmqpQueue(consumeChannel, "topic", "broker", "#")
		for message := range stream {
			routingKey := message.RoutingKey
			body, err := json.Marshal(map[string]string{"routingKey": routingKey, "payload": string(message.Body)})
			if err != nil {
				panic(err)
			}
			bodyStr := string(body)

			pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
			for pos != -1 && pos < len(routingKey) {
				index := strings.IndexRune(routingKey[pos+1:], '.')
				if index == -1 {
					pos = len(routingKey)
				} else {
					pos += 1 + index
				}
				prefix := routingKey[:pos]
				routeMapMutex.RLock()
				channels := routeMap[prefix]
				routeMapMutex.RUnlock()
				for _, channel := range channels {
					select {
					case channel <- bodyStr:
						// successful
					default:
						log.Warn("Dropped message")
					}
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package client

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const computeScope = compute.ComputeScope
const storageScope = storage.DevstorageFull_controlScope

type GoogleClient struct {
	project        string
	jsonKey        string
	defaultZone    string
	computeService *compute.Service
	storageService *storage.Service
}

func NewGoogleClient(
	project string,
	jsonKey string,
	defaultZone string,
) (GoogleClient, error) {
	computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(jsonKey), computeScope)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
	}

	computeClient := computeJwtConf.Client(oauth2.NoContext)
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}

	storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(jsonKey), storageScope)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
	}

	storageClient := storageJwtConf.Client(oauth2.NoContext)
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}

	return GoogleClient{
		project:        project,
		jsonKey:        jsonKey,
		defaultZone:    defaultZone,
		computeService: computeService,
		storageService: storageService,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.project
}

func (c GoogleClient) DefaultZone() string {
	return c.defaultZone
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<commit_msg>devstorage scope name constant has changed at latest compute api<commit_after>package client

import (
	bosherr "github.com/cloudfoundry/bosh-agent/errors"

	"golang.org/x/oauth2"
	oauthgoogle "golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/storage/v1"
)

const computeScope = compute.ComputeScope
const storageScope = storage.DevstorageFullControlScope

type GoogleClient struct {
	project        string
	jsonKey        string
	defaultZone    string
	computeService *compute.Service
	storageService *storage.Service
}

func NewGoogleClient(
	project string,
	jsonKey string,
	defaultZone string,
) (GoogleClient, error) {
	computeJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(jsonKey), computeScope)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
	}

	computeClient := computeJwtConf.Client(oauth2.NoContext)
	computeService, err := compute.New(computeClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Compute Service client")
	}

	storageJwtConf, err := oauthgoogle.JWTConfigFromJSON([]byte(jsonKey), storageScope)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Reading Google JSON Key")
	}

	storageClient := storageJwtConf.Client(oauth2.NoContext)
	storageService, err := storage.New(storageClient)
	if err != nil {
		return GoogleClient{}, bosherr.WrapError(err, "Creating a Google Storage Service client")
	}

	return GoogleClient{
		project:        project,
		jsonKey:        jsonKey,
		defaultZone:    defaultZone,
		computeService: computeService,
		storageService: storageService,
	}, nil
}

func (c GoogleClient) Project() string {
	return c.project
}

func (c GoogleClient) DefaultZone() string {
	return c.defaultZone
}

func (c GoogleClient) ComputeService() *compute.Service {
	return c.computeService
}

func (c GoogleClient) StorageService() *storage.Service {
	return c.storageService
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"encoding/gob"
	"errors"
	"net/http"
	"net/url"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/satori/go.uuid"
	"google.golang.org/api/plus/v1"

	"github.com/GoogleCloudPlatform/golang-samples/getting-started/bookshelf"
)

const (
	defaultSessionID = "default"
	// The following keys are used for the default session. For example:
	//  session, _ := bookshelf.SessionStore.New(r, defaultSessionID)
	//  session.Values[oauthTokenSessionKey]
	googleProfileSessionKey = "google_profile"
	oauthTokenSessionKey    = "oauth_token"

	// This key is used in the OAuth flow session to store the URL to redirect the
	// user to after the OAuth flow is complete.
	oauthFlowRedirectKey = "redirect"
)

func init() {
	// Gob encoding for gorilla/sessions
	gob.Register(&oauth2.Token{})
	gob.Register(&Profile{})
}

// loginHandler initiates an OAuth flow to authenticate the user.
func loginHandler(w http.ResponseWriter, r *http.Request) *appError {
	sessionID := uuid.NewV4().String()

	oauthFlowSession, err := bookshelf.SessionStore.New(r, sessionID)
	if err != nil {
		return appErrorf(err, "could not create oauth session: %v", err)
	}
	oauthFlowSession.Options.MaxAge = 10 * 60 // 10 minutes

	redirectURL, err := validateRedirectURL(r.FormValue("redirect"))
	if err != nil {
		return appErrorf(err, "invalid redirect URL: %v", err)
	}
	oauthFlowSession.Values[oauthFlowRedirectKey] = redirectURL

	if err := oauthFlowSession.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}

	// Use the session ID for the "state" parameter.
	// This protects against CSRF (cross-site request forgery).
	// See https://godoc.org/golang.org/x/oauth2#Config.AuthCodeURL for more detail.
	url := bookshelf.OAuthConfig.AuthCodeURL(sessionID, oauth2.ApprovalForce,
		oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusFound)
	return nil
}

// validateRedirectURL checks that the URL provided is valid.
// If the URL is missing, redirect the user to the application's root.
// The URL must not be absolute (i.e., the URL must refer to a path within this
// application).
func validateRedirectURL(path string) (string, error) {
	if path == "" {
		return "/", nil
	}

	// Ensure redirect URL is valid and not pointing to a different server.
	parsedURL, err := url.Parse(path)
	if err != nil {
		return "/", err
	}
	if parsedURL.IsAbs() {
		return "/", errors.New("URL must be absolute")
	}
	return path, nil
}

// oauthCallbackHandler completes the OAuth flow, retreives the user's profile
// information and stores it in a session.
func oauthCallbackHandler(w http.ResponseWriter, r *http.Request) *appError {
	oauthFlowSession, err := bookshelf.SessionStore.Get(r, r.FormValue("state"))
	if err != nil {
		return appErrorf(err, "invalid state parameter. try logging in again.")
	}

	redirectURL, ok := oauthFlowSession.Values[oauthFlowRedirectKey].(string)
	// Validate this callback request came from the app.
	if !ok {
		return appErrorf(err, "invalid state parameter. try logging in again.")
	}

	code := r.FormValue("code")
	tok, err := bookshelf.OAuthConfig.Exchange(context.Background(), code)
	if err != nil {
		return appErrorf(err, "could not get auth token: %v", err)
	}

	session, err := bookshelf.SessionStore.New(r, defaultSessionID)
	if err != nil {
		return appErrorf(err, "could not get default session: %v", err)
	}

	ctx := context.Background()
	profile, err := fetchProfile(ctx, tok)
	if err != nil {
		return appErrorf(err, "could not fetch Google profile: %v", err)
	}

	session.Values[oauthTokenSessionKey] = tok
	// Strip the profile to only the fields we need. Otherwise the struct is too big.
	session.Values[googleProfileSessionKey] = stripProfile(profile)
	if err := session.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}

	http.Redirect(w, r, redirectURL, http.StatusFound)
	return nil
}

// fetchProfile retrieves the Google+ profile of the user associated with the
// provided OAuth token.
func fetchProfile(ctx context.Context, tok *oauth2.Token) (*plus.Person, error) {
	client := oauth2.NewClient(ctx, bookshelf.OAuthConfig.TokenSource(ctx, tok))
	plusService, err := plus.New(client)
	if err != nil {
		return nil, err
	}
	return plusService.People.Get("me").Do()
}

// logoutHandler clears the default session.
func logoutHandler(w http.ResponseWriter, r *http.Request) *appError {
	session, err := bookshelf.SessionStore.New(r, defaultSessionID)
	if err != nil {
		return appErrorf(err, "could not get default session: %v", err)
	}
	session.Options.MaxAge = -1 // Clear session.
	if err := session.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}
	redirectURL := r.FormValue("redirect")
	if redirectURL == "" {
		redirectURL = "/"
	}
	http.Redirect(w, r, redirectURL, http.StatusFound)
	return nil
}

// profileFromSession retreives the Google+ profile from the default session.
// Returns nil if the profile cannot be retreived (e.g. user is logged out).
func profileFromSession(r *http.Request) *Profile {
	session, err := bookshelf.SessionStore.Get(r, defaultSessionID)
	if err != nil {
		return nil
	}
	tok, ok := session.Values[oauthTokenSessionKey].(*oauth2.Token)
	if !ok || !tok.Valid() {
		return nil
	}
	profile, ok := session.Values[googleProfileSessionKey].(*Profile)
	if !ok {
		return nil
	}
	return profile
}

type Profile struct {
	ID, DisplayName, ImageURL string
}

// stripProfile returns a subset of a plus.Person.
func stripProfile(p *plus.Person) *Profile {
	return &Profile{
		ID:          p.Id,
		DisplayName: p.DisplayName,
		ImageURL:    p.Image.Url,
	}
}
<commit_msg>getting-started/bookshelf/app: fix auth error message (#325)<commit_after>// Copyright 2015 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"encoding/gob"
	"errors"
	"net/http"
	"net/url"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"

	"github.com/satori/go.uuid"
	"google.golang.org/api/plus/v1"

	"github.com/GoogleCloudPlatform/golang-samples/getting-started/bookshelf"
)

const (
	defaultSessionID = "default"
	// The following keys are used for the default session. For example:
	//  session, _ := bookshelf.SessionStore.New(r, defaultSessionID)
	//  session.Values[oauthTokenSessionKey]
	googleProfileSessionKey = "google_profile"
	oauthTokenSessionKey    = "oauth_token"

	// This key is used in the OAuth flow session to store the URL to redirect the
	// user to after the OAuth flow is complete.
	oauthFlowRedirectKey = "redirect"
)

func init() {
	// Gob encoding for gorilla/sessions
	gob.Register(&oauth2.Token{})
	gob.Register(&Profile{})
}

// loginHandler initiates an OAuth flow to authenticate the user.
func loginHandler(w http.ResponseWriter, r *http.Request) *appError {
	sessionID := uuid.NewV4().String()

	oauthFlowSession, err := bookshelf.SessionStore.New(r, sessionID)
	if err != nil {
		return appErrorf(err, "could not create oauth session: %v", err)
	}
	oauthFlowSession.Options.MaxAge = 10 * 60 // 10 minutes

	redirectURL, err := validateRedirectURL(r.FormValue("redirect"))
	if err != nil {
		return appErrorf(err, "invalid redirect URL: %v", err)
	}
	oauthFlowSession.Values[oauthFlowRedirectKey] = redirectURL

	if err := oauthFlowSession.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}

	// Use the session ID for the "state" parameter.
	// This protects against CSRF (cross-site request forgery).
	// See https://godoc.org/golang.org/x/oauth2#Config.AuthCodeURL for more detail.
	url := bookshelf.OAuthConfig.AuthCodeURL(sessionID, oauth2.ApprovalForce,
		oauth2.AccessTypeOnline)
	http.Redirect(w, r, url, http.StatusFound)
	return nil
}

// validateRedirectURL checks that the URL provided is valid.
// If the URL is missing, redirect the user to the application's root.
// The URL must not be absolute (i.e., the URL must refer to a path within this
// application).
func validateRedirectURL(path string) (string, error) {
	if path == "" {
		return "/", nil
	}

	// Ensure redirect URL is valid and not pointing to a different server.
	parsedURL, err := url.Parse(path)
	if err != nil {
		return "/", err
	}
	if parsedURL.IsAbs() {
		return "/", errors.New("URL must not be absolute")
	}
	return path, nil
}

// oauthCallbackHandler completes the OAuth flow, retreives the user's profile
// information and stores it in a session.
func oauthCallbackHandler(w http.ResponseWriter, r *http.Request) *appError {
	oauthFlowSession, err := bookshelf.SessionStore.Get(r, r.FormValue("state"))
	if err != nil {
		return appErrorf(err, "invalid state parameter. try logging in again.")
	}

	redirectURL, ok := oauthFlowSession.Values[oauthFlowRedirectKey].(string)
	// Validate this callback request came from the app.
	if !ok {
		return appErrorf(err, "invalid state parameter. try logging in again.")
	}

	code := r.FormValue("code")
	tok, err := bookshelf.OAuthConfig.Exchange(context.Background(), code)
	if err != nil {
		return appErrorf(err, "could not get auth token: %v", err)
	}

	session, err := bookshelf.SessionStore.New(r, defaultSessionID)
	if err != nil {
		return appErrorf(err, "could not get default session: %v", err)
	}

	ctx := context.Background()
	profile, err := fetchProfile(ctx, tok)
	if err != nil {
		return appErrorf(err, "could not fetch Google profile: %v", err)
	}

	session.Values[oauthTokenSessionKey] = tok
	// Strip the profile to only the fields we need. Otherwise the struct is too big.
	session.Values[googleProfileSessionKey] = stripProfile(profile)
	if err := session.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}

	http.Redirect(w, r, redirectURL, http.StatusFound)
	return nil
}

// fetchProfile retrieves the Google+ profile of the user associated with the
// provided OAuth token.
func fetchProfile(ctx context.Context, tok *oauth2.Token) (*plus.Person, error) {
	client := oauth2.NewClient(ctx, bookshelf.OAuthConfig.TokenSource(ctx, tok))
	plusService, err := plus.New(client)
	if err != nil {
		return nil, err
	}
	return plusService.People.Get("me").Do()
}

// logoutHandler clears the default session.
func logoutHandler(w http.ResponseWriter, r *http.Request) *appError {
	session, err := bookshelf.SessionStore.New(r, defaultSessionID)
	if err != nil {
		return appErrorf(err, "could not get default session: %v", err)
	}
	session.Options.MaxAge = -1 // Clear session.
	if err := session.Save(r, w); err != nil {
		return appErrorf(err, "could not save session: %v", err)
	}
	redirectURL := r.FormValue("redirect")
	if redirectURL == "" {
		redirectURL = "/"
	}
	http.Redirect(w, r, redirectURL, http.StatusFound)
	return nil
}

// profileFromSession retreives the Google+ profile from the default session.
// Returns nil if the profile cannot be retreived (e.g. user is logged out).
func profileFromSession(r *http.Request) *Profile {
	session, err := bookshelf.SessionStore.Get(r, defaultSessionID)
	if err != nil {
		return nil
	}
	tok, ok := session.Values[oauthTokenSessionKey].(*oauth2.Token)
	if !ok || !tok.Valid() {
		return nil
	}
	profile, ok := session.Values[googleProfileSessionKey].(*Profile)
	if !ok {
		return nil
	}
	return profile
}

type Profile struct {
	ID, DisplayName, ImageURL string
}

// stripProfile returns a subset of a plus.Person.
func stripProfile(p *plus.Person) *Profile {
	return &Profile{
		ID:          p.Id,
		DisplayName: p.DisplayName,
		ImageURL:    p.Image.Url,
	}
}
<|endoftext|>


<commit_before>package boomer

import (
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"sync/atomic"
	"time"
)

const (
	stateInit     = "ready"
	stateHatching = "hatching"
	stateRunning  = "running"
	stateStopped  = "stopped"
)

const (
	slaveReportInterval = 3 * time.Second
)

// Task is like locust's task.
// when boomer receive start message, it will spawn several
// goroutines to run Task.Fn .
type Task struct {
	Weight int
	Fn     func()
	Name   string
}

type runner struct {
	tasks       []*Task
	numClients  int
	hatchRate   int
	stopChannel chan bool
	state       string
	client      client
	nodeID      string
}

func (r *runner) safeRun(fn func()) {
	defer func() {
		// don't panic
		err := recover()
		if err != nil {
			debug.PrintStack()
			Events.Publish("request_failure", "unknown", "panic", 0.0, fmt.Sprintf("%v", err))
		}
	}()
	fn()
}

func (r *runner) spawnGoRoutines(spawnCount int, quit chan bool) {

	log.Println("Hatching and swarming", spawnCount, "clients at the rate", r.hatchRate, "clients/s...")

	weightSum := 0
	for _, task := range r.tasks {
		weightSum += task.Weight
	}

	for _, task := range r.tasks {

		percent := float64(task.Weight) / float64(weightSum)
		amount := int(round(float64(spawnCount)*percent, .5, 0))

		if weightSum == 0 {
			amount = int(float64(spawnCount) / float64(len(r.tasks)))
		}

		for i := 1; i <= amount; i++ {
			if i%r.hatchRate == 0 {
				time.Sleep(1 * time.Second)
			}
			go func(fn func()) {
				for {
					select {
					case <-quit:
						return
					default:
						if maxRPSEnabled {
							token := atomic.AddInt64(&maxRPSThreshold, -1)
							if token < 0 {
								// max RPS is reached, wait until next second
								<-maxRPSControlChannel
							} else {
								r.safeRun(fn)
							}
						} else {
							r.safeRun(fn)
						}
					}
				}
			}(task.Fn)
		}

	}

	r.hatchComplete()

}

func (r *runner) startHatching(spawnCount int, hatchRate int) {

	if r.state != stateRunning && r.state != stateHatching {
		clearStatsChannel <- true
		r.stopChannel = make(chan bool)
	}

	if r.state == stateRunning {
		// stop previous goroutines without blocking
		// those goroutines will exit when r.safeRun returns
		close(r.stopChannel)
	}

	r.stopChannel = make(chan bool)
	r.state = stateHatching

	r.hatchRate = hatchRate
	r.numClients = spawnCount
	r.spawnGoRoutines(r.numClients, r.stopChannel)

}

func (r *runner) hatchComplete() {

	data := make(map[string]interface{})
	data["count"] = r.numClients
	toServer <- newMessage("hatch_complete", data, r.nodeID)

	r.state = stateRunning
}

func (r *runner) onQuiting() {
	toServer <- newMessage("quit", nil, r.nodeID)
}

func (r *runner) stop() {

	if r.state == stateRunning {
		close(r.stopChannel)
		r.state = stateStopped
		log.Println("Recv stop message from master, all the goroutines are stopped")
	}

}

func (r *runner) getReady() {

	r.state = stateInit

	// read message from server
	go func() {
		for {
			msg := <-fromServer
			switch msg.Type {
			case "hatch":
				toServer <- newMessage("hatching", nil, r.nodeID)
				rate, _ := msg.Data["hatch_rate"]
				clients, _ := msg.Data["num_clients"]
				hatchRate := rate.(float64)
				workers := 0
				if _, ok := clients.(uint64); ok {
					workers = int(clients.(uint64))
				} else {
					workers = int(clients.(int64))
				}
				r.startHatching(workers, int(hatchRate))
			case "stop":
				r.stop()
				toServer <- newMessage("client_stopped", nil, r.nodeID)
				toServer <- newMessage("client_ready", nil, r.nodeID)
			case "quit":
				log.Println("Got quit message from master, shutting down...")
				os.Exit(0)
			}
		}
	}()

	// tell master, I'm ready
	toServer <- newMessage("client_ready", nil, r.nodeID)

	// report to server
	go func() {
		for {
			select {
			case data := <-messageToServerChannel:
				data["user_count"] = r.numClients
				toServer <- newMessage("stats", data, r.nodeID)
			}
		}
	}()

	if maxRPSEnabled {
		go func() {
			for {
				atomic.StoreInt64(&maxRPSThreshold, maxRPS)
				time.Sleep(1 * time.Second)
				// use channel to broadcast
				close(maxRPSControlChannel)
				maxRPSControlChannel = make(chan bool)
			}
		}()
	}
}
<commit_msg>ignore invalid hatch message<commit_after>package boomer

import (
	"fmt"
	"log"
	"os"
	"runtime/debug"
	"sync/atomic"
	"time"
)

const (
	stateInit     = "ready"
	stateHatching = "hatching"
	stateRunning  = "running"
	stateStopped  = "stopped"
)

const (
	slaveReportInterval = 3 * time.Second
)

// Task is like locust's task.
// when boomer receive start message, it will spawn several
// goroutines to run Task.Fn .
type Task struct {
	Weight int
	Fn     func()
	Name   string
}

type runner struct {
	tasks       []*Task
	numClients  int
	hatchRate   int
	stopChannel chan bool
	state       string
	client      client
	nodeID      string
}

func (r *runner) safeRun(fn func()) {
	defer func() {
		// don't panic
		err := recover()
		if err != nil {
			debug.PrintStack()
			Events.Publish("request_failure", "unknown", "panic", 0.0, fmt.Sprintf("%v", err))
		}
	}()
	fn()
}

func (r *runner) spawnGoRoutines(spawnCount int, quit chan bool) {

	log.Println("Hatching and swarming", spawnCount, "clients at the rate", r.hatchRate, "clients/s...")

	weightSum := 0
	for _, task := range r.tasks {
		weightSum += task.Weight
	}

	for _, task := range r.tasks {

		percent := float64(task.Weight) / float64(weightSum)
		amount := int(round(float64(spawnCount)*percent, .5, 0))

		if weightSum == 0 {
			amount = int(float64(spawnCount) / float64(len(r.tasks)))
		}

		for i := 1; i <= amount; i++ {
			if i%r.hatchRate == 0 {
				time.Sleep(1 * time.Second)
			}
			go func(fn func()) {
				for {
					select {
					case <-quit:
						return
					default:
						if maxRPSEnabled {
							token := atomic.AddInt64(&maxRPSThreshold, -1)
							if token < 0 {
								// max RPS is reached, wait until next second
								<-maxRPSControlChannel
							} else {
								r.safeRun(fn)
							}
						} else {
							r.safeRun(fn)
						}
					}
				}
			}(task.Fn)
		}

	}

	r.hatchComplete()

}

func (r *runner) startHatching(spawnCount int, hatchRate int) {

	if r.state != stateRunning && r.state != stateHatching {
		clearStatsChannel <- true
		r.stopChannel = make(chan bool)
	}

	if r.state == stateRunning {
		// stop previous goroutines without blocking
		// those goroutines will exit when r.safeRun returns
		close(r.stopChannel)
	}

	r.stopChannel = make(chan bool)
	r.state = stateHatching

	r.hatchRate = hatchRate
	r.numClients = spawnCount
	r.spawnGoRoutines(r.numClients, r.stopChannel)

}

func (r *runner) hatchComplete() {

	data := make(map[string]interface{})
	data["count"] = r.numClients
	toServer <- newMessage("hatch_complete", data, r.nodeID)

	r.state = stateRunning
}

func (r *runner) onQuiting() {
	toServer <- newMessage("quit", nil, r.nodeID)
}

func (r *runner) stop() {

	if r.state == stateRunning {
		close(r.stopChannel)
		r.state = stateStopped
		log.Println("Recv stop message from master, all the goroutines are stopped")
	}

}

func (r *runner) getReady() {

	r.state = stateInit

	// read message from server
	go func() {
		for {
			msg := <-fromServer
			switch msg.Type {
			case "hatch":
				toServer <- newMessage("hatching", nil, r.nodeID)
				rate, _ := msg.Data["hatch_rate"]
				clients, _ := msg.Data["num_clients"]
				hatchRate := int(rate.(float64))
				workers := 0
				if _, ok := clients.(uint64); ok {
					workers = int(clients.(uint64))
				} else {
					workers = int(clients.(int64))
				}
				if workers == 0 || hatchRate == 0 {
					log.Printf("Invalid hatch message from master, num_clients is %d, hatch_rate is %d\n",
						workers, hatchRate)
				} else {
					r.startHatching(workers, hatchRate)
				}
			case "stop":
				r.stop()
				toServer <- newMessage("client_stopped", nil, r.nodeID)
				toServer <- newMessage("client_ready", nil, r.nodeID)
			case "quit":
				log.Println("Got quit message from master, shutting down...")
				os.Exit(0)
			}
		}
	}()

	// tell master, I'm ready
	toServer <- newMessage("client_ready", nil, r.nodeID)

	// report to server
	go func() {
		for {
			select {
			case data := <-messageToServerChannel:
				data["user_count"] = r.numClients
				toServer <- newMessage("stats", data, r.nodeID)
			}
		}
	}()

	if maxRPSEnabled {
		go func() {
			for {
				atomic.StoreInt64(&maxRPSThreshold, maxRPS)
				time.Sleep(1 * time.Second)
				// use channel to broadcast
				close(maxRPSControlChannel)
				maxRPSControlChannel = make(chan bool)
			}
		}()
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (C) 2014  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package client

type User struct {
	Self         string
	Key          string
	Name         string
	EmailAddress string `json:"emailAddress"`
	AvatarURLs   struct {
		Size16 string
		Size24 string
		Size32 string
		Size48 string
	} `json:"avatarUrls"`
	DisplayName string `json:"displayName"`
	Active      bool
	TimeZone    string `json:"timeZone"`
	Groups      struct {
		Size  int
		Items []struct {
			Name string
			Self string
		}
	}
}
<commit_msg>git-trunk/modules/jira/client: JSON tags for User<commit_after>/*
   Copyright (C) 2014  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package client

type User struct {
	Self         string `json:"self,omitempty"`
	Key          string `json:"key,omitempty"`
	Name         string `json:"name,omitempty"`
	EmailAddress string `json:"emailAddress,omitempty"`
	AvatarURLs   struct {
		Size16 string `json:"16x16,omitempty"`
		Size24 string `json:"24x24,omitempty"`
		Size32 string `json:"32x32,omitempty"`
		Size48 string `json:"48x48,omitempty"`
	} `json:"avatarUrls,omitempty"`
	DisplayName string `json:"displayName,omitempty"`
	Active      bool   `json:"active,omitempty"`
	TimeZone    string `json:"timeZone,omitempty"`
	Groups      struct {
		Size  int `json:"size,omitempty"`
		Items []struct {
			Name string `json:"name,omitempty"`
			Self string `json:"self,omitempty"`
		} `json:"items,omitempty"`
	} `json:"groups,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"context"
	//"errors"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type CmdTeamGenerateSeitan struct {
	libkb.Contextified
	Team string
	Role keybase1.TeamRole
}

func newCmdTeamGenerateSeitan(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "generate-invite-token",
		ArgumentHelp: "<team name>",
		Usage:        "Generate an invite token that you can send via SMS, iMessage, or other similar mechanism.",
		Action: func(c *cli.Context) {
			cmd := NewCmdTeamGenerateSeitanRunner(g)
			cl.ChooseCommand(cmd, "generate-invite-token", c)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "r, role",
				Usage: "team role (owner, admin, writer, reader) [required]",
			},
		},
		Description: teamGenerateSeitanDoc,
	}
}

func NewCmdTeamGenerateSeitanRunner(g *libkb.GlobalContext) *CmdTeamGenerateSeitan {
	return &CmdTeamGenerateSeitan{Contextified: libkb.NewContextified(g)}
}

func (c *CmdTeamGenerateSeitan) ParseArgv(ctx *cli.Context) error {
	var err error
	c.Team, err = ParseOneTeamName(ctx)
	if err != nil {
		return err
	}

	c.Role, err = ParseRole(ctx)
	if err != nil {
		return err
	}

	return nil
}

func (c *CmdTeamGenerateSeitan) Run() error {
	cli, err := GetTeamsClient(c.G())
	if err != nil {
		return err
	}

	arg := keybase1.TeamCreateSeitanTokenArg{
		Name: c.Team,
		Role: c.Role,
	}

	res, err := cli.TeamCreateSeitanToken(context.Background(), arg)
	if err != nil {
		return err
	}

	dui := c.G().UI.GetDumbOutputUI()
	dui.Printf("Generated token: %q. Tell your friend!\n", res)

	return nil
}

func (c *CmdTeamGenerateSeitan) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}

const teamGenerateSeitanDoc = `"keybase team generate-token" allows you to create a one-time use,
expiring, cryptographically secure token that someone can use to join
a team.`
<commit_msg>Remove commented out import<commit_after>// Copyright 2017 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
)

type CmdTeamGenerateSeitan struct {
	libkb.Contextified
	Team string
	Role keybase1.TeamRole
}

func newCmdTeamGenerateSeitan(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "generate-invite-token",
		ArgumentHelp: "<team name>",
		Usage:        "Generate an invite token that you can send via SMS, iMessage, or other similar mechanism.",
		Action: func(c *cli.Context) {
			cmd := NewCmdTeamGenerateSeitanRunner(g)
			cl.ChooseCommand(cmd, "generate-invite-token", c)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "r, role",
				Usage: "team role (owner, admin, writer, reader) [required]",
			},
		},
		Description: teamGenerateSeitanDoc,
	}
}

func NewCmdTeamGenerateSeitanRunner(g *libkb.GlobalContext) *CmdTeamGenerateSeitan {
	return &CmdTeamGenerateSeitan{Contextified: libkb.NewContextified(g)}
}

func (c *CmdTeamGenerateSeitan) ParseArgv(ctx *cli.Context) error {
	var err error
	c.Team, err = ParseOneTeamName(ctx)
	if err != nil {
		return err
	}

	c.Role, err = ParseRole(ctx)
	if err != nil {
		return err
	}

	return nil
}

func (c *CmdTeamGenerateSeitan) Run() error {
	cli, err := GetTeamsClient(c.G())
	if err != nil {
		return err
	}

	arg := keybase1.TeamCreateSeitanTokenArg{
		Name: c.Team,
		Role: c.Role,
	}

	res, err := cli.TeamCreateSeitanToken(context.Background(), arg)
	if err != nil {
		return err
	}

	dui := c.G().UI.GetDumbOutputUI()
	dui.Printf("Generated token: %q. Tell your friend!\n", res)

	return nil
}

func (c *CmdTeamGenerateSeitan) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		API:       true,
		KbKeyring: true,
	}
}

const teamGenerateSeitanDoc = `"keybase team generate-token" allows you to create a one-time use,
expiring, cryptographically secure token that someone can use to join
a team.`
<|endoftext|>


<commit_before>// Package tunnel is responsible of setting up and connecting to a tunnel
// server.
package tunnel

import (
	"errors"
	"net/url"
	"strings"
	"sync"
	"time"

	"koding/kites/tunnelproxy"
	"koding/klient/info/publicip"

	"github.com/boltdb/bolt"
	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

var (
	// ErrKeyNotFound
	ErrKeyNotFound = errors.New("key not found")

	// ErrNoDatabase
	ErrNoDatabase = errors.New("local database is not available")
)

// Tunnel
type Tunnel struct {
	db     Storage
	opts   *Options
	client *tunnelproxy.Client

	// Used to wait for first successful
	// tunnel server registration.
	register sync.WaitGroup
	once     sync.Once
}

// Options
type Options struct {
	TunnelName    string        `json:"tunnelName,omitempty"`
	TunnelKiteURL string        `json:"tunnelKiteURL,omitempty"`
	LocalAddr     string        `json:"localAddr,omitempty"`
	VirtualHost   string        `json:"virtualHost,omitempty"`
	Timeout       time.Duration `json:"timeout,omitempty"`

	// IP reachability test cache
	LastAddr      string `json:"lastAddr,omitempty"`
	LastReachable bool   `json:"lastReachable,omitempty"`

	DB     *bolt.DB       `json:"-"`
	Log    kite.Logger    `json:"-"`
	Config *config.Config `json:"-"`
	Debug  bool           `json:"-"`
}

// updateEmpty overwrites each zero-value field of opts with defaults (merge-in).
func (opts *Options) updateEmpty(defaults *Options) {
	if opts.TunnelName == "" {
		opts.TunnelName = defaults.TunnelName
	}

	if opts.TunnelKiteURL == "" {
		opts.TunnelKiteURL = defaults.TunnelKiteURL
	}

	if opts.LocalAddr == "" {
		opts.LocalAddr = defaults.LocalAddr
	}

	if opts.VirtualHost == "" {
		opts.VirtualHost = defaults.VirtualHost
	}

	if opts.Timeout == 0 {
		opts.Timeout = defaults.Timeout
	}

	if opts.LastAddr == "" {
		opts.LastAddr = defaults.LastAddr
	}

	if !opts.LastReachable {
		opts.LastReachable = defaults.LastReachable
	}

	// set defaults
	if opts.Timeout == 0 {
		opts.Timeout = 5 * time.Minute
	}
}

func (opts *Options) copy() *Options {
	optsCopy := *opts

	if opts.Config != nil {
		optsCopy.Config = opts.Config.Copy()
	}

	return &optsCopy
}

// New
func New(opts *Options) *Tunnel {
	optsCopy := *opts

	t := &Tunnel{
		db:   newStorage(&optsCopy),
		opts: &optsCopy,
	}

	t.register.Add(1)

	return t
}

func (t *Tunnel) clientOptions() *tunnelproxy.ClientOptions {
	return &tunnelproxy.ClientOptions{
		TunnelName:      t.opts.TunnelName,
		TunnelKiteURL:   t.opts.TunnelKiteURL,
		LastVirtualHost: t.opts.VirtualHost,
		Config:          t.opts.Config,
		Timeout:         t.opts.Timeout,
		OnRegister:      t.updateOptions,
		Debug:           t.opts.Debug,
	}
}

func (t *Tunnel) updateOptions(reg *tunnelproxy.RegisterResult) {
	t.opts.VirtualHost = reg.VirtualHost
	t.opts.TunnelName = guessTunnelName(reg.VirtualHost)

	if err := t.db.UpdateOptions(t.opts); err != nil {
		t.opts.Log.Warning("tunnel: unable to update options: %s", err)
	}

	t.once.Do(t.register.Done)
}

// buildOptions finalizes build of tunnel options; the contructed options
func (t *Tunnel) buildOptions(final *Options) {
	t.opts.updateEmpty(final)

	storageOpts, err := t.db.Options()
	if err != nil {
		t.opts.Log.Warning("tunnel: unable to read options: %s", err)
	} else {
		t.opts.updateEmpty(storageOpts)
	}

	if err = t.db.UpdateOptions(t.opts); err != nil {
		t.opts.Log.Warning("tunnel: unable to update options: %s", err)
	}
}

// Start setups the client and connects to a tunnel server based on the given
// configuration. It's non blocking and should be called only once.
func (t *Tunnel) Start(opts *Options, registerURL *url.URL) (*url.URL, error) {
	t.buildOptions(opts)

	if t.opts.LastAddr != registerURL.Host {
		ok, err := publicip.IsReachableRetry(registerURL.Host, 10, 5*time.Second, t.opts.Log)
		if err != nil {
			t.opts.Log.Warning("tunnel: unable to test %q: %s", registerURL.Host, err)
			return registerURL, nil
		}

		t.opts.LastAddr = registerURL.Host
		t.opts.LastReachable = ok

		if err := t.db.UpdateOptions(t.opts); err != nil {
			t.opts.Log.Warning("tunnel: unable to update options: %s", err)
		}
	}

	if t.opts.LastReachable {
		return registerURL, nil
	}

	t.opts.Log.Debug("starting tunnel client: %# v", opts)

	client, err := tunnelproxy.NewClient(t.clientOptions())
	if err != nil {
		return nil, err
	}

	t.client = client
	t.client.Start()
	t.register.Wait()

	u := *registerURL
	u.Host = t.opts.VirtualHost
	u.Path = "/klient/kite"

	t.opts.Log.Info("tunnel: connected as %q", u.Host)

	return &u, nil
}

func guessTunnelName(vhost string) string {
	// If vhost is <tunnelName>.<user>.koding.me return the
	// <tunnelName> part (production environment).
	//
	// Example: 62ee1f899a4e.rafal.koding.me
	i := strings.LastIndex(vhost, ".koding.me")
	if i != -1 {
		i = strings.LastIndex(vhost[:i], ".")
		if i != -1 {
			return vhost[:i]
		}
	}

	// If vhost is <tunnelName>.<customBaseVirtualHost> return the
	// <tunnelName> part (development environment).
	//
	// Example: macbook.rafal.t.dev.koding.io:8081
	if strings.Count(vhost, ".") > 1 {
		return vhost[:strings.IndexRune(vhost, '.')]
	}

	return ""
}
<commit_msg>klient/tunnel: merge in all Options fields<commit_after>// Package tunnel is responsible of setting up and connecting to a tunnel
// server.
package tunnel

import (
	"errors"
	"net/url"
	"strings"
	"sync"
	"time"

	"koding/kites/tunnelproxy"
	"koding/klient/info/publicip"

	"github.com/boltdb/bolt"
	"github.com/koding/kite"
	"github.com/koding/kite/config"
)

var (
	// ErrKeyNotFound
	ErrKeyNotFound = errors.New("key not found")

	// ErrNoDatabase
	ErrNoDatabase = errors.New("local database is not available")
)

// Tunnel
type Tunnel struct {
	db     Storage
	opts   *Options
	client *tunnelproxy.Client

	// Used to wait for first successful
	// tunnel server registration.
	register sync.WaitGroup
	once     sync.Once
}

// Options
type Options struct {
	TunnelName    string        `json:"tunnelName,omitempty"`
	TunnelKiteURL string        `json:"tunnelKiteURL,omitempty"`
	LocalAddr     string        `json:"localAddr,omitempty"`
	VirtualHost   string        `json:"virtualHost,omitempty"`
	Timeout       time.Duration `json:"timeout,omitempty"`

	// IP reachability test cache
	LastAddr      string `json:"lastAddr,omitempty"`
	LastReachable bool   `json:"lastReachable,omitempty"`

	DB     *bolt.DB       `json:"-"`
	Log    kite.Logger    `json:"-"`
	Config *config.Config `json:"-"`
	Debug  bool           `json:"-"`
}

// updateEmpty overwrites each zero-value field of opts with defaults (merge-in).
func (opts *Options) updateEmpty(defaults *Options) {
	if opts.TunnelName == "" {
		opts.TunnelName = defaults.TunnelName
	}

	if opts.TunnelKiteURL == "" {
		opts.TunnelKiteURL = defaults.TunnelKiteURL
	}

	if opts.LocalAddr == "" {
		opts.LocalAddr = defaults.LocalAddr
	}

	if opts.VirtualHost == "" {
		opts.VirtualHost = defaults.VirtualHost
	}

	if opts.Timeout == 0 {
		opts.Timeout = defaults.Timeout
	}

	if opts.LastAddr == "" {
		opts.LastAddr = defaults.LastAddr
	}

	if !opts.LastReachable {
		opts.LastReachable = defaults.LastReachable
	}

	if opts.DB == nil {
		opts.DB = defaults.DB
	}

	if opts.Log == nil {
		opts.Log = defaults.Log
	}

	if opts.Config == nil {
		opts.Config = defaults.Config
	}

	if !opts.Debug {
		opts.Debug = defaults.Debug
	}

	// set defaults
	if opts.Timeout == 0 {
		opts.Timeout = 5 * time.Minute
	}
}

func (opts *Options) copy() *Options {
	optsCopy := *opts

	if opts.Config != nil {
		optsCopy.Config = opts.Config.Copy()
	}

	return &optsCopy
}

// New
func New(opts *Options) *Tunnel {
	optsCopy := *opts

	t := &Tunnel{
		db:   newStorage(&optsCopy),
		opts: &optsCopy,
	}

	t.register.Add(1)

	return t
}

func (t *Tunnel) clientOptions() *tunnelproxy.ClientOptions {
	return &tunnelproxy.ClientOptions{
		TunnelName:      t.opts.TunnelName,
		TunnelKiteURL:   t.opts.TunnelKiteURL,
		LastVirtualHost: t.opts.VirtualHost,
		LocalAddr:       t.opts.LocalAddr,
		Config:          t.opts.Config,
		Timeout:         t.opts.Timeout,
		OnRegister:      t.updateOptions,
		Debug:           t.opts.Debug,
	}
}

func (t *Tunnel) updateOptions(reg *tunnelproxy.RegisterResult) {
	t.opts.VirtualHost = reg.VirtualHost
	t.opts.TunnelName = guessTunnelName(reg.VirtualHost)

	if err := t.db.UpdateOptions(t.opts); err != nil {
		t.opts.Log.Warning("tunnel: unable to update options: %s", err)
	}

	t.once.Do(t.register.Done)
}

// buildOptions finalizes build of tunnel options; the contructed options
func (t *Tunnel) buildOptions(final *Options) {
	t.opts.updateEmpty(final)

	t.opts.Log.Debug("buildOptions: final=%+v", t.opts)

	storageOpts, err := t.db.Options()
	if err != nil {
		t.opts.Log.Warning("tunnel: unable to read options: %s", err)
	} else {
		t.opts.updateEmpty(storageOpts)

		t.opts.Log.Debug("buildOptions: storage=%+v, built=%+v", storageOpts, t.opts)
	}

	if err = t.db.UpdateOptions(t.opts); err != nil {
		t.opts.Log.Warning("tunnel: unable to update options: %s", err)
	}
}

// Start setups the client and connects to a tunnel server based on the given
// configuration. It's non blocking and should be called only once.
func (t *Tunnel) Start(opts *Options, registerURL *url.URL) (*url.URL, error) {
	t.buildOptions(opts)

	if t.opts.LastAddr != registerURL.Host {
		t.opts.Log.Info("testing whether %q address is reachable...", registerURL.Host)

		ok, err := publicip.IsReachableRetry(registerURL.Host, 10, 5*time.Second, t.opts.Log)
		if err != nil {
			t.opts.Log.Warning("tunnel: unable to test %q: %s", registerURL.Host, err)
			return registerURL, nil
		}

		t.opts.LastAddr = registerURL.Host
		t.opts.LastReachable = ok

		if err := t.db.UpdateOptions(t.opts); err != nil {
			t.opts.Log.Warning("tunnel: unable to update options: %s", err)
		}
	}

	if t.opts.LastReachable {
		return registerURL, nil
	}

	t.opts.Log.Debug("starting tunnel client: %# v", opts)

	client, err := tunnelproxy.NewClient(t.clientOptions())
	if err != nil {
		return nil, err
	}

	t.client = client
	t.client.Start()
	t.register.Wait()

	u := *registerURL
	u.Host = t.opts.VirtualHost
	u.Path = "/klient/kite"

	t.opts.Log.Info("tunnel: connected as %q", u.Host)

	return &u, nil
}

func guessTunnelName(vhost string) string {
	// If vhost is <tunnelName>.<user>.koding.me return the
	// <tunnelName> part (production environment).
	//
	// Example: 62ee1f899a4e.rafal.koding.me
	i := strings.LastIndex(vhost, ".koding.me")
	if i != -1 {
		i = strings.LastIndex(vhost[:i], ".")
		if i != -1 {
			return vhost[:i]
		}
	}

	// If vhost is <tunnelName>.<customBaseVirtualHost> return the
	// <tunnelName> part (development environment).
	//
	// Example: macbook.rafal.t.dev.koding.io:8081
	if strings.Count(vhost, ".") > 1 {
		return vhost[:strings.IndexRune(vhost, '.')]
	}

	return ""
}
<|endoftext|>


<commit_before>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package grpcvtctlserver contains the gRPC implementation of the server side
of the remote execution of vtctl commands.
*/
package grpcvtctlserver

import (
	"sync"

	"google.golang.org/grpc"

	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtctl"
	"github.com/youtube/vitess/go/vt/wrangler"

	vtctldatapb "github.com/youtube/vitess/go/vt/proto/vtctldata"
	vtctlservicepb "github.com/youtube/vitess/go/vt/proto/vtctlservice"
)

// VtctlServer is our RPC server
type VtctlServer struct {
	ts topo.Server
}

// NewVtctlServer returns a new Vtctl Server for the topo server.
func NewVtctlServer(ts topo.Server) *VtctlServer {
	return &VtctlServer{ts}
}

// ExecuteVtctlCommand is part of the vtctldatapb.VtctlServer interface
func (s *VtctlServer) ExecuteVtctlCommand(args *vtctldatapb.ExecuteVtctlCommandRequest, stream vtctlservicepb.Vtctl_ExecuteVtctlCommandServer) (err error) {
	defer servenv.HandlePanic("vtctl", &err)

	// create a logger, send the result back to the caller
	logstream := logutil.NewChannelLogger(10)
	logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger())

	// send logs to the caller
	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		for e := range logstream {
			// Note we don't interrupt the loop here, as
			// we still need to flush and finish the
			// command, even if the channel to the client
			// has been broken. We'll just keep trying.
			stream.Send(&vtctldatapb.ExecuteVtctlCommandResponse{
				Event: e,
			})
		}
		wg.Done()
	}()

	// create the wrangler
	wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient())

	// execute the command
	err = vtctl.RunCommand(stream.Context(), wr, args.Args)

	// close the log channel, and wait for them all to be sent
	close(logstream)
	wg.Wait()

	return err
}

// StartServer registers the VtctlServer for RPCs
func StartServer(s *grpc.Server, ts topo.Server) {
	vtctlservicepb.RegisterVtctlServer(s, NewVtctlServer(ts))
}
<commit_msg>Fix goroutine leak if a vtctl command panics.<commit_after>// Copyright 2014, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package grpcvtctlserver contains the gRPC implementation of the server side
of the remote execution of vtctl commands.
*/
package grpcvtctlserver

import (
	"sync"

	"google.golang.org/grpc"

	"github.com/youtube/vitess/go/vt/logutil"
	"github.com/youtube/vitess/go/vt/servenv"
	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
	"github.com/youtube/vitess/go/vt/topo"
	"github.com/youtube/vitess/go/vt/vtctl"
	"github.com/youtube/vitess/go/vt/wrangler"

	vtctldatapb "github.com/youtube/vitess/go/vt/proto/vtctldata"
	vtctlservicepb "github.com/youtube/vitess/go/vt/proto/vtctlservice"
)

// VtctlServer is our RPC server
type VtctlServer struct {
	ts topo.Server
}

// NewVtctlServer returns a new Vtctl Server for the topo server.
func NewVtctlServer(ts topo.Server) *VtctlServer {
	return &VtctlServer{ts}
}

// ExecuteVtctlCommand is part of the vtctldatapb.VtctlServer interface
func (s *VtctlServer) ExecuteVtctlCommand(args *vtctldatapb.ExecuteVtctlCommandRequest, stream vtctlservicepb.Vtctl_ExecuteVtctlCommandServer) (err error) {
	defer servenv.HandlePanic("vtctl", &err)

	// create a logger, send the result back to the caller
	logstream := logutil.NewChannelLogger(10)
	logger := logutil.NewTeeLogger(logstream, logutil.NewConsoleLogger())

	// send logs to the caller
	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		for e := range logstream {
			// Note we don't interrupt the loop here, as
			// we still need to flush and finish the
			// command, even if the channel to the client
			// has been broken. We'll just keep trying.
			stream.Send(&vtctldatapb.ExecuteVtctlCommandResponse{
				Event: e,
			})
		}
		wg.Done()
	}()

	// Defer cleanup in case we panic.
	defer func() {
		// close the log channel, and wait for them all to be sent
		close(logstream)
		wg.Wait()
	}()

	// create the wrangler
	wr := wrangler.New(logger, s.ts, tmclient.NewTabletManagerClient())

	// execute the command
	err = vtctl.RunCommand(stream.Context(), wr, args.Args)

	return err
}

// StartServer registers the VtctlServer for RPCs
func StartServer(s *grpc.Server, ts topo.Server) {
	vtctlservicepb.RegisterVtctlServer(s, NewVtctlServer(ts))
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/goauth2/oauth"
	"flag"
	"github.com/Clever/clever-to-csv"
	csvSink "github.com/azylman/optimus/sinks/csv"
	"github.com/azylman/optimus/transformer"
	clevergo "gopkg.in/Clever/clever-go.v1"
	"log"
	"os"
	"strings"
	"sync"
)

func main() {
	host := flag.String("host", "https://api.clever.com", "base URL of Clever API")
	endpoints := flag.String("endpoints", "schools,sections,students,teachers", "comma-delimited list of endpoints to pull")
	token := flag.String("token", "", "API token to use for authentication")
	flag.Parse()

	for _, required := range []*string{host, endpoints, token} {
		if len(*required) == 0 {
			flag.Usage()
			os.Exit(1)
		}
	}
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: *token},
	}
	client := t.Client()
	clever := clevergo.New(client, *host)

	wg := sync.WaitGroup{}
	errors := make(chan error)
	go func() {
		wg.Wait()
		close(errors)
	}()
	for _, endpoint := range strings.Split(*endpoints, ",") {
		wg.Add(1)
		go func(endpoint string) {
			t := transformer.New(clevertable.New(endpoint, nil, clever)).
				Map(clevertable.FlattenRow).
				Map(clevertable.StringifyArrayVals).
				Table()
			if err := csvSink.New(t, endpoint+".csv"); err != nil {
				errors <- err
			}
			wg.Done()
		}(endpoint)
	}
	for err := range errors {
		log.Fatal(err)
	}
}
<commit_msg>cmd: support where queries on endpoints<commit_after>package main

import (
	"code.google.com/p/goauth2/oauth"
	"flag"
	"fmt"
	"github.com/Clever/clever-to-csv"
	csvSink "github.com/azylman/optimus/sinks/csv"
	"github.com/azylman/optimus/transformer"
	clevergo "gopkg.in/Clever/clever-go.v1"
	"log"
	"net/url"
	"os"
	"sync"
)

type endpointConf struct {
	Params url.Values
	Name   string
}

func (e *endpointConf) Set(val string) error {
	if val == "true" {
		return nil
	}
	e.Params = url.Values{"where": []string{val}}
	return nil
}

func (e *endpointConf) String() string {
	return ""
}

func (e *endpointConf) IsBoolFlag() bool {
	return true
}

var acceptedEndpoints = []string{"schools", "sections", "students", "teachers"}

func main() {
	host := flag.String("host", "https://api.clever.com", "base URL of Clever API")
	for _, endpoint := range acceptedEndpoints {
		end := &endpointConf{Name: endpoint}
		flag.Var(end, endpoint, fmt.Sprintf("if included, dumps the %s endpoint. can optionally include JSON-stringified map of 'where' query parameters", endpoint))
	}
	token := flag.String("token", "", "API token to use for authentication")
	flag.Parse()

	endpoints := []flag.Value{}
	flag.Visit(func(f *flag.Flag) {
		if f.Name != "host" && f.Name != "token" {
			endpoints = append(endpoints, f.Value)
		}
	})

	for _, required := range []*string{host, token} {
		if len(*required) == 0 {
			flag.Usage()
			os.Exit(1)
		}
	}

	transport := &oauth.Transport{
		Token: &oauth.Token{AccessToken: *token},
	}
	client := transport.Client()
	clever := clevergo.New(client, *host)

	wg := sync.WaitGroup{}
	errors := make(chan error)
	go func() {
		wg.Wait()
		close(errors)
	}()
	for _, end := range endpoints {
		wg.Add(1)
		go func(endp flag.Value) {
			defer wg.Done()
			endpoint, ok := endp.(*endpointConf)
			if !ok {
				errors <- fmt.Errorf("invalid flag value %#v", end)
				return
			}

			t := transformer.New(clevertable.New(endpoint.Name, endpoint.Params, clever)).
				Map(clevertable.FlattenRow).
				Map(clevertable.StringifyArrayVals).
				Table()
			if err := csvSink.New(t, endpoint.Name+".csv"); err != nil {
				errors <- err
			}
		}(end)
	}
	for err := range errors {
		log.Fatalf("got err %#v\n", err)
	}
}
<|endoftext|>