<commit_before>// Package jsonrpc provides an jsonrpc 2.0 client that sends jsonrpc requests and receives jsonrpc responses using http.
package jsonrpc

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"
)

// RPCRequest represents a jsonrpc request object.
//
// See: http://www.jsonrpc.org/specification#request_object
type RPCRequest struct {
	JSONRPC string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params,omitempty"`
	ID      uint        `json:"id"`
}

// RPCNotification represents a jsonrpc notification object.
// A notification object omits the id field since there will be no server response.
//
// See: http://www.jsonrpc.org/specification#notification
type RPCNotification struct {
	JSONRPC string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params,omitempty"`
}

// RPCResponse represents a jsonrpc response object.
// If no rpc specific error occurred Error field is nil.
//
// See: http://www.jsonrpc.org/specification#response_object
type RPCResponse struct {
	JSONRPC string      `json:"jsonrpc"`
	Result  interface{} `json:"result,omitempty"`
	Error   *RPCError   `json:"error,omitempty"`
	ID      uint        `json:"id"`
}

// BatchResponse a list of jsonrpc response objects as a result of a batch request
//
// if you are interested in the response of a specific request use: GetResponseOf(request)
type BatchResponse struct {
	rpcResponses []RPCResponse
}

// RPCError represents a jsonrpc error object if an rpc error occurred.
//
// See: http://www.jsonrpc.org/specification#error_object
type RPCError struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Data    interface{} `json:"data"`
}

// RPCClient sends jsonrpc requests over http to the provided rpc backend.
// RPCClient is created using the factory function NewRPCClient().
type RPCClient struct {
	endpoint        string
	httpClient      *http.Client
	customHeaders   map[string]string
	autoIncrementID bool
	nextID          uint
	idMutex         sync.Mutex
}

// NewRPCClient returns a new RPCClient instance with default configuration (no custom headers, default http.Client, autoincrement ids).
// Endpoint is the rpc-service url to which the rpc requests are sent.
func NewRPCClient(endpoint string) *RPCClient {
	return &RPCClient{
		endpoint:        endpoint,
		httpClient:      http.DefaultClient,
		autoIncrementID: true,
		nextID:          0,
		customHeaders:   make(map[string]string),
	}
}

// NewRPCRequestObject creates and returns a raw RPCRequest structure.
// It is mainly used when building batch requests. For single requests use RPCClient.Call().
// RPCRequest struct can also be created directly, but this function sets the ID and the jsonrpc field to the correct values.
func (client *RPCClient) NewRPCRequestObject(method string, params ...interface{}) *RPCRequest {
	client.idMutex.Lock()
	rpcRequest := RPCRequest{
		ID:      client.nextID,
		JSONRPC: "2.0",
		Method:  method,
		Params:  params,
	}
	if client.autoIncrementID == true {
		client.nextID++
	}
	client.idMutex.Unlock()

	if len(params) == 0 {
		rpcRequest.Params = nil
	}

	return &rpcRequest
}

// NewRPCNotificationObject creates and returns a raw RPCNotification structure.
// It is mainly used when building batch requests. For single notifications use RPCClient.Notification().
// NewRPCNotificationObject struct can also be created directly, but this function sets the ID and the jsonrpc field to the correct values.
func (client *RPCClient) NewRPCNotificationObject(method string, params ...interface{}) *RPCNotification {
	rpcNotification := RPCNotification{
		JSONRPC: "2.0",
		Method:  method,
		Params:  params,
	}

	if len(params) == 0 {
		rpcNotification.Params = nil
	}

	return &rpcNotification
}

// Call sends an jsonrpc request over http to the rpc-service url that was provided on client creation.
//
// If something went wrong on the network / http level or if json parsing failed it returns an error.
//
// If something went wrong on the rpc-service / protocol level the Error field of the returned RPCResponse is set
// and contains information about the error.
//
// If the request was successful the Error field is nil and the Result field of the RPCRespnse struct contains the rpc result.
func (client *RPCClient) Call(method string, params ...interface{}) (*RPCResponse, error) {
	httpRequest, err := client.newRequest(false, method, params...)
	if err != nil {
		return nil, err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer httpResponse.Body.Close()

	rpcResponse := RPCResponse{}
	decoder := json.NewDecoder(httpResponse.Body)
	decoder.UseNumber()
	err = decoder.Decode(&rpcResponse)
	if err != nil {
		return nil, err
	}

	return &rpcResponse, nil
}

// Notification sends a jsonrpc request to the rpc-service. The difference to Call() is that this request does not expect a response.
// The ID field of the request is omitted.
func (client *RPCClient) Notification(method string, params ...interface{}) error {
	httpRequest, err := client.newRequest(true, method, params...)
	if err != nil {
		return err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return err
	}
	defer httpResponse.Body.Close()
	return nil
}

// Batch sends a jsonrpc batch request to the rpc-service.
// The parameter is a list of requests the could be one of:
//	RPCRequest
//	RPCNotification.
//
// The batch requests returns a list of RPCResponse structs.
func (client *RPCClient) Batch(requests ...interface{}) (*BatchResponse, error) {
	for _, r := range requests {
		switch r := r.(type) {
		default:
			return nil, fmt.Errorf("Invalid parameter: %s", r)
		case *RPCRequest:
		case *RPCNotification:
		}
	}

	httpRequest, err := client.newBatchRequest(requests...)
	if err != nil {
		return nil, err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer httpResponse.Body.Close()

	rpcResponses := []RPCResponse{}
	decoder := json.NewDecoder(httpResponse.Body)
	decoder.UseNumber()
	err = decoder.Decode(&rpcResponses)
	if err != nil {
		return nil, err
	}

	return &BatchResponse{rpcResponses: rpcResponses}, nil
}

// SetAutoIncrementID if set to true, the id field of an rpcjson request will be incremented automatically
func (client *RPCClient) SetAutoIncrementID(flag bool) {
	client.autoIncrementID = flag
}

// SetNextID can be used to manually set the next id / reset the id.
func (client *RPCClient) SetNextID(id uint) {
	client.idMutex.Lock()
	client.nextID = id
	client.idMutex.Unlock()
}

// SetCustomHeader is used to set a custom header for each rpc request.
// You could for example set the Authorization Bearer here.
func (client *RPCClient) SetCustomHeader(key string, value string) {
	client.customHeaders[key] = value
}

// UnsetCustomHeader is used to removes a custom header that was added before.
func (client *RPCClient) UnsetCustomHeader(key string) {
	delete(client.customHeaders, key)
}

// SetBasicAuth is a helper function that sets the header for the given basic authentication credentials.
// To reset / disable authentication just set username or password to an empty string value.
func (client *RPCClient) SetBasicAuth(username string, password string) {
	if username == "" || password == "" {
		delete(client.customHeaders, "Authorization")
		return
	}
	auth := username + ":" + password
	client.customHeaders["Authorization"] = "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
}

// SetHTTPClient can be used to set a custom http.Client.
// This can be useful for example if you want to customize the http.Client behaviour (e.g. proxy settings)
func (client *RPCClient) SetHTTPClient(httpClient *http.Client) {
	if httpClient == nil {
		panic("httpClient cannot be nil")
	}
	client.httpClient = httpClient
}

func (client *RPCClient) newRequest(notification bool, method string, params ...interface{}) (*http.Request, error) {

	// TODO: easier way to remove ID from RPCRequest without extra struct
	var rpcRequest interface{}
	if notification {
		rpcNotification := RPCNotification{
			JSONRPC: "2.0",
			Method:  method,
			Params:  params,
		}
		if len(params) == 0 {
			rpcNotification.Params = nil
		}
		rpcRequest = rpcNotification
	} else {
		client.idMutex.Lock()
		request := RPCRequest{
			ID:      client.nextID,
			JSONRPC: "2.0",
			Method:  method,
			Params:  params,
		}
		if client.autoIncrementID == true {
			client.nextID++
		}
		client.idMutex.Unlock()
		if len(params) == 0 {
			request.Params = nil
		}
		rpcRequest = request
	}

	body, err := json.Marshal(rpcRequest)
	if err != nil {
		return nil, err
	}

	request, err := http.NewRequest("POST", client.endpoint, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	for k, v := range client.customHeaders {
		request.Header.Add(k, v)
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept", "application/json")

	return request, nil
}

func (client *RPCClient) newBatchRequest(requests ...interface{}) (*http.Request, error) {

	body, err := json.Marshal(requests)
	if err != nil {
		return nil, err
	}

	request, err := http.NewRequest("POST", client.endpoint, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	for k, v := range client.customHeaders {
		request.Header.Add(k, v)
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept", "application/json")

	return request, nil
}

// UpdateRequestID updates the ID of an RPCRequest structure.
//
// This is used if a request is sent another time and the request should get an updated id.
//
// This does only make sense when used on with Batch() since Call() and Notififcation() do update the id automatically.
func (client *RPCClient) UpdateRequestID(rpcRequest *RPCRequest) {
	if rpcRequest == nil {
		return
	}
	client.idMutex.Lock()
	defer client.idMutex.Unlock()
	rpcRequest.ID = client.nextID
	if client.autoIncrementID == true {
		client.nextID++
	}
}

// GetInt converts the rpc response to an int and returns it.
//
// This is a convenient function. Int could be 32 or 64 bit, depending on the architecture the code is running on.
// For a deterministic result use GetInt64().
//
// If result was not an integer an error is returned.
func (rpcResponse *RPCResponse) GetInt() (int, error) {
	i, err := rpcResponse.GetInt64()
	return int(i), err
}

// GetInt64 converts the rpc response to an int64 and returns it.
//
// If result was not an integer an error is returned.
func (rpcResponse *RPCResponse) GetInt64() (int64, error) {
	val, ok := rpcResponse.Result.(json.Number)
	if !ok {
		return 0, fmt.Errorf("could not parse int64 from %s", rpcResponse.Result)
	}

	i, err := val.Int64()
	if err != nil {
		return 0, err
	}

	return i, nil
}

// GetFloat64 converts the rpc response to an float64 and returns it.
//
// If result was not an float64 an error is returned.
func (rpcResponse *RPCResponse) GetFloat64() (float64, error) {
	val, ok := rpcResponse.Result.(json.Number)
	if !ok {
		return 0, fmt.Errorf("could not parse float64 from %s", rpcResponse.Result)
	}

	f, err := val.Float64()
	if err != nil {
		return 0, err
	}

	return f, nil
}

// GetBool converts the rpc response to a bool and returns it.
//
// If result was not a bool an error is returned.
func (rpcResponse *RPCResponse) GetBool() (bool, error) {
	val, ok := rpcResponse.Result.(bool)
	if !ok {
		return false, fmt.Errorf("could not parse int from %s", rpcResponse.Result)
	}

	return val, nil
}

// GetString converts the rpc response to a string and returns it.
//
// If result was not a string an error is returned.
func (rpcResponse *RPCResponse) GetString() (string, error) {
	val, ok := rpcResponse.Result.(string)
	if !ok {
		return "", fmt.Errorf("could not parse int from %s", rpcResponse.Result)
	}

	return val, nil
}

// GetObject converts the rpc response to an object (e.g. a struct) and returns it.
// The parameter should be a structure that can hold the data of the response object.
//
// For example if the following json return value is expected: {"name": "alex", age: 33, "country": "Germany"}
// the struct should look like
//  type Person struct {
//    Name string
//    Age int
//    Country string
//  }
func (rpcResponse *RPCResponse) GetObject(toType interface{}) error {
	js, err := json.Marshal(rpcResponse.Result)
	if err != nil {
		return err
	}

	err = json.Unmarshal(js, toType)
	if err != nil {
		return err
	}

	return nil
}

// GetResponseOf returns the rpc reponse of the corresponding request by matching the id.
//
// For this method to work, autoincrementID should be set to true (default).
func (batchResponse *BatchResponse) GetResponseOf(request *RPCRequest) (*RPCResponse, error) {
	if request == nil {
		return nil, errors.New("parameter cannot be nil")
	}
	for _, elem := range batchResponse.rpcResponses {
		if elem.ID == request.ID {
			return &elem, nil
		}
	}

	return nil, fmt.Errorf("element with id %d not found", request.ID)
}
<commit_msg>spelling<commit_after>// Package jsonrpc provides an jsonrpc 2.0 client that sends jsonrpc requests and receives jsonrpc responses using http.
package jsonrpc

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"sync"
)

// RPCRequest represents a jsonrpc request object.
//
// See: http://www.jsonrpc.org/specification#request_object
type RPCRequest struct {
	JSONRPC string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params,omitempty"`
	ID      uint        `json:"id"`
}

// RPCNotification represents a jsonrpc notification object.
// A notification object omits the id field since there will be no server response.
//
// See: http://www.jsonrpc.org/specification#notification
type RPCNotification struct {
	JSONRPC string      `json:"jsonrpc"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params,omitempty"`
}

// RPCResponse represents a jsonrpc response object.
// If no rpc specific error occurred Error field is nil.
//
// See: http://www.jsonrpc.org/specification#response_object
type RPCResponse struct {
	JSONRPC string      `json:"jsonrpc"`
	Result  interface{} `json:"result,omitempty"`
	Error   *RPCError   `json:"error,omitempty"`
	ID      uint        `json:"id"`
}

// BatchResponse a list of jsonrpc response objects as a result of a batch request
//
// if you are interested in the response of a specific request use: GetResponseOf(request)
type BatchResponse struct {
	rpcResponses []RPCResponse
}

// RPCError represents a jsonrpc error object if an rpc error occurred.
//
// See: http://www.jsonrpc.org/specification#error_object
type RPCError struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Data    interface{} `json:"data"`
}

// RPCClient sends jsonrpc requests over http to the provided rpc backend.
// RPCClient is created using the factory function NewRPCClient().
type RPCClient struct {
	endpoint        string
	httpClient      *http.Client
	customHeaders   map[string]string
	autoIncrementID bool
	nextID          uint
	idMutex         sync.Mutex
}

// NewRPCClient returns a new RPCClient instance with default configuration (no custom headers, default http.Client, autoincrement ids).
// Endpoint is the rpc-service url to which the rpc requests are sent.
func NewRPCClient(endpoint string) *RPCClient {
	return &RPCClient{
		endpoint:        endpoint,
		httpClient:      http.DefaultClient,
		autoIncrementID: true,
		nextID:          0,
		customHeaders:   make(map[string]string),
	}
}

// NewRPCRequestObject creates and returns a raw RPCRequest structure.
// It is mainly used when building batch requests. For single requests use RPCClient.Call().
// RPCRequest struct can also be created directly, but this function sets the ID and the jsonrpc field to the correct values.
func (client *RPCClient) NewRPCRequestObject(method string, params ...interface{}) *RPCRequest {
	client.idMutex.Lock()
	rpcRequest := RPCRequest{
		ID:      client.nextID,
		JSONRPC: "2.0",
		Method:  method,
		Params:  params,
	}
	if client.autoIncrementID == true {
		client.nextID++
	}
	client.idMutex.Unlock()

	if len(params) == 0 {
		rpcRequest.Params = nil
	}

	return &rpcRequest
}

// NewRPCNotificationObject creates and returns a raw RPCNotification structure.
// It is mainly used when building batch requests. For single notifications use RPCClient.Notification().
// NewRPCNotificationObject struct can also be created directly, but this function sets the ID and the jsonrpc field to the correct values.
func (client *RPCClient) NewRPCNotificationObject(method string, params ...interface{}) *RPCNotification {
	rpcNotification := RPCNotification{
		JSONRPC: "2.0",
		Method:  method,
		Params:  params,
	}

	if len(params) == 0 {
		rpcNotification.Params = nil
	}

	return &rpcNotification
}

// Call sends an jsonrpc request over http to the rpc-service url that was provided on client creation.
//
// If something went wrong on the network / http level or if json parsing failed it returns an error.
//
// If something went wrong on the rpc-service / protocol level the Error field of the returned RPCResponse is set
// and contains information about the error.
//
// If the request was successful the Error field is nil and the Result field of the RPCRespnse struct contains the rpc result.
func (client *RPCClient) Call(method string, params ...interface{}) (*RPCResponse, error) {
	httpRequest, err := client.newRequest(false, method, params...)
	if err != nil {
		return nil, err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer httpResponse.Body.Close()

	rpcResponse := RPCResponse{}
	decoder := json.NewDecoder(httpResponse.Body)
	decoder.UseNumber()
	err = decoder.Decode(&rpcResponse)
	if err != nil {
		return nil, err
	}

	return &rpcResponse, nil
}

// Notification sends a jsonrpc request to the rpc-service. The difference to Call() is that this request does not expect a response.
// The ID field of the request is omitted.
func (client *RPCClient) Notification(method string, params ...interface{}) error {
	httpRequest, err := client.newRequest(true, method, params...)
	if err != nil {
		return err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return err
	}
	defer httpResponse.Body.Close()
	return nil
}

// Batch sends a jsonrpc batch request to the rpc-service.
// The parameter is a list of requests the could be one of:
//	RPCRequest
//	RPCNotification.
//
// The batch requests returns a list of RPCResponse structs.
func (client *RPCClient) Batch(requests ...interface{}) (*BatchResponse, error) {
	for _, r := range requests {
		switch r := r.(type) {
		default:
			return nil, fmt.Errorf("Invalid parameter: %s", r)
		case *RPCRequest:
		case *RPCNotification:
		}
	}

	httpRequest, err := client.newBatchRequest(requests...)
	if err != nil {
		return nil, err
	}

	httpResponse, err := client.httpClient.Do(httpRequest)
	if err != nil {
		return nil, err
	}
	defer httpResponse.Body.Close()

	rpcResponses := []RPCResponse{}
	decoder := json.NewDecoder(httpResponse.Body)
	decoder.UseNumber()
	err = decoder.Decode(&rpcResponses)
	if err != nil {
		return nil, err
	}

	return &BatchResponse{rpcResponses: rpcResponses}, nil
}

// SetAutoIncrementID if set to true, the id field of an rpcjson request will be incremented automatically
func (client *RPCClient) SetAutoIncrementID(flag bool) {
	client.autoIncrementID = flag
}

// SetNextID can be used to manually set the next id / reset the id.
func (client *RPCClient) SetNextID(id uint) {
	client.idMutex.Lock()
	client.nextID = id
	client.idMutex.Unlock()
}

// SetCustomHeader is used to set a custom header for each rpc request.
// You could for example set the Authorization Bearer here.
func (client *RPCClient) SetCustomHeader(key string, value string) {
	client.customHeaders[key] = value
}

// UnsetCustomHeader is used to removes a custom header that was added before.
func (client *RPCClient) UnsetCustomHeader(key string) {
	delete(client.customHeaders, key)
}

// SetBasicAuth is a helper function that sets the header for the given basic authentication credentials.
// To reset / disable authentication just set username or password to an empty string value.
func (client *RPCClient) SetBasicAuth(username string, password string) {
	if username == "" || password == "" {
		delete(client.customHeaders, "Authorization")
		return
	}
	auth := username + ":" + password
	client.customHeaders["Authorization"] = "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
}

// SetHTTPClient can be used to set a custom http.Client.
// This can be useful for example if you want to customize the http.Client behaviour (e.g. proxy settings)
func (client *RPCClient) SetHTTPClient(httpClient *http.Client) {
	if httpClient == nil {
		panic("httpClient cannot be nil")
	}
	client.httpClient = httpClient
}

func (client *RPCClient) newRequest(notification bool, method string, params ...interface{}) (*http.Request, error) {

	// TODO: easier way to remove ID from RPCRequest without extra struct
	var rpcRequest interface{}
	if notification {
		rpcNotification := RPCNotification{
			JSONRPC: "2.0",
			Method:  method,
			Params:  params,
		}
		if len(params) == 0 {
			rpcNotification.Params = nil
		}
		rpcRequest = rpcNotification
	} else {
		client.idMutex.Lock()
		request := RPCRequest{
			ID:      client.nextID,
			JSONRPC: "2.0",
			Method:  method,
			Params:  params,
		}
		if client.autoIncrementID == true {
			client.nextID++
		}
		client.idMutex.Unlock()
		if len(params) == 0 {
			request.Params = nil
		}
		rpcRequest = request
	}

	body, err := json.Marshal(rpcRequest)
	if err != nil {
		return nil, err
	}

	request, err := http.NewRequest("POST", client.endpoint, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	for k, v := range client.customHeaders {
		request.Header.Add(k, v)
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept", "application/json")

	return request, nil
}

func (client *RPCClient) newBatchRequest(requests ...interface{}) (*http.Request, error) {

	body, err := json.Marshal(requests)
	if err != nil {
		return nil, err
	}

	request, err := http.NewRequest("POST", client.endpoint, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}

	for k, v := range client.customHeaders {
		request.Header.Add(k, v)
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept", "application/json")

	return request, nil
}

// UpdateRequestID updates the ID of an RPCRequest structure.
//
// This is used if a request is sent another time and the request should get an updated id.
//
// This does only make sense when used on with Batch() since Call() and Notififcation() do update the id automatically.
func (client *RPCClient) UpdateRequestID(rpcRequest *RPCRequest) {
	if rpcRequest == nil {
		return
	}
	client.idMutex.Lock()
	defer client.idMutex.Unlock()
	rpcRequest.ID = client.nextID
	if client.autoIncrementID == true {
		client.nextID++
	}
}

// GetInt converts the rpc response to an int and returns it.
//
// This is a convenient function. Int could be 32 or 64 bit, depending on the architecture the code is running on.
// For a deterministic result use GetInt64().
//
// If result was not an integer an error is returned.
func (rpcResponse *RPCResponse) GetInt() (int, error) {
	i, err := rpcResponse.GetInt64()
	return int(i), err
}

// GetInt64 converts the rpc response to an int64 and returns it.
//
// If result was not an integer an error is returned.
func (rpcResponse *RPCResponse) GetInt64() (int64, error) {
	val, ok := rpcResponse.Result.(json.Number)
	if !ok {
		return 0, fmt.Errorf("could not parse int64 from %s", rpcResponse.Result)
	}

	i, err := val.Int64()
	if err != nil {
		return 0, err
	}

	return i, nil
}

// GetFloat64 converts the rpc response to an float64 and returns it.
//
// If result was not an float64 an error is returned.
func (rpcResponse *RPCResponse) GetFloat64() (float64, error) {
	val, ok := rpcResponse.Result.(json.Number)
	if !ok {
		return 0, fmt.Errorf("could not parse float64 from %s", rpcResponse.Result)
	}

	f, err := val.Float64()
	if err != nil {
		return 0, err
	}

	return f, nil
}

// GetBool converts the rpc response to a bool and returns it.
//
// If result was not a bool an error is returned.
func (rpcResponse *RPCResponse) GetBool() (bool, error) {
	val, ok := rpcResponse.Result.(bool)
	if !ok {
		return false, fmt.Errorf("could not parse int from %s", rpcResponse.Result)
	}

	return val, nil
}

// GetString converts the rpc response to a string and returns it.
//
// If result was not a string an error is returned.
func (rpcResponse *RPCResponse) GetString() (string, error) {
	val, ok := rpcResponse.Result.(string)
	if !ok {
		return "", fmt.Errorf("could not parse int from %s", rpcResponse.Result)
	}

	return val, nil
}

// GetObject converts the rpc response to an object (e.g. a struct) and returns it.
// The parameter should be a structure that can hold the data of the response object.
//
// For example if the following json return value is expected: {"name": "alex", age: 33, "country": "Germany"}
// the struct should look like
//  type Person struct {
//    Name string
//    Age int
//    Country string
//  }
func (rpcResponse *RPCResponse) GetObject(toType interface{}) error {
	js, err := json.Marshal(rpcResponse.Result)
	if err != nil {
		return err
	}

	err = json.Unmarshal(js, toType)
	if err != nil {
		return err
	}

	return nil
}

// GetResponseOf returns the rpc response of the corresponding request by matching the id.
//
// For this method to work, autoincrementID should be set to true (default).
func (batchResponse *BatchResponse) GetResponseOf(request *RPCRequest) (*RPCResponse, error) {
	if request == nil {
		return nil, errors.New("parameter cannot be nil")
	}
	for _, elem := range batchResponse.rpcResponses {
		if elem.ID == request.ID {
			return &elem, nil
		}
	}

	return nil, fmt.Errorf("element with id %d not found", request.ID)
}
<|endoftext|>


<commit_before>package kdtree

import (
	"github.com/hongshibao/go-algo"
)

type Point interface {
	Dim() int
	GetValue(dim int) float64
	Distance(p Point) float64
}

type kdTreeNode struct {
	axis           int
	splittingPoint Point
	leftChild      *kdTreeNode
	rightChild     *kdTreeNode
}

type KDTree struct {
	root *kdTreeNode
	dim  int
}

func (t *KDTree) Dim() int {
	return t.dim
}

func (t *KDTree) KNN(k int) []Point {
	// TODO
	return nil
}

func NewKDTree(points []Point) *KDTree {
	if len(points) == 0 {
		return nil
	}
	ret := &KDTree{
		dim:  points[0].Dim(),
		root: createKDTree(points, 0),
	}
	return ret
}

func createKDTree(points []Point, depth int) *kdTreeNode {
	if len(points) == 0 {
		return nil
	}
	dim := points[0].Dim()
	ret := &kdTreeNode{
		axis: depth % dim,
	}
	if len(points) == 1 {
		ret.splittingPoint = points[0]
		return ret
	}
	idx := selectSplittingPoint(points, ret.axis)
	if idx == -1 {
		return nil
	}
	ret.splittingPoint = points[idx]
	ret.leftChild = createKDTree(points[0:idx-1], depth+1)
	ret.rightChild = createKDTree(points[idx+1:len(points)], depth+1)
	return ret
}

type selectionHelper struct {
	axis   int
	points []Point
}

func (h *selectionHelper) Len() int {
	return len(h.points)
}

func (h *selectionHelper) Less(i, j int) bool {
	return h.points[i].GetValue(h.axis) < h.points[j].GetValue(h.axis)
}

func (h *selectionHelper) Swap(i, j int) {
	h.points[i], h.points[j] = h.points[j], h.points[i]
}

func selectSplittingPoint(points []Point, axis int) int {
	helper := &selectionHelper{
		axis:   axis,
		points: points,
	}
	mid := len(points)/2 + 1
	err := algo.QuickSelect(helper, mid)
	if err != nil {
		return -1
	}
	return mid - 1
}

type kNNHeapNode struct {
	point    Point
	distance float64
}
<commit_msg>kNNHeapHelper done<commit_after>package kdtree

import (
	"github.com/hongshibao/go-algo"
)

type Point interface {
	Dim() int
	GetValue(dim int) float64
	Distance(p Point) float64
}

type kdTreeNode struct {
	axis           int
	splittingPoint Point
	leftChild      *kdTreeNode
	rightChild     *kdTreeNode
}

type KDTree struct {
	root *kdTreeNode
	dim  int
}

func (t *KDTree) Dim() int {
	return t.dim
}

func (t *KDTree) KNN(k int) []Point {
	// TODO
	return nil
}

func NewKDTree(points []Point) *KDTree {
	if len(points) == 0 {
		return nil
	}
	ret := &KDTree{
		dim:  points[0].Dim(),
		root: createKDTree(points, 0),
	}
	return ret
}

func createKDTree(points []Point, depth int) *kdTreeNode {
	if len(points) == 0 {
		return nil
	}
	dim := points[0].Dim()
	ret := &kdTreeNode{
		axis: depth % dim,
	}
	if len(points) == 1 {
		ret.splittingPoint = points[0]
		return ret
	}
	idx := selectSplittingPoint(points, ret.axis)
	if idx == -1 {
		return nil
	}
	ret.splittingPoint = points[idx]
	ret.leftChild = createKDTree(points[0:idx-1], depth+1)
	ret.rightChild = createKDTree(points[idx+1:len(points)], depth+1)
	return ret
}

type selectionHelper struct {
	axis   int
	points []Point
}

func (h *selectionHelper) Len() int {
	return len(h.points)
}

func (h *selectionHelper) Less(i, j int) bool {
	return h.points[i].GetValue(h.axis) < h.points[j].GetValue(h.axis)
}

func (h *selectionHelper) Swap(i, j int) {
	h.points[i], h.points[j] = h.points[j], h.points[i]
}

func selectSplittingPoint(points []Point, axis int) int {
	helper := &selectionHelper{
		axis:   axis,
		points: points,
	}
	mid := len(points)/2 + 1
	err := algo.QuickSelect(helper, mid)
	if err != nil {
		return -1
	}
	return mid - 1
}

type kNNHeapNode struct {
	point    Point
	distance float64
}

type kNNHeapHelper []*kNNHeapNode

func (h kNNHeapHelper) Len() int {
	return len(h)
}

func (h kNNHeapHelper) Less(i, j int) bool {
	return h[i].distance > h[j].distance
}

func (h kNNHeapHelper) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

func (h *kNNHeapHelper) Push(x interface{}) {
	item := x.(*kNNHeapNode)
	*h = append(*h, item)
}

func (h *kNNHeapHelper) Pop() interface{} {
	old := *h
	n := len(old)
	item := old[n-1]
	*h = old[0 : n-1]
	return item
}
<|endoftext|>


<commit_before>// Package kinesis provide GOlang API for http://aws.amazon.com/kinesis/
package kinesis

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

const (
	ActionKey     = "Action"
	RegionEnvName = "AWS_REGION_NAME"

	// Regions
	USEast1      = "us-east-1"
	USWest2      = "us-west-2"
	EUWest1      = "eu-west-1"
	EUCentral1   = "eu-central-1"
	APSouthEast1 = "ap-southeast-1"
	APSouthEast2 = "ap-southeast-2"
	APNortheast1 = "ap-northeast-1"

	kinesisURL = "https://kinesis.%s.amazonaws.com"
)

// NewRegionFromEnv creates a region from the an expected environment variable
func NewRegionFromEnv() string {
	return os.Getenv(RegionEnvName)
}

// Structure for kinesis client
type Kinesis struct {
	client   *Client
	endpoint string
	region   string
	version  string
}

// KinesisClient interface implemented by Kinesis
type KinesisClient interface {
	CreateStream(StreamName string, ShardCount int) error
	DeleteStream(StreamName string) error
	DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error)
	GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error)
	GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error)
	ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error)
	MergeShards(args *RequestArgs) error
	PutRecord(args *RequestArgs) (resp *PutRecordResp, err error)
	PutRecords(args *RequestArgs) (resp *PutRecordsResp, err error)
	SplitShard(args *RequestArgs) error
}

// New returns an initialized AWS Kinesis client using the canonical live “production” endpoint
// for AWS Kinesis, i.e. https://kinesis.{region}.amazonaws.com
func New(auth Auth, region string) *Kinesis {
	endpoint := fmt.Sprintf(kinesisURL, region)
	return NewWithEndpoint(auth, region, endpoint)
}

// NewWithClient returns an initialized AWS Kinesis client using the canonical live “production” endpoint
// for AWS Kinesis, i.e. https://kinesis.{region}.amazonaws.com but with the ability to create a custom client
// with specific configurations like a timeout
func NewWithClient(region string, client *Client) *Kinesis {
	endpoint := fmt.Sprintf(kinesisURL, region)
	return &Kinesis{client: client, version: "20131202", region: region, endpoint: endpoint}
}

// NewWithEndpoint returns an initialized AWS Kinesis client using the specified endpoint.
// This is generally useful for testing, so a local Kinesis server can be used.
func NewWithEndpoint(auth Auth, region string, endpoint string) *Kinesis {
	// TODO: remove trailing slash on endpoint if there is one? does it matter?
	// TODO: validate endpoint somehow?
	return &Kinesis{client: NewClient(auth), version: "20131202", region: region, endpoint: endpoint}
}

// Create params object for request
func makeParams(action string) map[string]string {
	params := make(map[string]string)
	params[ActionKey] = action
	return params
}

// RequestArgs store params for request
type RequestArgs struct {
	params  map[string]interface{}
	Records []Record
}

// NewFilter creates a new Filter.
func NewArgs() *RequestArgs {
	return &RequestArgs{
		params: make(map[string]interface{}),
	}
}

// Add appends a filtering parameter with the given name and value(s).
func (f *RequestArgs) Add(name string, value interface{}) {
	f.params[name] = value
}

func (f *RequestArgs) AddData(value []byte) {
	f.params["Data"] = value
}

// Error represent error from Kinesis API
type Error struct {
	// HTTP status code (200, 403, ...)
	StatusCode int
	// error code ("UnsupportedOperation", ...)
	Code string
	// The human-oriented error message
	Message   string
	RequestId string
}

// Return error message from error object
func (err *Error) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Message, err.Code)
}

type jsonErrors struct {
	Code    string `json:"__type"`
	Message string
}

func buildError(r *http.Response) error {
	// Reading the body into a []byte because we might need to put it into an error
	// message after having the JSON decoding fail to produce a message.
	body, ioerr := ioutil.ReadAll(r.Body)
	if ioerr != nil {
		return fmt.Errorf("Could not read response body: %s", ioerr)
	}

	errors := jsonErrors{}
	json.NewDecoder(bytes.NewReader(body)).Decode(&errors)

	var err Error
	err.Message = errors.Message
	err.Code = errors.Code
	err.StatusCode = r.StatusCode
	if err.Message == "" {
		err.Message = fmt.Sprintf("%s: %s", r.Status, body)
	}
	return &err
}

// Query by AWS API
func (kinesis *Kinesis) query(params map[string]string, data interface{}, resp interface{}) error {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return err
	}

	// request
	request, err := http.NewRequest(
		"POST",
		kinesis.endpoint,
		bytes.NewReader(jsonData),
	)

	if err != nil {
		return err
	}

	// headers
	request.Header.Set("Content-Type", "application/x-amz-json-1.1")
	request.Header.Set("X-Amz-Target", fmt.Sprintf("Kinesis_%s.%s", kinesis.version, params[ActionKey]))
	request.Header.Set("User-Agent", "Golang Kinesis")

	// response
	response, err := kinesis.client.Do(request)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return buildError(response)
	}

	if resp == nil {
		return nil
	}

	return json.NewDecoder(response.Body).Decode(resp)
}

// CreateStream adds a new Amazon Kinesis stream to your AWS account
// StreamName is a name of stream, ShardCount is number of shards
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_CreateStream.html
func (kinesis *Kinesis) CreateStream(StreamName string, ShardCount int) error {
	params := makeParams("CreateStream")
	requestParams := struct {
		StreamName string
		ShardCount int
	}{
		StreamName,
		ShardCount,
	}
	err := kinesis.query(params, requestParams, nil)
	if err != nil {
		return err
	}
	return nil
}

// DeleteStream deletes a stream and all of its shards and data from your AWS account
// StreamName is a name of stream
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DeleteStream.html
func (kinesis *Kinesis) DeleteStream(StreamName string) error {
	params := makeParams("DeleteStream")
	requestParams := struct {
		StreamName string
	}{
		StreamName,
	}
	err := kinesis.query(params, requestParams, nil)
	if err != nil {
		return err
	}
	return nil
}

// MergeShards merges two adjacent shards in a stream and combines them into a single shard to reduce the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_MergeShards.html
func (kinesis *Kinesis) MergeShards(args *RequestArgs) error {
	params := makeParams("MergeShards")
	err := kinesis.query(params, args.params, nil)
	if err != nil {
		return err
	}
	return nil
}

// SplitShard splits a shard into two new shards in the stream, to increase the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_SplitShard.html
func (kinesis *Kinesis) SplitShard(args *RequestArgs) error {
	params := makeParams("SplitShard")
	err := kinesis.query(params, args.params, nil)
	if err != nil {
		return err
	}
	return nil
}

// ListStreamsResp stores the information that provides by ListStreams API call
type ListStreamsResp struct {
	HasMoreStreams bool
	StreamNames    []string
}

// ListStreams returns an array of the names of all the streams that are associated with the AWS account making the ListStreams request
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListStreams.html
func (kinesis *Kinesis) ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error) {
	params := makeParams("ListStreams")
	resp = &ListStreamsResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// DescribeStreamShards stores the information about list of shards inside DescribeStreamResp
type DescribeStreamShards struct {
	AdjacentParentShardId string
	HashKeyRange          struct {
		EndingHashKey   string
		StartingHashKey string
	}
	ParentShardId       string
	SequenceNumberRange struct {
		EndingSequenceNumber   string
		StartingSequenceNumber string
	}
	ShardId string
}

// DescribeStreamResp stores the information that provides by DescribeStream API call
type DescribeStreamResp struct {
	StreamDescription struct {
		HasMoreShards bool
		Shards        []DescribeStreamShards
		StreamARN     string
		StreamName    string
		StreamStatus  string
	}
}

// DescribeStream returns the following information about the stream: the current status of the stream,
// the stream Amazon Resource Name (ARN), and an array of shard objects that comprise the stream.
// For each shard object there is information about the hash key and sequence number ranges that
// the shard spans, and the IDs of any earlier shards that played in a role in a MergeShards or
// SplitShard operation that created the shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html
func (kinesis *Kinesis) DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error) {
	params := makeParams("DescribeStream")
	resp = &DescribeStreamResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// GetShardIteratorResp stores the information that provides by GetShardIterator API call
type GetShardIteratorResp struct {
	ShardIterator string
}

// GetShardIterator returns a shard iterator
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html
func (kinesis *Kinesis) GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error) {
	params := makeParams("GetShardIterator")
	resp = &GetShardIteratorResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// GetNextRecordsRecords stores the information that provides by GetNextRecordsResp
type GetRecordsRecords struct {
	ApproximateArrivalTimestamp int64
	Data                        []byte
	PartitionKey                string
	SequenceNumber              string
}

func (r GetRecordsRecords) GetData() []byte {
	return r.Data
}

// GetNextRecordsResp stores the information that provides by GetNextRecords API call
type GetRecordsResp struct {
	MillisBehindLatest int64
	NextShardIterator  string
	Records            []GetRecordsRecords
}

// GetRecords returns one or more data records from a shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
func (kinesis *Kinesis) GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error) {
	params := makeParams("GetRecords")
	resp = &GetRecordsResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// PutRecordResp stores the information that provides by PutRecord API call
type PutRecordResp struct {
	SequenceNumber string
	ShardId        string
}

// PutRecord puts a data record into an Amazon Kinesis stream from a producer.
// args must contain a single record added with AddRecord.
// More info: http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html
func (kinesis *Kinesis) PutRecord(args *RequestArgs) (resp *PutRecordResp, err error) {
	params := makeParams("PutRecord")

	if _, ok := args.params["Data"]; !ok && len(args.Records) == 0 {
		return nil, errors.New("PutRecord requires its args param to contain a record added with either AddRecord or AddData.")
	} else if ok && len(args.Records) > 0 {
		return nil, errors.New("PutRecord requires its args param to contain a record added with either AddRecord or AddData but not both.")
	} else if len(args.Records) > 1 {
		return nil, errors.New("PutRecord does not support more than one record.")
	}

	if len(args.Records) > 0 {
		args.AddData(args.Records[0].Data)
		args.Add("PartitionKey", args.Records[0].PartitionKey)
	}

	resp = &PutRecordResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// PutRecords puts multiple data records into an Amazon Kinesis stream from a producer
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
func (kinesis *Kinesis) PutRecords(args *RequestArgs) (resp *PutRecordsResp, err error) {
	params := makeParams("PutRecords")
	resp = &PutRecordsResp{}
	args.Add("Records", args.Records)
	err = kinesis.query(params, args.params, resp)

	if err != nil {
		return nil, err
	}
	return
}

// PutRecordsResp stores the information that provides by PutRecord API call
type PutRecordsResp struct {
	FailedRecordCount int
	Records           []PutRecordsRespRecord
}

// RecordResp stores individual Record information provided by PutRecords API call
type PutRecordsRespRecord struct {
	ErrorCode      string
	ErrorMessage   string
	SequenceNumber string
	ShardId        string
}

// Add data and partition for sending multiple Records to Kinesis in one API call
func (f *RequestArgs) AddRecord(value []byte, partitionKey string) {
	r := Record{
		Data:         value,
		PartitionKey: partitionKey,
	}
	f.Records = append(f.Records, r)
}

// Record stores the Data and PartitionKey for PutRecord or PutRecords calls to Kinesis API
type Record struct {
	Data         []byte
	PartitionKey string
}
<commit_msg>get additional fields from GetRecords<commit_after>// Package kinesis provide GOlang API for http://aws.amazon.com/kinesis/
package kinesis

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
)

const (
	ActionKey     = "Action"
	RegionEnvName = "AWS_REGION_NAME"

	// Regions
	USEast1      = "us-east-1"
	USWest2      = "us-west-2"
	EUWest1      = "eu-west-1"
	EUCentral1   = "eu-central-1"
	APSouthEast1 = "ap-southeast-1"
	APSouthEast2 = "ap-southeast-2"
	APNortheast1 = "ap-northeast-1"

	kinesisURL = "https://kinesis.%s.amazonaws.com"
)

// NewRegionFromEnv creates a region from the an expected environment variable
func NewRegionFromEnv() string {
	return os.Getenv(RegionEnvName)
}

// Structure for kinesis client
type Kinesis struct {
	client   *Client
	endpoint string
	region   string
	version  string
}

// KinesisClient interface implemented by Kinesis
type KinesisClient interface {
	CreateStream(StreamName string, ShardCount int) error
	DeleteStream(StreamName string) error
	DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error)
	GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error)
	GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error)
	ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error)
	MergeShards(args *RequestArgs) error
	PutRecord(args *RequestArgs) (resp *PutRecordResp, err error)
	PutRecords(args *RequestArgs) (resp *PutRecordsResp, err error)
	SplitShard(args *RequestArgs) error
}

// New returns an initialized AWS Kinesis client using the canonical live “production” endpoint
// for AWS Kinesis, i.e. https://kinesis.{region}.amazonaws.com
func New(auth Auth, region string) *Kinesis {
	endpoint := fmt.Sprintf(kinesisURL, region)
	return NewWithEndpoint(auth, region, endpoint)
}

// NewWithClient returns an initialized AWS Kinesis client using the canonical live “production” endpoint
// for AWS Kinesis, i.e. https://kinesis.{region}.amazonaws.com but with the ability to create a custom client
// with specific configurations like a timeout
func NewWithClient(region string, client *Client) *Kinesis {
	endpoint := fmt.Sprintf(kinesisURL, region)
	return &Kinesis{client: client, version: "20131202", region: region, endpoint: endpoint}
}

// NewWithEndpoint returns an initialized AWS Kinesis client using the specified endpoint.
// This is generally useful for testing, so a local Kinesis server can be used.
func NewWithEndpoint(auth Auth, region string, endpoint string) *Kinesis {
	// TODO: remove trailing slash on endpoint if there is one? does it matter?
	// TODO: validate endpoint somehow?
	return &Kinesis{client: NewClient(auth), version: "20131202", region: region, endpoint: endpoint}
}

// Create params object for request
func makeParams(action string) map[string]string {
	params := make(map[string]string)
	params[ActionKey] = action
	return params
}

// RequestArgs store params for request
type RequestArgs struct {
	params  map[string]interface{}
	Records []Record
}

// NewFilter creates a new Filter.
func NewArgs() *RequestArgs {
	return &RequestArgs{
		params: make(map[string]interface{}),
	}
}

// Add appends a filtering parameter with the given name and value(s).
func (f *RequestArgs) Add(name string, value interface{}) {
	f.params[name] = value
}

func (f *RequestArgs) AddData(value []byte) {
	f.params["Data"] = value
}

// Error represent error from Kinesis API
type Error struct {
	// HTTP status code (200, 403, ...)
	StatusCode int
	// error code ("UnsupportedOperation", ...)
	Code string
	// The human-oriented error message
	Message   string
	RequestId string
}

// Return error message from error object
func (err *Error) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Message, err.Code)
}

type jsonErrors struct {
	Code    string `json:"__type"`
	Message string
}

func buildError(r *http.Response) error {
	// Reading the body into a []byte because we might need to put it into an error
	// message after having the JSON decoding fail to produce a message.
	body, ioerr := ioutil.ReadAll(r.Body)
	if ioerr != nil {
		return fmt.Errorf("Could not read response body: %s", ioerr)
	}

	errors := jsonErrors{}
	json.NewDecoder(bytes.NewReader(body)).Decode(&errors)

	var err Error
	err.Message = errors.Message
	err.Code = errors.Code
	err.StatusCode = r.StatusCode
	if err.Message == "" {
		err.Message = fmt.Sprintf("%s: %s", r.Status, body)
	}
	return &err
}

// Query by AWS API
func (kinesis *Kinesis) query(params map[string]string, data interface{}, resp interface{}) error {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return err
	}

	// request
	request, err := http.NewRequest(
		"POST",
		kinesis.endpoint,
		bytes.NewReader(jsonData),
	)

	if err != nil {
		return err
	}

	// headers
	request.Header.Set("Content-Type", "application/x-amz-json-1.1")
	request.Header.Set("X-Amz-Target", fmt.Sprintf("Kinesis_%s.%s", kinesis.version, params[ActionKey]))
	request.Header.Set("User-Agent", "Golang Kinesis")

	// response
	response, err := kinesis.client.Do(request)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return buildError(response)
	}

	if resp == nil {
		return nil
	}

	return json.NewDecoder(response.Body).Decode(resp)
}

// CreateStream adds a new Amazon Kinesis stream to your AWS account
// StreamName is a name of stream, ShardCount is number of shards
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_CreateStream.html
func (kinesis *Kinesis) CreateStream(StreamName string, ShardCount int) error {
	params := makeParams("CreateStream")
	requestParams := struct {
		StreamName string
		ShardCount int
	}{
		StreamName,
		ShardCount,
	}
	err := kinesis.query(params, requestParams, nil)
	if err != nil {
		return err
	}
	return nil
}

// DeleteStream deletes a stream and all of its shards and data from your AWS account
// StreamName is a name of stream
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DeleteStream.html
func (kinesis *Kinesis) DeleteStream(StreamName string) error {
	params := makeParams("DeleteStream")
	requestParams := struct {
		StreamName string
	}{
		StreamName,
	}
	err := kinesis.query(params, requestParams, nil)
	if err != nil {
		return err
	}
	return nil
}

// MergeShards merges two adjacent shards in a stream and combines them into a single shard to reduce the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_MergeShards.html
func (kinesis *Kinesis) MergeShards(args *RequestArgs) error {
	params := makeParams("MergeShards")
	err := kinesis.query(params, args.params, nil)
	if err != nil {
		return err
	}
	return nil
}

// SplitShard splits a shard into two new shards in the stream, to increase the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_SplitShard.html
func (kinesis *Kinesis) SplitShard(args *RequestArgs) error {
	params := makeParams("SplitShard")
	err := kinesis.query(params, args.params, nil)
	if err != nil {
		return err
	}
	return nil
}

// ListStreamsResp stores the information that provides by ListStreams API call
type ListStreamsResp struct {
	HasMoreStreams bool
	StreamNames    []string
}

// ListStreams returns an array of the names of all the streams that are associated with the AWS account making the ListStreams request
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListStreams.html
func (kinesis *Kinesis) ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error) {
	params := makeParams("ListStreams")
	resp = &ListStreamsResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// DescribeStreamShards stores the information about list of shards inside DescribeStreamResp
type DescribeStreamShards struct {
	AdjacentParentShardId string
	HashKeyRange          struct {
		EndingHashKey   string
		StartingHashKey string
	}
	ParentShardId       string
	SequenceNumberRange struct {
		EndingSequenceNumber   string
		StartingSequenceNumber string
	}
	ShardId string
}

// DescribeStreamResp stores the information that provides by DescribeStream API call
type DescribeStreamResp struct {
	StreamDescription struct {
		HasMoreShards bool
		Shards        []DescribeStreamShards
		StreamARN     string
		StreamName    string
		StreamStatus  string
	}
}

// DescribeStream returns the following information about the stream: the current status of the stream,
// the stream Amazon Resource Name (ARN), and an array of shard objects that comprise the stream.
// For each shard object there is information about the hash key and sequence number ranges that
// the shard spans, and the IDs of any earlier shards that played in a role in a MergeShards or
// SplitShard operation that created the shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html
func (kinesis *Kinesis) DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error) {
	params := makeParams("DescribeStream")
	resp = &DescribeStreamResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// GetShardIteratorResp stores the information that provides by GetShardIterator API call
type GetShardIteratorResp struct {
	ShardIterator string
}

// GetShardIterator returns a shard iterator
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html
func (kinesis *Kinesis) GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error) {
	params := makeParams("GetShardIterator")
	resp = &GetShardIteratorResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// GetNextRecordsRecords stores the information that provides by GetNextRecordsResp
type GetRecordsRecords struct {
	ApproximateArrivalTimestamp float64
	Data                        []byte
	PartitionKey                string
	SequenceNumber              string
}

func (r GetRecordsRecords) GetData() []byte {
	return r.Data
}

// GetNextRecordsResp stores the information that provides by GetNextRecords API call
type GetRecordsResp struct {
	MillisBehindLatest int64
	NextShardIterator  string
	Records            []GetRecordsRecords
}

// GetRecords returns one or more data records from a shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
func (kinesis *Kinesis) GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error) {
	params := makeParams("GetRecords")
	resp = &GetRecordsResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// PutRecordResp stores the information that provides by PutRecord API call
type PutRecordResp struct {
	SequenceNumber string
	ShardId        string
}

// PutRecord puts a data record into an Amazon Kinesis stream from a producer.
// args must contain a single record added with AddRecord.
// More info: http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html
func (kinesis *Kinesis) PutRecord(args *RequestArgs) (resp *PutRecordResp, err error) {
	params := makeParams("PutRecord")

	if _, ok := args.params["Data"]; !ok && len(args.Records) == 0 {
		return nil, errors.New("PutRecord requires its args param to contain a record added with either AddRecord or AddData.")
	} else if ok && len(args.Records) > 0 {
		return nil, errors.New("PutRecord requires its args param to contain a record added with either AddRecord or AddData but not both.")
	} else if len(args.Records) > 1 {
		return nil, errors.New("PutRecord does not support more than one record.")
	}

	if len(args.Records) > 0 {
		args.AddData(args.Records[0].Data)
		args.Add("PartitionKey", args.Records[0].PartitionKey)
	}

	resp = &PutRecordResp{}
	err = kinesis.query(params, args.params, resp)
	if err != nil {
		return nil, err
	}
	return
}

// PutRecords puts multiple data records into an Amazon Kinesis stream from a producer
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
func (kinesis *Kinesis) PutRecords(args *RequestArgs) (resp *PutRecordsResp, err error) {
	params := makeParams("PutRecords")
	resp = &PutRecordsResp{}
	args.Add("Records", args.Records)
	err = kinesis.query(params, args.params, resp)

	if err != nil {
		return nil, err
	}
	return
}

// PutRecordsResp stores the information that provides by PutRecord API call
type PutRecordsResp struct {
	FailedRecordCount int
	Records           []PutRecordsRespRecord
}

// RecordResp stores individual Record information provided by PutRecords API call
type PutRecordsRespRecord struct {
	ErrorCode      string
	ErrorMessage   string
	SequenceNumber string
	ShardId        string
}

// Add data and partition for sending multiple Records to Kinesis in one API call
func (f *RequestArgs) AddRecord(value []byte, partitionKey string) {
	r := Record{
		Data:         value,
		PartitionKey: partitionKey,
	}
	f.Records = append(f.Records, r)
}

// Record stores the Data and PartitionKey for PutRecord or PutRecords calls to Kinesis API
type Record struct {
	Data         []byte
	PartitionKey string
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"time"
	"reflect"
	"strconv"
	"syscall"
	"os/signal"
	"math/rand"
)

type Kernel struct {
	Configuration *Configuration
	Components map[string]Component
	components []Component
	Id string
	Logger
	Pid int
}

type Component struct {
	componentId string
	singleton interface{}
	startMethodName string
	stopMethodName string
}

// Access another component. This method will panic if you attempt to reference a
// non-existent component. If the component id has a length of zero, it is also panics.
func (self *Kernel) GetComponent(componentId string) interface{} {

	if len(componentId) == 0 {
		panic("kernel.GetComponent called with an empty component id")
	}

	if _, found := self.Components[componentId]; !found {
		panic(fmt.Sprintf("kernel.GetComponent called with an invalid component id: %s", componentId))
	}

	return self.Components[componentId].singleton.(interface{})
}

// Register a component with a start and stop methods.
func (self *Kernel) AddComponentWithStartStopMethods(componentId string, singleton interface{}, startMethodName, stopMethodName string) {

	component := Component{ componentId : componentId, singleton : singleton, startMethodName : startMethodName, stopMethodName : stopMethodName }

	self.components = append(self.components , component)
	self.Components[componentId] = component
}

// Register a component with a start method.
func (self *Kernel) AddComponentWithStartMethod(componentId string, singleton interface{}, startMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, startMethodName, "")
}

// Register a component with a stop method.
func (self *Kernel) AddComponentWithStopMethod(componentId string, singleton interface{}, stopMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", stopMethodName)
}

// Register a component without a start or stop method.
func (self *Kernel) AddComponent(componentId string, singleton interface{}) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", "")
}

// Called by the kernel during Start/Stop.
func callStartStopMethod(methodTypeName, methodName string, singleton interface{}, kernel *Kernel) error {

	value := reflect.ValueOf(singleton)

	methodValue := value.MethodByName(methodName)

	if !methodValue.IsValid() {
		return fmt.Errorf("Start method: %s is NOT found on struct: %s", methodName, value.Type())
	}

	methodType := methodValue.Type()

	if methodType.NumOut() > 1 {
		panic(fmt.Sprintf("The %s method: %s on struct: %s has more than one return value - you can only return error or nothing", methodTypeName, methodName, value.Type()))
	}

	if methodType.NumIn() > 1 {
		return fmt.Errorf("The %s method: %s on struct: %s has more than one parameter - you can only accept Kernel or nothing", methodTypeName, methodName, value.Type())
	}

	// Verify the return type is error
	if methodType.NumOut() == 1 && methodType.Out(0).Name() != "error"  {

		return fmt.Errorf("The %s method: %s on struct: %s has an invalid return type - you can return nothing or error", methodTypeName, methodName, value.Type())
	}

	methodInputs := make([]reflect.Value, 0)
	if methodType.NumIn() == 1 {
		methodInputs = append(methodInputs, reflect.ValueOf(kernel))
	}

	returnValues := methodValue.Call(methodInputs)

	// Check to see if there was an error
	if len(returnValues) == 1 {
		err := returnValues[0].Interface()
		if err != nil {
			return err.(error)
		}
	}

	return nil
}

// Call this after the kernel has been created and components registered.
func (self *Kernel) Start() error {

	rand.Seed(time.Now().UTC().UnixNano())

	self.Logf(Info, "Starting %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := range self.components {
		if len(self.components[i].startMethodName) > 0 {
			if err := callStartStopMethod("start", self.components[i].startMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Started %s - version: %s - config file: %s ", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

// Stop the kernel. Call this before exiting.
func (self *Kernel) Stop() error {

	self.Logf(Info, "Stopping %s - version: %s - config file %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := len(self.components)-1 ; i >= 0 ; i-- {

		if len(self.components[i].stopMethodName) > 0 {
			if err := callStartStopMethod("stop", self.components[i].stopMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Stopped %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func newKernel(id, configFileName string) (*Kernel, error) {

	// Init the application configuration
	conf, err := NewConfiguration(configFileName)

	if err != nil {
		return nil, err
	}

	// TODO: Add a logging structure to the configuration file and configure.

	logger := Logger {
		Prefix: id,
		Appenders: [] Appender{
			LevelFilter(Debug, StdErrAppender()),
		},
	}

	kernel := &Kernel{ Components : make(map[string]Component), Configuration : conf }
	kernel.Logger = logger
	kernel.Id = id

	if err = writePidFile(kernel); err != nil {
		return nil, err
	}

	return kernel, nil
}

func writePidFile(kernel *Kernel) error {
	kernel.Pid = os.Getpid()
	pidFile, err := os.Create(kernel.Configuration.PidFile)
	if err != nil {
		return NewStackError("Unable to start kernel - problem creating pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}
	defer pidFile.Close()

	if _, err := pidFile.Write([]byte(strconv.Itoa(kernel.Pid))); err != nil {
		return NewStackError("Unable to start kernel - problem writing pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}

	return nil
}

// Call this from your main to create the kernel. After init kernel is called you must add
// your components and then call kernel.Start()
func StartKernel(id string, configFileName string, addComponentsFunction func(kernel *Kernel)) (*Kernel, error) {

	kernel, err := newKernel(id, configFileName)
	if err != nil {
		return nil, err
	}

	addComponentsFunction(kernel)

    if err = kernel.Start(); err != nil {
		return nil, err
	}

	return kernel, nil
}

// ListenForInterrupt blocks until an interrupt signal is detected.
func (self *Kernel) ListenForInterrupt() error {
	quitChannel := make(chan bool)

	// Register the interrupt listener.
	interruptSignalChannel := make(chan os.Signal, 1)
	signal.Notify(interruptSignalChannel, os.Interrupt)
	go func() {
		for sig := range interruptSignalChannel {
			if sig == syscall.SIGINT {
				quitChannel <- true
			}
		}
	}()

	// Block until we receive the stop notification.
	select {
		case <- quitChannel: {
			return self.Stop()
		}
	}

	// Should never happen
	panic("How did we end up here?")
}

<commit_msg>added syslog appender<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"os"
	"fmt"
	"time"
	"reflect"
	"strconv"
	"syscall"
	"os/signal"
	"math/rand"
)

type Kernel struct {
	Configuration *Configuration
	Components map[string]Component
	components []Component
	Id string
	Logger
	Pid int
}

type Component struct {
	componentId string
	singleton interface{}
	startMethodName string
	stopMethodName string
}

// Access another component. This method will panic if you attempt to reference a
// non-existent component. If the component id has a length of zero, it is also panics.
func (self *Kernel) GetComponent(componentId string) interface{} {

	if len(componentId) == 0 {
		panic("kernel.GetComponent called with an empty component id")
	}

	if _, found := self.Components[componentId]; !found {
		panic(fmt.Sprintf("kernel.GetComponent called with an invalid component id: %s", componentId))
	}

	return self.Components[componentId].singleton.(interface{})
}

// Register a component with a start and stop methods.
func (self *Kernel) AddComponentWithStartStopMethods(componentId string, singleton interface{}, startMethodName, stopMethodName string) {

	component := Component{ componentId : componentId, singleton : singleton, startMethodName : startMethodName, stopMethodName : stopMethodName }

	self.components = append(self.components , component)
	self.Components[componentId] = component
}

// Register a component with a start method.
func (self *Kernel) AddComponentWithStartMethod(componentId string, singleton interface{}, startMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, startMethodName, "")
}

// Register a component with a stop method.
func (self *Kernel) AddComponentWithStopMethod(componentId string, singleton interface{}, stopMethodName string) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", stopMethodName)
}

// Register a component without a start or stop method.
func (self *Kernel) AddComponent(componentId string, singleton interface{}) {
	self.AddComponentWithStartStopMethods(componentId, singleton, "", "")
}

// Called by the kernel during Start/Stop.
func callStartStopMethod(methodTypeName, methodName string, singleton interface{}, kernel *Kernel) error {

	value := reflect.ValueOf(singleton)

	methodValue := value.MethodByName(methodName)

	if !methodValue.IsValid() {
		return fmt.Errorf("Start method: %s is NOT found on struct: %s", methodName, value.Type())
	}

	methodType := methodValue.Type()

	if methodType.NumOut() > 1 {
		panic(fmt.Sprintf("The %s method: %s on struct: %s has more than one return value - you can only return error or nothing", methodTypeName, methodName, value.Type()))
	}

	if methodType.NumIn() > 1 {
		return fmt.Errorf("The %s method: %s on struct: %s has more than one parameter - you can only accept Kernel or nothing", methodTypeName, methodName, value.Type())
	}

	// Verify the return type is error
	if methodType.NumOut() == 1 && methodType.Out(0).Name() != "error"  {

		return fmt.Errorf("The %s method: %s on struct: %s has an invalid return type - you can return nothing or error", methodTypeName, methodName, value.Type())
	}

	methodInputs := make([]reflect.Value, 0)
	if methodType.NumIn() == 1 {
		methodInputs = append(methodInputs, reflect.ValueOf(kernel))
	}

	returnValues := methodValue.Call(methodInputs)

	// Check to see if there was an error
	if len(returnValues) == 1 {
		err := returnValues[0].Interface()
		if err != nil {
			return err.(error)
		}
	}

	return nil
}

// Call this after the kernel has been created and components registered.
func (self *Kernel) Start() error {

	rand.Seed(time.Now().UTC().UnixNano())

	self.Logf(Info, "Starting %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := range self.components {
		if len(self.components[i].startMethodName) > 0 {
			if err := callStartStopMethod("start", self.components[i].startMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Started %s - version: %s - config file: %s ", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

// Stop the kernel. Call this before exiting.
func (self *Kernel) Stop() error {

	self.Logf(Info, "Stopping %s - version: %s - config file %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	for i := len(self.components)-1 ; i >= 0 ; i-- {

		if len(self.components[i].stopMethodName) > 0 {
			if err := callStartStopMethod("stop", self.components[i].stopMethodName, self.components[i].singleton, self); err != nil {
				return err
			}
		}
	}

	self.Logf(Info, "Stopped %s - version: %s - config file: %s", self.Id, self.Configuration.Version, self.Configuration.FileName)

	return nil
}

func newKernel(id, configFileName string) (*Kernel, error) {

	// Init the application configuration
	conf, err := NewConfiguration(configFileName)

	if err != nil {
		return nil, err
	}

	// TODO: Add a logging structure to the configuration file and configure. Make
	// sure this supports configuring syslog.

	syslogAppender, err := NewSyslogAppender("tcp", "127.0.0.1", id)
	if err != nil {
		return nil, err
	}

	logger := Logger {
		Prefix: id,
		Appenders: [] Appender{
			LevelFilter(Debug, StdErrAppender()),
			LevelFilter(Debug, syslogAppender),
		},
	}

	kernel := &Kernel{ Components : make(map[string]Component), Configuration : conf }
	kernel.Logger = logger
	kernel.Id = id

	if err = writePidFile(kernel); err != nil {
		return nil, err
	}

	return kernel, nil
}

func writePidFile(kernel *Kernel) error {
	kernel.Pid = os.Getpid()
	pidFile, err := os.Create(kernel.Configuration.PidFile)
	if err != nil {
		return NewStackError("Unable to start kernel - problem creating pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}
	defer pidFile.Close()

	if _, err := pidFile.Write([]byte(strconv.Itoa(kernel.Pid))); err != nil {
		return NewStackError("Unable to start kernel - problem writing pid file %s - error: %v", kernel.Configuration.PidFile, err)
	}

	return nil
}

// Call this from your main to create the kernel. After init kernel is called you must add
// your components and then call kernel.Start()
func StartKernel(id string, configFileName string, addComponentsFunction func(kernel *Kernel)) (*Kernel, error) {

	kernel, err := newKernel(id, configFileName)
	if err != nil {
		return nil, err
	}

	addComponentsFunction(kernel)

    if err = kernel.Start(); err != nil {
		return nil, err
	}

	return kernel, nil
}

// ListenForInterrupt blocks until an interrupt signal is detected.
func (self *Kernel) ListenForInterrupt() error {
	quitChannel := make(chan bool)

	// Register the interrupt listener.
	interruptSignalChannel := make(chan os.Signal, 1)
	signal.Notify(interruptSignalChannel, os.Interrupt)
	go func() {
		for sig := range interruptSignalChannel {
			if sig == syscall.SIGINT {
				quitChannel <- true
			}
		}
	}()

	// Block until we receive the stop notification.
	select {
		case <- quitChannel: {
			return self.Stop()
		}
	}

	// Should never happen
	panic("How did we end up here?")
}

<|endoftext|>


<commit_before>package setup

import (
	"compress/bzip2"
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_async"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_util"
	"github.com/watermint/toolbox/domain/dropbox/filesystem/dbx_fs_copier_batch"
	mo_path2 "github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/essentials/api/api_request"
	"github.com/watermint/toolbox/essentials/go/es_lang"
	"github.com/watermint/toolbox/essentials/io/es_rewinder"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/essentials/model/mo_int"
	"github.com/watermint/toolbox/essentials/model/mo_path"
	"github.com/watermint/toolbox/essentials/time/ut_format"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"golang.org/x/sync/semaphore"
	"io"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type Contributor struct {
	Username string `xml:"username"`
	Id       string `xml:"id"`
}

type Revision struct {
	Id          string      `xml:"id"`
	ParentId    string      `xml:"parentid"`
	Timestamp   string      `xml:"timestamp"`
	Contributor Contributor `xml:"contributor"`
	Comment     string      `xml:"comment"`
	Model       string      `xml:"model"`
	Format      string      `xml:"format"`
	Text        string      `xml:"text"`
	Sha1        string      `xml:"sha1"`
}

type Page struct {
	Title    string      `xml:"title"`
	Ns       string      `xml:"ns"`
	Id       string      `xml:"id"`
	Revision []*Revision `xml:"revision"`
}

type WikimediaLoader struct {
	l         esl.Logger
	skip      int
	batchSize int
}

func (z WikimediaLoader) LoadBz2(path string, handler func(p Page) error) error {
	f, err := os.Open(path)
	if err != nil {
		z.l.Warn("Can't open the file", esl.Error(err))
		return err
	}
	defer func() {
		_ = f.Close()
	}()

	bf := bzip2.NewReader(f)

	return z.load(bf, handler)
}

func (z WikimediaLoader) LoadXml(path string, handler func(p Page) error) error {
	f, err := os.Open(path)
	if err != nil {
		z.l.Warn("Can't open the file", esl.Error(err))
		return err
	}
	defer func() {
		_ = f.Close()
	}()
	return z.load(f, handler)
}

func (z WikimediaLoader) load(stream io.Reader, handler func(p Page) error) error {
	d := xml.NewDecoder(stream)
	index := 0
	lastMark := time.Now()
	firstMark := lastMark
	for {
		t, err := d.Token()
		if err != nil {
			z.l.Warn("cannot parse", esl.Error(err))
			return err
		}
		if t == nil {
			z.l.Debug("Reached to EOL")
			return nil
		}

		switch se := t.(type) {
		case xml.StartElement:
			e := se.Name.Local
			switch e {
			case "page":
				var page Page
				if err := d.DecodeElement(&page, &se); err != nil {
					z.l.Warn("Can't error", esl.Error(err))
					return err
				}
				index++
				if pageId, err := strconv.ParseInt(page.Id, 10, 64); err == nil {
					if pageId < int64(z.skip) {
						continue
					}
				}
				if index%z.batchSize == 0 {
					var estimatedThroughputSpan, estimatedThroughputTotal float64

					lastSpan := time.Now().Sub(lastMark).Seconds()
					if 0 < lastSpan {
						estimatedThroughputSpan = float64(z.batchSize) / lastSpan
					}
					totalSpan := time.Now().Sub(firstMark).Seconds()
					if 0 < totalSpan {
						estimatedThroughputTotal = float64(index) / totalSpan
					}
					z.l.Info("Loaded",
						esl.Time("time", time.Now()),
						esl.String("pageId", page.Id),
						esl.Int("index", index),
						esl.Float64("tps (span)", estimatedThroughputSpan),
						esl.Float64("tps (total)", estimatedThroughputTotal),
					)
					lastMark = time.Now()
				}
				if err := handler(page); err != nil {
					z.l.Warn("Can't handle the page", esl.Error(err), esl.Any("page", page))
					return err
				}
			}
		}
	}
}

type Massfiles struct {
	rc_recipe.RemarkSecret
	Peer      dbx_conn.ConnScopedIndividual
	Source    mo_path.ExistingFileSystemPath
	Base      mo_path2.DropboxPath
	Offset    int
	ShardSize mo_int.RangeInt
	BatchSize mo_int.RangeInt
}

func (z *Massfiles) Preset() {
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesContentRead,
		dbx_auth.ScopeFilesContentWrite,
	)
	z.BatchSize.SetRange(0, 1000, 1000)
	z.ShardSize.SetRange(1, 1000, 20)
}

func (z *Massfiles) Exec(c app_control.Control) error {
	l := c.Log()

	sessions := make(map[string]Page)
	offsets := make(map[string]int64)
	sessionMutex := sync.Mutex{}
	batchSize := z.BatchSize.Value()
	ctx := z.Peer.Client()

	pageContent := func(p Page) string {
		switch len(p.Revision) {
		case 0:
			return p.Title
		default:
			return fmt.Sprintf("= %s =\n\n----\n%s", p.Title, p.Revision[0].Text)
		}
	}

	pageTime := func(p Page) string {
		if len(p.Revision) < 1 {
			return dbx_util.ToApiTimeString(time.Now())
		} else {
			pt, valid := ut_format.ParseTimestamp(p.Revision[0].Timestamp)
			if !valid {
				return dbx_util.ToApiTimeString(time.Now())
			} else {
				return dbx_util.ToApiTimeString(pt)
			}
		}
	}

	pageToPath := func(p Page) []string {
		altPageId := "p-" + p.Id + ".txt"
		pageId, err := strconv.ParseInt(p.Id, 10, 32)
		if err != nil {
			l.Debug("Unable to parse pageId", esl.Error(err), esl.String("pageId", p.Id))
			return []string{"unexpected_page_id", altPageId}
		}
		altPageId = fmt.Sprintf("%d/%d/p-%d.txt", pageId/1_000_000, pageId/1000, pageId)
		if len(p.Revision) < 1 {
			return []string{"no_revision", altPageId}
		}
		pt, valid := ut_format.ParseTimestamp(p.Revision[0].Timestamp)
		if valid {
			return []string{
				fmt.Sprintf("%02d", pageId%z.ShardSize.Value64()),
				fmt.Sprintf("%04d", pt.Year()),
				fmt.Sprintf("%04d-%02d", pt.Year(), pt.Month()),
				fmt.Sprintf("%04d-%02d-%02d", pt.Year(), pt.Month(), pt.Day()),
				fmt.Sprintf("%s.txt", p.Id),
			}
		}
		return []string{"invalid_time_format", altPageId}
	}

	commit := func() error {
		l.Info("Commit", esl.Int("size", len(sessions)))
		if len(sessions) < 1 {
			return nil
		}

		commits := make([]dbx_fs_copier_batch.UploadFinish, 0)
		paths := make([]string, 0)
		for sessionId, page := range sessions {
			path := z.Base.ChildPath(pageToPath(page)...).Path()
			offset := offsets[sessionId]
			paths = append(paths, path)
			commits = append(commits, dbx_fs_copier_batch.UploadFinish{
				Cursor: dbx_fs_copier_batch.UploadCursor{
					SessionId: sessionId,
					Offset:    offset,
				},
				Commit: dbx_fs_copier_batch.CommitInfo{
					Path:           path,
					Mode:           "add",
					Autorename:     true,
					ClientModified: pageTime(page),
					Mute:           true,
					StrictConflict: false,
				},
			})
		}

		finish := &dbx_fs_copier_batch.UploadFinishBatch{
			Entries: commits,
		}
		res := ctx.Async("files/upload_session/finish_batch", api_request.Param(finish)).Call(
			dbx_async.Status("files/upload_session/finish_batch/check"),
		)

		if err, f := res.Failure(); f {
			l.Debug("Unable to finish the batch", esl.Error(err))
			return err
		}

		// clean sessions
		sessions = make(map[string]Page)
		offsets = make(map[string]int64)
		l.Info("Commit batch", esl.Strings("paths", paths))
		return nil
	}

	h := func(p Page) error {
		sessionMutex.Lock()
		if batchSize <= len(sessions) {
			if err := commit(); err != nil {
				return err
			}
		}
		sessionMutex.Unlock()

		type StartSessionParam struct {
			Close       bool   `json:"close"`
			SessionType string `json:"session_type,omitempty"`
		}
		type SessionData struct {
			SessionId string `path:"session_id" json:"session_id"`
		}

		content := pageContent(p)

		ssp := &StartSessionParam{
			Close:       true,
			SessionType: "sequential",
		}
		sessionRes := ctx.Upload("files/upload_session/start",
			api_request.Content(es_rewinder.NewReadRewinderOnMemory([]byte(content))),
			api_request.Param(&ssp))
		if err, f := sessionRes.Failure(); f {
			l.Debug("Unable to start the session", esl.Error(err))
			return err
		}
		sessionData := SessionData{}
		if err := sessionRes.Success().Json().Model(&sessionData); err != nil {
			l.Debug("Unable to parse session data", esl.Error(err))
			return err
		}
		if sessionData.SessionId == "" {
			l.Debug("Unable to retrieve session id")
			return errors.New("no session id found")
		}

		sessionMutex.Lock()
		sessions[sessionData.SessionId] = p
		offsets[sessionData.SessionId] = int64(len([]byte(content)))
		sessionMutex.Unlock()

		return nil
	}

	var wl = &WikimediaLoader{
		l:         c.Log(),
		skip:      z.Offset,
		batchSize: z.BatchSize.Value(),
	}
	sourcePath := z.Source.Path()

	sem := semaphore.NewWeighted(int64(c.Feature().Concurrency()))
	uploader := func(p Page) error {
		if err := sem.Acquire(context.TODO(), 1); err != nil {
			l.Debug("Unable to acquire semaphore", esl.Error(err))
			return err
		}
		var lastUploadErr error
		go func() {
			lastUploadErr = h(p)
			sem.Release(1)
		}()
		return lastUploadErr
	}

	var loadErr error
	switch {
	case strings.HasSuffix(sourcePath, ".xml.bz2"):
		loadErr = wl.LoadBz2(sourcePath, func(p Page) error {
			return uploader(p)
		})
	case strings.HasSuffix(sourcePath, ".xml"):
		loadErr = wl.LoadXml(sourcePath, func(p Page) error {
			return uploader(p)
		})
	default:
		panic(fmt.Sprintf("Look like the file is not supported format %s", sourcePath))
	}

	return es_lang.NewMultiErrorOrNull(commit(), loadErr)
}

func (z *Massfiles) Test(c app_control.Control) error {
	return qt_errors.ErrorNoTestRequired
}
<commit_msg>#659 : update to adhoc queuing<commit_after>package setup

import (
	"compress/bzip2"
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_async"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_util"
	"github.com/watermint/toolbox/domain/dropbox/filesystem/dbx_fs_copier_batch"
	mo_path2 "github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/essentials/api/api_request"
	"github.com/watermint/toolbox/essentials/go/es_lang"
	"github.com/watermint/toolbox/essentials/io/es_rewinder"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/essentials/model/mo_int"
	"github.com/watermint/toolbox/essentials/model/mo_path"
	"github.com/watermint/toolbox/essentials/time/ut_format"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"golang.org/x/sync/semaphore"
	"io"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

type Contributor struct {
	Username string `xml:"username"`
	Id       string `xml:"id"`
}

type Revision struct {
	Id          string      `xml:"id"`
	ParentId    string      `xml:"parentid"`
	Timestamp   string      `xml:"timestamp"`
	Contributor Contributor `xml:"contributor"`
	Comment     string      `xml:"comment"`
	Model       string      `xml:"model"`
	Format      string      `xml:"format"`
	Text        string      `xml:"text"`
	Sha1        string      `xml:"sha1"`
}

type Page struct {
	Title    string      `xml:"title"`
	Ns       string      `xml:"ns"`
	Id       string      `xml:"id"`
	Revision []*Revision `xml:"revision"`
}

type WikimediaLoader struct {
	l         esl.Logger
	skip      int
	batchSize int
}

func (z WikimediaLoader) LoadBz2(path string, handler func(p Page) error) error {
	f, err := os.Open(path)
	if err != nil {
		z.l.Warn("Can't open the file", esl.Error(err))
		return err
	}
	defer func() {
		_ = f.Close()
	}()

	bf := bzip2.NewReader(f)

	return z.load(bf, handler)
}

func (z WikimediaLoader) LoadXml(path string, handler func(p Page) error) error {
	f, err := os.Open(path)
	if err != nil {
		z.l.Warn("Can't open the file", esl.Error(err))
		return err
	}
	defer func() {
		_ = f.Close()
	}()
	return z.load(f, handler)
}

func (z WikimediaLoader) load(stream io.Reader, handler func(p Page) error) error {
	d := xml.NewDecoder(stream)
	index := 0
	lastMark := time.Now()
	firstMark := lastMark
	for {
		t, err := d.Token()
		if err != nil {
			z.l.Warn("cannot parse", esl.Error(err))
			return err
		}
		if t == nil {
			z.l.Debug("Reached to EOL")
			return nil
		}

		switch se := t.(type) {
		case xml.StartElement:
			e := se.Name.Local
			switch e {
			case "page":
				var page Page
				if err := d.DecodeElement(&page, &se); err != nil {
					z.l.Warn("Can't error", esl.Error(err))
					return err
				}
				index++
				if pageId, err := strconv.ParseInt(page.Id, 10, 64); err == nil {
					if pageId < int64(z.skip) {
						continue
					}
				}
				if index%z.batchSize == 0 {
					var estimatedThroughputSpan, estimatedThroughputTotal float64

					lastSpan := time.Now().Sub(lastMark).Seconds()
					if 0 < lastSpan {
						estimatedThroughputSpan = float64(z.batchSize) / lastSpan
					}
					totalSpan := time.Now().Sub(firstMark).Seconds()
					if 0 < totalSpan {
						estimatedThroughputTotal = float64(index) / totalSpan
					}
					z.l.Info("Loaded",
						esl.Time("time", time.Now()),
						esl.String("pageId", page.Id),
						esl.Int("index", index),
						esl.Float64("tps (span)", estimatedThroughputSpan),
						esl.Float64("tps (total)", estimatedThroughputTotal),
					)
					lastMark = time.Now()
				}
				if err := handler(page); err != nil {
					z.l.Warn("Can't handle the page", esl.Error(err), esl.Any("page", page))
					return err
				}
			}
		}
	}
}

type Massfiles struct {
	rc_recipe.RemarkSecret
	Peer              dbx_conn.ConnScopedIndividual
	Source            mo_path.ExistingFileSystemPath
	Base              mo_path2.DropboxPath
	Offset            int
	ShardSize         mo_int.RangeInt
	BatchSize         mo_int.RangeInt
	CommitConcurrency mo_int.RangeInt
}

func (z *Massfiles) Preset() {
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesContentRead,
		dbx_auth.ScopeFilesContentWrite,
	)
	z.BatchSize.SetRange(0, 1000, 1000)
	z.ShardSize.SetRange(1, 1000, 20)
	z.CommitConcurrency.SetRange(1, 10, 3)
}

func (z *Massfiles) Exec(c app_control.Control) error {
	l := c.Log()

	sessions := make(map[string]Page)
	offsets := make(map[string]int64)
	sessionMutex := sync.Mutex{}
	batchSize := z.BatchSize.Value()
	ctx := z.Peer.Client()

	pageContent := func(p Page) string {
		switch len(p.Revision) {
		case 0:
			return p.Title
		default:
			return fmt.Sprintf("= %s =\n\n----\n%s", p.Title, p.Revision[0].Text)
		}
	}

	pageTime := func(p Page) string {
		if len(p.Revision) < 1 {
			return dbx_util.ToApiTimeString(time.Now())
		} else {
			pt, valid := ut_format.ParseTimestamp(p.Revision[0].Timestamp)
			if !valid {
				return dbx_util.ToApiTimeString(time.Now())
			} else {
				return dbx_util.ToApiTimeString(pt)
			}
		}
	}

	pageToPath := func(p Page) []string {
		altPageId := "p-" + p.Id + ".txt"
		pageId, err := strconv.ParseInt(p.Id, 10, 32)
		if err != nil {
			l.Debug("Unable to parse pageId", esl.Error(err), esl.String("pageId", p.Id))
			return []string{"unexpected_page_id", altPageId}
		}
		altPageId = fmt.Sprintf("%d/%d/p-%d.txt", pageId/1_000_000, pageId/1000, pageId)
		if len(p.Revision) < 1 {
			return []string{"no_revision", altPageId}
		}
		pt, valid := ut_format.ParseTimestamp(p.Revision[0].Timestamp)
		if valid {
			return []string{
				fmt.Sprintf("%02d", pageId%z.ShardSize.Value64()),
				fmt.Sprintf("%04d", pt.Year()),
				fmt.Sprintf("%04d-%02d", pt.Year(), pt.Month()),
				fmt.Sprintf("%04d-%02d-%02d", pt.Year(), pt.Month(), pt.Day()),
				fmt.Sprintf("%s.txt", p.Id),
			}
		}
		return []string{"invalid_time_format", altPageId}
	}

	commitSemaphore := semaphore.NewWeighted(z.CommitConcurrency.Value64())
	commit := func() error {
		l.Info("Commit", esl.Int("size", len(sessions)))
		if len(sessions) < 1 {
			return nil
		}

		commits := make([]dbx_fs_copier_batch.UploadFinish, 0)
		paths := make([]string, 0)
		for sessionId, page := range sessions {
			path := z.Base.ChildPath(pageToPath(page)...).Path()
			offset := offsets[sessionId]
			paths = append(paths, path)
			commits = append(commits, dbx_fs_copier_batch.UploadFinish{
				Cursor: dbx_fs_copier_batch.UploadCursor{
					SessionId: sessionId,
					Offset:    offset,
				},
				Commit: dbx_fs_copier_batch.CommitInfo{
					Path:           path,
					Mode:           "add",
					Autorename:     true,
					ClientModified: pageTime(page),
					Mute:           true,
					StrictConflict: false,
				},
			})
		}

		// clean sessions
		sessions = make(map[string]Page)
		offsets = make(map[string]int64)

		if err := commitSemaphore.Acquire(context.TODO(), 1); err != nil {
			l.Debug("Unable to acquire commit semaphore", esl.Error(err))
			return err
		}
		go func() {
			l.Debug("Commit batch (start)", esl.Strings("paths", paths))
			finish := &dbx_fs_copier_batch.UploadFinishBatch{
				Entries: commits,
			}
			res := ctx.Async("files/upload_session/finish_batch", api_request.Param(finish)).Call(
				dbx_async.Status("files/upload_session/finish_batch/check"),
			)
			if err, f := res.Failure(); f {
				l.Warn("Unable to finish the batch", esl.Error(err), esl.Any("entries", commits))
			}
			l.Debug("Commit batch (completed)", esl.Strings("paths", paths))
			commitSemaphore.Release(1)
		}()

		return nil
	}

	h := func(p Page) error {
		sessionMutex.Lock()
		if batchSize <= len(sessions) {
			if err := commit(); err != nil {
				return err
			}
		}
		sessionMutex.Unlock()

		type StartSessionParam struct {
			Close       bool   `json:"close"`
			SessionType string `json:"session_type,omitempty"`
		}
		type SessionData struct {
			SessionId string `path:"session_id" json:"session_id"`
		}

		content := pageContent(p)

		ssp := &StartSessionParam{
			Close:       true,
			SessionType: "sequential",
		}
		sessionRes := ctx.Upload("files/upload_session/start",
			api_request.Content(es_rewinder.NewReadRewinderOnMemory([]byte(content))),
			api_request.Param(&ssp))
		if err, f := sessionRes.Failure(); f {
			l.Debug("Unable to start the session", esl.Error(err))
			return err
		}
		sessionData := SessionData{}
		if err := sessionRes.Success().Json().Model(&sessionData); err != nil {
			l.Debug("Unable to parse session data", esl.Error(err))
			return err
		}
		if sessionData.SessionId == "" {
			l.Debug("Unable to retrieve session id")
			return errors.New("no session id found")
		}

		sessionMutex.Lock()
		sessions[sessionData.SessionId] = p
		offsets[sessionData.SessionId] = int64(len([]byte(content)))
		sessionMutex.Unlock()

		return nil
	}

	var wl = &WikimediaLoader{
		l:         c.Log(),
		skip:      z.Offset,
		batchSize: z.BatchSize.Value(),
	}
	sourcePath := z.Source.Path()

	uploadSemaphore := semaphore.NewWeighted(int64(c.Feature().Concurrency()))
	uploader := func(p Page) error {
		if err := uploadSemaphore.Acquire(context.TODO(), 1); err != nil {
			l.Debug("Unable to acquire semaphore", esl.Error(err))
			return err
		}
		var lastUploadErr error
		go func() {
			lastUploadErr = h(p)
			uploadSemaphore.Release(1)
		}()
		return lastUploadErr
	}

	var loadErr error
	switch {
	case strings.HasSuffix(sourcePath, ".xml.bz2"):
		loadErr = wl.LoadBz2(sourcePath, func(p Page) error {
			return uploader(p)
		})
	case strings.HasSuffix(sourcePath, ".xml"):
		loadErr = wl.LoadXml(sourcePath, func(p Page) error {
			return uploader(p)
		})
	default:
		panic(fmt.Sprintf("Look like the file is not supported format %s", sourcePath))
	}

	return es_lang.NewMultiErrorOrNull(commit(), loadErr)
}

func (z *Massfiles) Test(c app_control.Control) error {
	return qt_errors.ErrorNoTestRequired
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codeskyblue/go-sh"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

var kind string
var mycommand []string

func main() {
	kind = "po"
	mycommand = []string{"kubectl", "exec", "-it"}

	if len(os.Args) > 1 {
		kind = os.Args[1]
		mycommand = []string{"gcloud", "compute", "ssh"}
		gcloudssh()
	} else {
		kushell()
	}
}

func kuget() ([]byte, error) {
	session := sh.NewSession()
	return session.Command("kubectl", "get", kind, "-o", "wide").
		Command("awk", `{ print $1 }`).
		Command("tail", "-n", "+2").
		Output()
}

func getUserChoice(output string) string {
	// split the result based on newline
	re := regexp.MustCompile(`\n`)
	result := re.Split(output, -1)

	for k, v := range result {
		fmt.Printf("%d: %s\n", k, v)
	}

	var chosen int
	fmt.Scanf("%d", &chosen)

	return result[chosen]
}

func kushell() {
	lines, err := kuget()

	if err != nil {
		fmt.Println("error: ", err)
		return
	}

	selected := getUserChoice(string(lines))
	mycommand := append(mycommand, selected, "--", "/bin/sh")

	fmt.Println(strings.Join(mycommand, " "))
	binary, _ := exec.LookPath("kubectl")
	syscall.Exec(
		binary,
		mycommand,
		os.Environ())
}

func gcloudssh() {
	lines, err := kuget()

	if err != nil {
		fmt.Println("error: ", err)
		return
	}

	selected := getUserChoice(string(lines))
	mycommand := append(mycommand, selected)

	fmt.Println(strings.Join(mycommand, " "))
	binary, _ := exec.LookPath("gcloud")
	syscall.Exec(
		binary,
		mycommand,
		os.Environ())

}
<commit_msg>added another alpine instance to use ash<commit_after>package main

import (
	"fmt"
	"github.com/codeskyblue/go-sh"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

var kind string
var mycommand []string

func main() {
	kind = "po"
	mycommand = []string{"kubectl", "exec", "-it"}

	if len(os.Args) > 1 {
		kind = os.Args[1]
		mycommand = []string{"gcloud", "compute", "ssh"}
		gcloudssh()
	} else {
		kushell()
	}
}

func kuget() ([]byte, error) {
	fmt.Println("Retriving kind: " + kind)
	session := sh.NewSession()
	return session.Command("kubectl", "get", kind, "-o", "wide").
		Command("awk", `{ print $1 }`).
		Command("tail", "-n", "+2").
		Output()
}

func getUserChoice(output string) string {
	// split the result based on newline
	re := regexp.MustCompile(`\n`)
	result := re.Split(output, -1)
	result = result[0 : len(result)-1]

	for k, v := range result {
		fmt.Printf("%d: %s\n", k, v)
	}

	fmt.Printf("\n\nChoose one number to ssh: ")
	var chosen int
	fmt.Scanf("%d", &chosen)

	return result[chosen]
}

// check if a particular container/pods come with bash shell
func isBashNotExist(mystr string) bool {
	containers := []string{"ecv-go", "ecv-storage"}

	for _, v := range containers {
		var re = regexp.MustCompile(v)
		if re.MatchString(mystr) {
			return true
		}
	}

	return false
}

func kushell() {
	lines, err := kuget()
	shelltype := "/bin/bash"

	if err != nil {
		fmt.Println("error: ", err)
		return
	}

	selected := getUserChoice(string(lines))
	if isBashNotExist(selected) {
		shelltype = "/bin/sh"
	}
	mycommand = append(mycommand, selected, "--", shelltype)

	fmt.Println(strings.Join(mycommand, " ") + "\n")
	binary, _ := exec.LookPath("kubectl")
	syscall.Exec(
		binary,
		mycommand,
		os.Environ())
}

func gcloudssh() {
	lines, err := kuget()

	if err != nil {
		fmt.Println("error: ", err)
		return
	}

	selected := getUserChoice(string(lines))
	mycommand = append(mycommand, selected)

	fmt.Println(strings.Join(mycommand, " "))
	binary, _ := exec.LookPath("gcloud")
	syscall.Exec(
		binary,
		mycommand,
		os.Environ())

}
<|endoftext|>


<commit_before>package proctl

import (
	"fmt"
	"os"
	"syscall"
)

type DebuggedProcess struct {
	Pid          int
	Regs         *syscall.PtraceRegs
	Process      *os.Process
	ProcessState *os.ProcessState
}

func NewDebugProcess(pid int) (*DebuggedProcess, error) {
	err := syscall.PtraceAttach(pid)
	if err != nil {
		return nil, err
	}

	proc, err := os.FindProcess(pid)
	if err != nil {
		return nil, err
	}

	ps, err := proc.Wait()
	if err != nil {
		return nil, err
	}

	debuggedProc := DebuggedProcess{
		Pid:          pid,
		Regs:         &syscall.PtraceRegs{},
		Process:      proc,
		ProcessState: ps,
	}

	return &debuggedProc, nil
}

func (dbp *DebuggedProcess) Registers() (*syscall.PtraceRegs, error) {
	err := syscall.PtraceGetRegs(dbp.Pid, dbp.Regs)
	if err != nil {
		return nil, fmt.Errorf("Registers():", err)
	}

	return dbp.Regs, nil
}

func (dbp *DebuggedProcess) Step() error {
	return dbp.Exec(func() error {
		return syscall.PtraceSingleStep(dbp.Pid)
	})
}

func (dbp *DebuggedProcess) Continue() error {
	return dbp.Exec(func() error {
		return syscall.PtraceCont(dbp.Pid, 0)
	})
}

func (dbp *DebuggedProcess) Exec(ptracefunc func() error) error {
	err := ptracefunc()
	if err != nil {
		return err
	}

	ps, err := dbp.Process.Wait()
	if err != nil {
		return err
	}

	dbp.ProcessState = ps

	return nil
}
<commit_msg>Take error directly in Exec()<commit_after>package proctl

import (
	"fmt"
	"os"
	"syscall"
)

type DebuggedProcess struct {
	Pid          int
	Regs         *syscall.PtraceRegs
	Process      *os.Process
	ProcessState *os.ProcessState
}

func NewDebugProcess(pid int) (*DebuggedProcess, error) {
	err := syscall.PtraceAttach(pid)
	if err != nil {
		return nil, err
	}

	proc, err := os.FindProcess(pid)
	if err != nil {
		return nil, err
	}

	ps, err := proc.Wait()
	if err != nil {
		return nil, err
	}

	debuggedProc := DebuggedProcess{
		Pid:          pid,
		Regs:         &syscall.PtraceRegs{},
		Process:      proc,
		ProcessState: ps,
	}

	return &debuggedProc, nil
}

func (dbp *DebuggedProcess) Registers() (*syscall.PtraceRegs, error) {
	err := syscall.PtraceGetRegs(dbp.Pid, dbp.Regs)
	if err != nil {
		return nil, fmt.Errorf("Registers():", err)
	}

	return dbp.Regs, nil
}

func (dbp *DebuggedProcess) Step() error {
	return dbp.Exec(syscall.PtraceSingleStep(dbp.Pid))
}

func (dbp *DebuggedProcess) Continue() error {
	return dbp.Exec(syscall.PtraceCont(dbp.Pid, 0))
}

func (dbp *DebuggedProcess) Exec(err error) error {
	if err != nil {
		return err
	}

	ps, err := dbp.Process.Wait()
	if err != nil {
		return err
	}

	dbp.ProcessState = ps

	return nil
}
<|endoftext|>