<commit_before>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package http

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"strings"

	driver "github.com/arangodb/go-driver"
	"github.com/arangodb/go-driver/cluster"
	"github.com/arangodb/go-driver/util"
	velocypack "github.com/arangodb/go-velocypack"
)

const (
	keyRawResponse = "arangodb-rawResponse"
	keyResponse    = "arangodb-response"
)

// ConnectionConfig provides all configuration options for a HTTP connection.
type ConnectionConfig struct {
	// Endpoints holds 1 or more URL's used to connect to the database.
	// In case of a connection to an ArangoDB cluster, you must provide the URL's of all coordinators.
	Endpoints []string
	// TLSConfig holds settings used to configure a TLS (HTTPS) connection.
	// This is only used for endpoints using the HTTPS scheme.
	TLSConfig *tls.Config
	// Transport allows the use of a custom round tripper.
	// If Transport is not of type `*http.Transport`, the `TLSConfig` property is not used.
	// Otherwise a `TLSConfig` property other than `nil` will overwrite the `TLSClientConfig`
	// property of `Transport`.
	Transport http.RoundTripper
	// Cluster configuration settings
	cluster.ConnectionConfig
	// ContentType specified type of content encoding to use.
	ContentType driver.ContentType
}

// NewConnection creates a new HTTP connection based on the given configuration settings.
func NewConnection(config ConnectionConfig) (driver.Connection, error) {
	c, err := cluster.NewConnection(config.ConnectionConfig, func(endpoint string) (driver.Connection, error) {
		conn, err := newHTTPConnection(endpoint, config)
		if err != nil {
			return nil, driver.WithStack(err)
		}
		return conn, nil
	}, config.Endpoints)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	return c, nil
}

// newHTTPConnection creates a new HTTP connection for a single endpoint and the remainder of the given configuration settings.
func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connection, error) {
	endpoint = util.FixupEndpointURLScheme(endpoint)
	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	var httpTransport *http.Transport
	if config.Transport != nil {
		httpTransport, _ = config.Transport.(*http.Transport)
	} else {
		httpTransport = &http.Transport{}
		config.Transport = httpTransport
	}
	if config.TLSConfig != nil && httpTransport != nil {
		httpTransport.TLSClientConfig = config.TLSConfig
	}
	c := &httpConnection{
		endpoint:    *u,
		contentType: config.ContentType,
		client: &http.Client{
			Transport: config.Transport,
		},
	}
	return c, nil
}

// httpConnection implements an HTTP + JSON connection to an arangodb server.
type httpConnection struct {
	endpoint    url.URL
	contentType driver.ContentType
	client      *http.Client
}

// String returns the endpoint as string
func (c *httpConnection) String() string {
	return c.endpoint.String()
}

// NewRequest creates a new request with given method and path.
func (c *httpConnection) NewRequest(method, path string) (driver.Request, error) {
	switch method {
	case "GET", "POST", "DELETE", "HEAD", "PATCH", "PUT", "OPTIONS":
	// Ok
	default:
		return nil, driver.WithStack(driver.InvalidArgumentError{Message: fmt.Sprintf("Invalid method '%s'", method)})
	}
	ct := c.contentType
	if ct != driver.ContentTypeJSON && strings.Contains(path, "_api/gharial") {
		// Currently (3.1.18) calls to this API do not work well with vpack.
		ct = driver.ContentTypeJSON
	}
	switch ct {
	case driver.ContentTypeJSON:
		r := &httpJSONRequest{
			method: method,
			path:   path,
		}
		return r, nil
	case driver.ContentTypeVelocypack:
		r := &httpVPackRequest{
			method: method,
			path:   path,
		}
		return r, nil
	default:
		return nil, driver.WithStack(fmt.Errorf("Unsupported content type %d", int(c.contentType)))
	}
}

// Do performs a given request, returning its response.
func (c *httpConnection) Do(ctx context.Context, req driver.Request) (driver.Response, error) {
	httpReq, ok := req.(httpRequest)
	if !ok {
		return nil, driver.WithStack(driver.InvalidArgumentError{Message: "request is not a httpRequest"})
	}
	r, err := httpReq.createHTTPRequest(c.endpoint)
	rctx := ctx
	if rctx == nil {
		rctx = context.Background()
	}
	rctx = httptrace.WithClientTrace(rctx, &httptrace.ClientTrace{
		WroteRequest: func(info httptrace.WroteRequestInfo) {
			httpReq.WroteRequest(info)
		},
	})
	r = r.WithContext(rctx)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	resp, err := c.client.Do(r)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	var rawResponse *[]byte
	if ctx != nil {
		if v := ctx.Value(keyRawResponse); v != nil {
			if buf, ok := v.(*[]byte); ok {
				rawResponse = buf
			}
		}
	}

	// Read response body
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	if rawResponse != nil {
		*rawResponse = body
	}

	ct := resp.Header.Get("Content-Type")
	var httpResp driver.Response
	switch strings.Split(ct, ";")[0] {
	case "application/json":
		httpResp = &httpJSONResponse{resp: resp, rawResponse: body}
	case "application/x-velocypack":
		httpResp = &httpVPackResponse{resp: resp, rawResponse: body}
	default:
		if resp.StatusCode == http.StatusUnauthorized {
			// When unauthorized the server sometimes return a `text/plain` response.
			return nil, driver.WithStack(driver.ArangoError{
				HasError:     true,
				Code:         resp.StatusCode,
				ErrorMessage: string(body),
			})
		}
		return nil, driver.WithStack(fmt.Errorf("Unsupported content type: %s", ct))
	}
	if ctx != nil {
		if v := ctx.Value(keyResponse); v != nil {
			if respPtr, ok := v.(*driver.Response); ok {
				*respPtr = httpResp
			}
		}
	}
	return httpResp, nil
}

// Unmarshal unmarshals the given raw object into the given result interface.
func (c *httpConnection) Unmarshal(data driver.RawObject, result interface{}) error {
	ct := c.contentType
	if ct == driver.ContentTypeVelocypack && len(data) >= 2 {
		// Poor mans auto detection of json
		l := len(data)
		if (data[0] == '{' && data[l-1] == '}') || (data[0] == '[' && data[l-1] == ']') {
			ct = driver.ContentTypeJSON
		}
	}
	switch ct {
	case driver.ContentTypeJSON:
		if err := json.Unmarshal(data, result); err != nil {
			return driver.WithStack(err)
		}
	case driver.ContentTypeVelocypack:
		//panic(velocypack.Slice(data))
		if err := velocypack.Unmarshal(velocypack.Slice(data), result); err != nil {
			return driver.WithStack(err)
		}
	default:
		return driver.WithStack(fmt.Errorf("Unsupported content type %d", int(c.contentType)))
	}
	return nil
}

// Endpoints returns the endpoints used by this connection.
func (c *httpConnection) Endpoints() []string {
	return []string{c.endpoint.String()}
}

// UpdateEndpoints reconfigures the connection to use the given endpoints.
func (c *httpConnection) UpdateEndpoints(endpoints []string) error {
	// Do nothing here.
	// The real updating is done in cluster Connection.
	return nil
}

// Configure the authentication used for this connection.
func (c *httpConnection) SetAuthentication(auth driver.Authentication) (driver.Connection, error) {
	var httpAuth httpAuthentication
	switch auth.Type() {
	case driver.AuthenticationTypeBasic:
		userName := auth.Get("username")
		password := auth.Get("password")
		httpAuth = newBasicAuthentication(userName, password)
	case driver.AuthenticationTypeJWT:
		userName := auth.Get("username")
		password := auth.Get("password")
		httpAuth = newJWTAuthentication(userName, password)
	default:
		return nil, driver.WithStack(fmt.Errorf("Unsupported authentication type %d", int(auth.Type())))
	}

	result, err := newAuthenticatedConnection(c, httpAuth)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	return result, nil
}

// Protocols returns all protocols used by this connection.
func (c *httpConnection) Protocols() driver.ProtocolSet {
	return driver.ProtocolSet{driver.ProtocolHTTP}
}
<commit_msg>Added status code in case on unknown content type<commit_after>//
// DISCLAIMER
//
// Copyright 2017 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package http

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"strings"

	driver "github.com/arangodb/go-driver"
	"github.com/arangodb/go-driver/cluster"
	"github.com/arangodb/go-driver/util"
	velocypack "github.com/arangodb/go-velocypack"
)

const (
	keyRawResponse = "arangodb-rawResponse"
	keyResponse    = "arangodb-response"
)

// ConnectionConfig provides all configuration options for a HTTP connection.
type ConnectionConfig struct {
	// Endpoints holds 1 or more URL's used to connect to the database.
	// In case of a connection to an ArangoDB cluster, you must provide the URL's of all coordinators.
	Endpoints []string
	// TLSConfig holds settings used to configure a TLS (HTTPS) connection.
	// This is only used for endpoints using the HTTPS scheme.
	TLSConfig *tls.Config
	// Transport allows the use of a custom round tripper.
	// If Transport is not of type `*http.Transport`, the `TLSConfig` property is not used.
	// Otherwise a `TLSConfig` property other than `nil` will overwrite the `TLSClientConfig`
	// property of `Transport`.
	Transport http.RoundTripper
	// Cluster configuration settings
	cluster.ConnectionConfig
	// ContentType specified type of content encoding to use.
	ContentType driver.ContentType
}

// NewConnection creates a new HTTP connection based on the given configuration settings.
func NewConnection(config ConnectionConfig) (driver.Connection, error) {
	c, err := cluster.NewConnection(config.ConnectionConfig, func(endpoint string) (driver.Connection, error) {
		conn, err := newHTTPConnection(endpoint, config)
		if err != nil {
			return nil, driver.WithStack(err)
		}
		return conn, nil
	}, config.Endpoints)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	return c, nil
}

// newHTTPConnection creates a new HTTP connection for a single endpoint and the remainder of the given configuration settings.
func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connection, error) {
	endpoint = util.FixupEndpointURLScheme(endpoint)
	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	var httpTransport *http.Transport
	if config.Transport != nil {
		httpTransport, _ = config.Transport.(*http.Transport)
	} else {
		httpTransport = &http.Transport{}
		config.Transport = httpTransport
	}
	if config.TLSConfig != nil && httpTransport != nil {
		httpTransport.TLSClientConfig = config.TLSConfig
	}
	c := &httpConnection{
		endpoint:    *u,
		contentType: config.ContentType,
		client: &http.Client{
			Transport: config.Transport,
		},
	}
	return c, nil
}

// httpConnection implements an HTTP + JSON connection to an arangodb server.
type httpConnection struct {
	endpoint    url.URL
	contentType driver.ContentType
	client      *http.Client
}

// String returns the endpoint as string
func (c *httpConnection) String() string {
	return c.endpoint.String()
}

// NewRequest creates a new request with given method and path.
func (c *httpConnection) NewRequest(method, path string) (driver.Request, error) {
	switch method {
	case "GET", "POST", "DELETE", "HEAD", "PATCH", "PUT", "OPTIONS":
	// Ok
	default:
		return nil, driver.WithStack(driver.InvalidArgumentError{Message: fmt.Sprintf("Invalid method '%s'", method)})
	}
	ct := c.contentType
	if ct != driver.ContentTypeJSON && strings.Contains(path, "_api/gharial") {
		// Currently (3.1.18) calls to this API do not work well with vpack.
		ct = driver.ContentTypeJSON
	}
	switch ct {
	case driver.ContentTypeJSON:
		r := &httpJSONRequest{
			method: method,
			path:   path,
		}
		return r, nil
	case driver.ContentTypeVelocypack:
		r := &httpVPackRequest{
			method: method,
			path:   path,
		}
		return r, nil
	default:
		return nil, driver.WithStack(fmt.Errorf("Unsupported content type %d", int(c.contentType)))
	}
}

// Do performs a given request, returning its response.
func (c *httpConnection) Do(ctx context.Context, req driver.Request) (driver.Response, error) {
	httpReq, ok := req.(httpRequest)
	if !ok {
		return nil, driver.WithStack(driver.InvalidArgumentError{Message: "request is not a httpRequest"})
	}
	r, err := httpReq.createHTTPRequest(c.endpoint)
	rctx := ctx
	if rctx == nil {
		rctx = context.Background()
	}
	rctx = httptrace.WithClientTrace(rctx, &httptrace.ClientTrace{
		WroteRequest: func(info httptrace.WroteRequestInfo) {
			httpReq.WroteRequest(info)
		},
	})
	r = r.WithContext(rctx)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	resp, err := c.client.Do(r)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	var rawResponse *[]byte
	if ctx != nil {
		if v := ctx.Value(keyRawResponse); v != nil {
			if buf, ok := v.(*[]byte); ok {
				rawResponse = buf
			}
		}
	}

	// Read response body
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	if rawResponse != nil {
		*rawResponse = body
	}

	ct := resp.Header.Get("Content-Type")
	var httpResp driver.Response
	switch strings.Split(ct, ";")[0] {
	case "application/json":
		httpResp = &httpJSONResponse{resp: resp, rawResponse: body}
	case "application/x-velocypack":
		httpResp = &httpVPackResponse{resp: resp, rawResponse: body}
	default:
		if resp.StatusCode == http.StatusUnauthorized {
			// When unauthorized the server sometimes return a `text/plain` response.
			return nil, driver.WithStack(driver.ArangoError{
				HasError:     true,
				Code:         resp.StatusCode,
				ErrorMessage: string(body),
			})
		}
		return nil, driver.WithStack(fmt.Errorf("Unsupported content type '%s' with status %d", ct, resp.StatusCode))
	}
	if ctx != nil {
		if v := ctx.Value(keyResponse); v != nil {
			if respPtr, ok := v.(*driver.Response); ok {
				*respPtr = httpResp
			}
		}
	}
	return httpResp, nil
}

// Unmarshal unmarshals the given raw object into the given result interface.
func (c *httpConnection) Unmarshal(data driver.RawObject, result interface{}) error {
	ct := c.contentType
	if ct == driver.ContentTypeVelocypack && len(data) >= 2 {
		// Poor mans auto detection of json
		l := len(data)
		if (data[0] == '{' && data[l-1] == '}') || (data[0] == '[' && data[l-1] == ']') {
			ct = driver.ContentTypeJSON
		}
	}
	switch ct {
	case driver.ContentTypeJSON:
		if err := json.Unmarshal(data, result); err != nil {
			return driver.WithStack(err)
		}
	case driver.ContentTypeVelocypack:
		//panic(velocypack.Slice(data))
		if err := velocypack.Unmarshal(velocypack.Slice(data), result); err != nil {
			return driver.WithStack(err)
		}
	default:
		return driver.WithStack(fmt.Errorf("Unsupported content type %d", int(c.contentType)))
	}
	return nil
}

// Endpoints returns the endpoints used by this connection.
func (c *httpConnection) Endpoints() []string {
	return []string{c.endpoint.String()}
}

// UpdateEndpoints reconfigures the connection to use the given endpoints.
func (c *httpConnection) UpdateEndpoints(endpoints []string) error {
	// Do nothing here.
	// The real updating is done in cluster Connection.
	return nil
}

// Configure the authentication used for this connection.
func (c *httpConnection) SetAuthentication(auth driver.Authentication) (driver.Connection, error) {
	var httpAuth httpAuthentication
	switch auth.Type() {
	case driver.AuthenticationTypeBasic:
		userName := auth.Get("username")
		password := auth.Get("password")
		httpAuth = newBasicAuthentication(userName, password)
	case driver.AuthenticationTypeJWT:
		userName := auth.Get("username")
		password := auth.Get("password")
		httpAuth = newJWTAuthentication(userName, password)
	default:
		return nil, driver.WithStack(fmt.Errorf("Unsupported authentication type %d", int(auth.Type())))
	}

	result, err := newAuthenticatedConnection(c, httpAuth)
	if err != nil {
		return nil, driver.WithStack(err)
	}
	return result, nil
}

// Protocols returns all protocols used by this connection.
func (c *httpConnection) Protocols() driver.ProtocolSet {
	return driver.ProtocolSet{driver.ProtocolHTTP}
}
<|endoftext|>


<commit_before>package xmlwriter_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"
)

func TestNoDeps(t *testing.T) {
	if os.Getenv("XMLWRITER_SKIP_MOD") != "" {
		// Use this to avoid this check if you need to use spew.Dump in tests:
		t.Skip()
	}

	fix, err := ioutil.ReadFile("go.mod.fix")
	if err != nil {
		panic(err)
	}

	bts, err := ioutil.ReadFile("go.mod")
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fix, bts) {
		t.Fatal("go.mod contains unexpected content:\n" + string(bts))
	}
}
<commit_msg>Fix line endings<commit_after>package xmlwriter_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"testing"
)

func TestNoDeps(t *testing.T) {
	if os.Getenv("XMLWRITER_SKIP_MOD") != "" {
		// Use this to avoid this check if you need to use spew.Dump in tests:
		t.Skip()
	}

	fix, err := ioutil.ReadFile("go.mod.fix")
	if err != nil {
		panic(err)
	}
	bts, err := ioutil.ReadFile("go.mod")
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fixNL(fix), fixNL(bts)) {
		t.Fatal("go.mod contains unexpected content:\n" + string(bts))
	}
}

func fixNL(d []byte) []byte {
	d = bytes.Replace(d, []byte{13, 10}, []byte{10}, -1)
	d = bytes.Replace(d, []byte{13}, []byte{10}, -1)
	return d
}
<|endoftext|>


<commit_before>package main

import (
	facebook "github.com/Agon/go-facebook/facebook"
	"fmt"
)

func main() {
	resp, err := facebook.Call("platform", map[string]string{})
	if err != nil {
		fmt.Println(err.String())
		return
	}
	platform := resp.Data.(map[string]interface{})
	info := "Name: " + platform["name"].(string) + "\n"
	fmt.Print(info)
}
<commit_msg>Update printplatform example to latest changes.<commit_after>package main

import (
	facebook "github.com/Agon/go-facebook/facebook"
	"fmt"
)

func main() {
	platform, err := facebook.GetPage("platform")
	if err != nil {
		fmt.Println(err.String())
		return
	}
	fmt.Printf("Name: %s\n", platform.Name)
}
<|endoftext|>


<commit_before>package goalone

import (
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"errors"
	"hash"
	"sync"
	"time"
)

// Options that can be configured and passed to New()
type Options struct {
	// hash algorithm to use when signing tokens, ex. sha1.New
	Algorithm func() hash.Hash

	// Epoch to use for Timestamps, when signing/parsing Tokens
	// Number of seconds since January 1, 1970 UTC
	Epoch int64

	// Should the Sign method add a timestamp to tokens?
	Timestamp bool
}

// Sword is a magical Wooden Sword to be used for protection, because it's dangerous out
// there... Also, it is the main struct used to sign and unsign data using this
// package.
type Sword struct {
	sync.Mutex
	hash  hash.Hash
	dirty bool

	Options
}

// ErrInvalidSignature is returned by Unsign when the provided token's
// signatuire is not valid.
var ErrInvalidSignature = errors.New("invalid signature")

// ErrShortToken is returned by Unsign when the provided token's length
// is too short to be a vlaid token.
var ErrShortToken = errors.New("token is too small to be valid")

// New takes a secret key and returns a new Sword.  If no Options are provided
// then minimal defaults will be used.
func New(key []byte, o *Options) *Sword {

	// Create a map for decoding Base58.  This speeds up the process tremendously.
	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[i] = 0xFF
	}
	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[encodeBase58Map[i]] = byte(i)
	}

	if key == nil {
		return &Sword{}
	}

	if o == nil {
		return &Sword{hash: hmac.New(sha1.New, key)}
	}

	s := &Sword{Options: *o}
	if s.Algorithm == nil {
		s.hash = hmac.New(sha1.New, key)
	} else {
		s.hash = hmac.New(s.Algorithm, key)
	}

	return s
}

// Sign signs data and returns []byte in the format `data.signature`. Optionally
// add a timestamp and return in the format `data.timestamp.signature`
func (s *Sword) Sign(data []byte) []byte {

	// Build the payload
	el := base64.RawURLEncoding.EncodedLen(s.hash.Size())
	var t []byte

	if s.Timestamp {
		ts := time.Now().Unix() - s.Epoch
		etl := encodeBase58Len(ts)
		t = make([]byte, 0, len(data)+etl+el+2) // +2 for "." chars
		t = append(t, data...)
		t = append(t, '.')
		t = t[0 : len(t)+etl] // expand for timestamp
		encodeBase58(ts, t)
	} else {
		t = make([]byte, 0, len(data)+el+1)
		t = append(t, data...)
	}

	// Append and encode signature to token
	t = append(t, '.')
	tl := len(t)
	t = t[0 : tl+el]

	// Add the signature to the token
	s.sign(t[tl:], t[0:tl-1])

	// Return the token to the caller
	return t
}

// Unsign validates a signature and if successful returns the data portion of
// the []byte. If unsuccessful it will return an error and nil for the data.
func (s *Sword) Unsign(token []byte) ([]byte, error) {

	tl := len(token)
	el := base64.RawURLEncoding.EncodedLen(s.hash.Size())

	// A token must be at least el+2 bytes long to be valid.
	if tl < el+2 {
		return nil, ErrShortToken
	}

	// Get the signature of the payload
	dst := make([]byte, el)
	s.sign(dst, token[0:tl-(el+1)])

	if subtle.ConstantTimeCompare(token[tl-el:], dst) != 1 {
		return nil, ErrInvalidSignature
	}

	return token[0 : tl-(el+1)], nil
}

///////////////////////////////////////////////////////////////////////////////
// Unexported Code ////////////////////////////////////////////////////////////

// This is the map of characters used during base58 encoding.  These replicate
// the flickr shortid mapping.
const encodeBase58Map = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"

// Used to create a decode map so we can decode base58 fairly fast.
var decodeBase58Map [256]byte

// sign creates the encoded signature of payload and writes to dst
func (s *Sword) sign(dst, payload []byte) {

	s.Lock()
	if s.dirty {
		s.hash.Reset()
	}
	s.dirty = true
	s.hash.Write(payload)
	h := s.hash.Sum(nil)
	s.Unlock()

	base64.RawURLEncoding.Encode(dst, h)
}

func encodeBase58Len(i int64) int {

	var l = 1
	for i >= 58 {
		l++
		i /= 58
	}
	return l
}

// encode time int64 into b []byte
func encodeBase58(i int64, b []byte) {
	p := len(b) - 1
	for i >= 58 {
		b[p] = encodeBase58Map[i%58]
		p--
		i /= 58
	}
	b[p] = encodeBase58Map[i]
}

// parses a base58 []byte into a int64
func decodeBase58(b []byte) int64 {
	var id int64
	for p := range b {
		id = id*58 + int64(decodeBase58Map[b[p]])
	}
	return id
}
<commit_msg>Remove deadcode.<commit_after>package goalone

import (
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"errors"
	"hash"
	"sync"
	"time"
)

// Options that can be configured and passed to New()
type Options struct {
	// hash algorithm to use when signing tokens, ex. sha1.New
	Algorithm func() hash.Hash

	// Epoch to use for Timestamps, when signing/parsing Tokens
	// Number of seconds since January 1, 1970 UTC
	Epoch int64

	// Should the Sign method add a timestamp to tokens?
	Timestamp bool
}

// Sword is a magical Wooden Sword to be used for protection, because it's dangerous out
// there... Also, it is the main struct used to sign and unsign data using this
// package.
type Sword struct {
	sync.Mutex
	hash  hash.Hash
	dirty bool

	Options
}

// ErrInvalidSignature is returned by Unsign when the provided token's
// signatuire is not valid.
var ErrInvalidSignature = errors.New("invalid signature")

// ErrShortToken is returned by Unsign when the provided token's length
// is too short to be a vlaid token.
var ErrShortToken = errors.New("token is too small to be valid")

// New takes a secret key and returns a new Sword.  If no Options are provided
// then minimal defaults will be used.
func New(key []byte, o *Options) *Sword {

	// Create a map for decoding Base58.  This speeds up the process tremendously.
	for i := 0; i < len(encodeBase58Map); i++ {
		decodeBase58Map[encodeBase58Map[i]] = byte(i)
	}

	if key == nil {
		return &Sword{}
	}

	if o == nil {
		return &Sword{hash: hmac.New(sha1.New, key)}
	}

	s := &Sword{Options: *o}
	if s.Algorithm == nil {
		s.hash = hmac.New(sha1.New, key)
	} else {
		s.hash = hmac.New(s.Algorithm, key)
	}

	return s
}

// Sign signs data and returns []byte in the format `data.signature`. Optionally
// add a timestamp and return in the format `data.timestamp.signature`
func (s *Sword) Sign(data []byte) []byte {

	// Build the payload
	el := base64.RawURLEncoding.EncodedLen(s.hash.Size())
	var t []byte

	if s.Timestamp {
		ts := time.Now().Unix() - s.Epoch
		etl := encodeBase58Len(ts)
		t = make([]byte, 0, len(data)+etl+el+2) // +2 for "." chars
		t = append(t, data...)
		t = append(t, '.')
		t = t[0 : len(t)+etl] // expand for timestamp
		encodeBase58(ts, t)
	} else {
		t = make([]byte, 0, len(data)+el+1)
		t = append(t, data...)
	}

	// Append and encode signature to token
	t = append(t, '.')
	tl := len(t)
	t = t[0 : tl+el]

	// Add the signature to the token
	s.sign(t[tl:], t[0:tl-1])

	// Return the token to the caller
	return t
}

// Unsign validates a signature and if successful returns the data portion of
// the []byte. If unsuccessful it will return an error and nil for the data.
func (s *Sword) Unsign(token []byte) ([]byte, error) {

	tl := len(token)
	el := base64.RawURLEncoding.EncodedLen(s.hash.Size())

	// A token must be at least el+2 bytes long to be valid.
	if tl < el+2 {
		return nil, ErrShortToken
	}

	// Get the signature of the payload
	dst := make([]byte, el)
	s.sign(dst, token[0:tl-(el+1)])

	if subtle.ConstantTimeCompare(token[tl-el:], dst) != 1 {
		return nil, ErrInvalidSignature
	}

	return token[0 : tl-(el+1)], nil
}

///////////////////////////////////////////////////////////////////////////////
// Unexported Code ////////////////////////////////////////////////////////////

// This is the map of characters used during base58 encoding.  These replicate
// the flickr shortid mapping.
const encodeBase58Map = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"

// Used to create a decode map so we can decode base58 fairly fast.
var decodeBase58Map [256]byte

// sign creates the encoded signature of payload and writes to dst
func (s *Sword) sign(dst, payload []byte) {

	s.Lock()
	if s.dirty {
		s.hash.Reset()
	}
	s.dirty = true
	s.hash.Write(payload)
	h := s.hash.Sum(nil)
	s.Unlock()

	base64.RawURLEncoding.Encode(dst, h)
}

func encodeBase58Len(i int64) int {

	var l = 1
	for i >= 58 {
		l++
		i /= 58
	}
	return l
}

// encode time int64 into b []byte
func encodeBase58(i int64, b []byte) {
	p := len(b) - 1
	for i >= 58 {
		b[p] = encodeBase58Map[i%58]
		p--
		i /= 58
	}
	b[p] = encodeBase58Map[i]
}

// parses a base58 []byte into a int64
func decodeBase58(b []byte) int64 {
	var id int64
	for p := range b {
		id = id*58 + int64(decodeBase58Map[b[p]])
	}
	return id
}
<|endoftext|>


<commit_before>/*
Gotriki server of the Trikipedia - the truth encyclopedia.
*/
package main // import "gopkg.in/triki.v0"

import (
	"net/http"
	"os"
	"os/signal"

	"github.com/gorilla/mux"
	"gopkg.in/kornel661/nserv.v0"
	"gopkg.in/triki.v0/conf"
	"gopkg.in/triki.v0/db"
	"gopkg.in/triki.v0/log"
)

const (
	staticPrefix = "/static"
	apiPrefix    = "/api/"
)

func main() {
	// panic trap (for fatal errors from log)
	defer func() {
		if r := recover(); r != nil {
			if fatal, ok := r.(log.FatalErrorPanic); ok {
				log.Infof("Panic: %s.\n", fatal)
			} else {
				panic(r)
			}
		}
	}()
	conf.Setup()

	// catch signals & shutdown the server
	server := nserv.Server{}
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, os.Kill)
	// "reroute" signals channel to server.Shutdown
	go func() {
		<-signals
		log.Infoln("Caught signal. Exiting (waiting for open connections).")
		server.Stop()
	}()

	// setup database connections, etc.
	db.Setup()
	defer db.Cleanup()

	// setup routing
	r := mux.NewRouter()
	apiRouter := r.PathPrefix(apiPrefix).Subrouter()
	routeAPI(apiRouter)

	// serve static content from staticPrefix and "/"
	r.PathPrefix(staticPrefix).Handler(http.FileServer(http.Dir(conf.Server.Root)))
	r.PathPrefix("/").Handler(http.FileServer(http.Dir(conf.Server.Root + staticPrefix)))

	// start server
	log.Infof("Serving triki via www: http://%s\n", conf.Server.Addr)
	server.Addr = conf.Server.Addr
	server.Handler = r
	if err := server.ListenAndServe(); err != nil {
		log.Fatal(err)
	}
	log.Infoln("Exiting gracefully...")
}
<commit_msg>move gotriki.go -> main.go<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"gopkg.in/fsnotify.v1"
)

var (
	watchedRun = ""
	watchedFmt = "."
	noRun      = flag.Bool("n", false, "only run gofmt")
	inFile     = flag.String("in", "", "input file")

	delay      = flag.Int("d", 1, "delay time before detecting file change")
	isPipe     = false
	exitCode   = 0
	lastReport = make(map[string]time.Time)
)

func getPackageNameAndImport(sourceName string) (packageName string, imports []string) {
	fset := token.NewFileSet() // positions are relative to fset

	// parse the file containing this very example
	// but stop after processing the imports.
	f, err := parser.ParseFile(fset, sourceName, nil, parser.ImportsOnly)
	if err != nil {
		fmt.Println(err)
		return
	}

	// print the imports from the file's AST.
	packageName = f.Name.Name

	for _, s := range f.Imports {
		importedPackageName := s.Path.Value[1 : len(s.Path.Value)-1]
		imports = append(imports, importedPackageName)
	}

	return
}

func isGoFile(f os.FileInfo) bool {
	// ignore non-Go files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}

func isMainPackage(sourceName string) bool {
	fset := token.NewFileSet()
	f, _ := parser.ParseFile(fset, sourceName, nil, parser.ImportsOnly)

	return f.Name.Name == "main"
}

func isMainFile(sourceName string) bool {
	fset := token.NewFileSet()

	f, err := parser.ParseFile(fset, sourceName, nil, 0)
	if err != nil {
		return false
	}

	if o := f.Scope.Lookup("main"); f.Name.Name == "main" && o != nil && o.Kind == ast.Fun {
		return true
	}

	return false
}

func visitFile(path string, info os.FileInfo, err error) error {
	log.Println(info.Name())
	if isGoFile(info) {
		format(path)
		if isMainFile(path) {
			fmt.Println("main file -", path)
		}
	}
	return nil
}

func format(path string) error {
	log.Println("gofmt -w", path)
	command := exec.Command("gofmt", "-w", path)
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	command.Run()
	return nil
}

func goFiles(path string) (goFiles, mainFiles []string) {
	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if isGoFile(info) {
			goFiles = append(goFiles, path)
			if isMainFile(path) {
				mainFiles = append(mainFiles, path)
			}
		}
		return nil
	})
	return
}

var runCmd *exec.Cmd

func goRun(sourceName string) {
	if runCmd != nil && runCmd.Process != nil {
		log.Println("kill process", runCmd.Process.Pid)
		// to properly kill go run and its children
		// we need to set runCmd.SysProcAttr.Setid to true
		// and send kill signal to the process group
		// with negative PID
		syscall.Kill(-runCmd.Process.Pid, syscall.SIGKILL)
		runCmd.Process.Wait()
	}

	runCmd = exec.Command("go", "run", sourceName)

	func(runCmd *exec.Cmd) {
		if len(*inFile) > 0 {
			f, err := os.Open(*inFile)
			if err != nil {
				runCmd.Stdin = os.Stdin
			} else {
				runCmd.Stdin = f
				defer f.Close()
			}
		}
		runCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
		runCmd.Stdout = os.Stdout
		runCmd.Stderr = os.Stderr
		runCmd.Start()
		log.Println("go run", watchedRun, "("+strconv.Itoa(runCmd.Process.Pid)+")")
		runCmd.Wait()
	}(runCmd)
}

func cleanUp() {
	if runCmd != nil && runCmd.Process != nil {
		syscall.Kill(-runCmd.Process.Pid, syscall.SIGKILL)
		runCmd.Process.Wait()
	}
}

func main() {
	gowatchMain()
	os.Exit(exitCode)
}

func gowatchMain() {
	flag.Parse()

	path, err := filepath.Abs(watchedFmt)
	if err != nil {
		panic(err)
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		panic(err)
	}

	err = watcher.Add(path)
	if err != nil {
		panic(err)
	}
	defer watcher.Close()

	goFiles, mainFiles := goFiles(path)

	// pre run gofmt on found Go files
	for _, f := range goFiles {
		format(f)
	}

	if len(mainFiles) == 1 {
		watchedRun = mainFiles[0]
		log.Println("found a main Go file, watch and run", mainFiles[0])
	} else if len(mainFiles) > 1 {
		watchedRun = mainFiles[0]
		log.Println("found more than one main Go files, watch and run", mainFiles[0])
	} else {
		watchedRun = ""
		log.Println("main Go files not found")
	}

	if len(watchedRun) > 0 && !*noRun {
		go goRun(watchedRun)
        defer cleanUp()
	}

	log.Println("watching", path)

	fi, _ := os.Stdout.Stat()
	isPipe = fi.Mode()&os.ModeNamedPipe != 0

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt, os.Kill)

	for {
		select {
		case <-sig:
			return
		case event, more := <-watcher.Events:
			if more {
				if event.Op == fsnotify.Create || event.Op == fsnotify.Write {
					if time.Since(lastReport[event.Name]) < time.Duration(*delay)*time.Second {
						continue
					}
					lastReport[event.Name] = time.Now()
					relPath, _ := filepath.Rel(path, event.Name)
					if isPipe {
						io.WriteString(os.Stdout, relPath+"\n")
					} else {
						if f, _ := os.Stat(event.Name); isGoFile(f) {
							format(event.Name)
							if event.Name == watchedRun && !*noRun {
								go goRun(watchedRun)
							}
						}
					}
				}
			} else {
				return
			}
		}
	}
}
<commit_msg>Add -r flag to specify file to run<commit_after>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"gopkg.in/fsnotify.v1"
)

var (
	watchedRun = flag.String("r", "", "main go file to run")
	watchedFmt = "."
	noRun      = flag.Bool("n", false, "only run gofmt")
	inFile     = flag.String("in", "", "input file")

	delay      = flag.Int("d", 1, "delay time before detecting file change")
	isPipe     = false
	exitCode   = 0
	lastReport = make(map[string]time.Time)
)

func getPackageNameAndImport(sourceName string) (packageName string, imports []string) {
	fset := token.NewFileSet() // positions are relative to fset

	// parse the file containing this very example
	// but stop after processing the imports.
	f, err := parser.ParseFile(fset, sourceName, nil, parser.ImportsOnly)
	if err != nil {
		fmt.Println(err)
		return
	}

	// print the imports from the file's AST.
	packageName = f.Name.Name

	for _, s := range f.Imports {
		importedPackageName := s.Path.Value[1 : len(s.Path.Value)-1]
		imports = append(imports, importedPackageName)
	}

	return
}

func isGoFile(f os.FileInfo) bool {
	// ignore non-Go files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
}

func isMainPackage(sourceName string) bool {
	fset := token.NewFileSet()
	f, _ := parser.ParseFile(fset, sourceName, nil, parser.ImportsOnly)

	return f.Name.Name == "main"
}

func isMainFile(sourceName string) bool {
	fset := token.NewFileSet()

	f, err := parser.ParseFile(fset, sourceName, nil, 0)
	if err != nil {
		return false
	}

	if o := f.Scope.Lookup("main"); f.Name.Name == "main" && o != nil && o.Kind == ast.Fun {
		return true
	}

	return false
}

func format(path string) error {
	log.Println("fmt", path)
	command := exec.Command("gofmt", "-w", path)
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr
	command.Run()
	return nil
}

func goFiles(path string) (goFiles, mainFiles []string) {
	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if isGoFile(info) {
			goFiles = append(goFiles, path)
			if isMainFile(path) {
				mainFiles = append(mainFiles, path)
			}
		}
		return nil
	})
	return
}

var goRunCmd *exec.Cmd

func runGoRun(sourceName string) {
	if goRunCmd != nil && goRunCmd.Process != nil {
		// to properly kill go run and its children
		// we need to set goRunCmd.SysProcAttr.Setid to true
		// and send kill signal to the process group
		// with negative PID
		syscall.Kill(-goRunCmd.Process.Pid, syscall.SIGKILL)
		goRunCmd.Process.Wait()
	}

	goRunCmd = exec.Command("go", "run", sourceName)

	func(goRunCmd *exec.Cmd) {
		if len(*inFile) > 0 {
			f, err := os.Open(*inFile)
			if err != nil {
				goRunCmd.Stdin = os.Stdin
			} else {
				goRunCmd.Stdin = f
				defer f.Close()
			}
		}
		goRunCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
		goRunCmd.Stdout = os.Stdout
		goRunCmd.Stderr = os.Stderr
		goRunCmd.Start()
		log.Println("run", sourceName)
		goRunCmd.Wait()
	}(goRunCmd)
}

func killGoRun() {
	if goRunCmd != nil && goRunCmd.Process != nil && goRunCmd.ProcessState == nil {
		// to properly kill go run and its children
		// we need to set goRunCmd.SysProcAttr.Setid to true
		// and send kill signal to the process group
		// with negative PID
		syscall.Kill(-goRunCmd.Process.Pid, syscall.SIGKILL)
		goRunCmd.Process.Wait()
	}
}

func main() {
	gowatchMain()
	os.Exit(exitCode)
}

func gowatchMain() {
	flag.Parse()

	path, err := filepath.Abs(watchedFmt)
	if err != nil {
		panic(err)
	}

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		panic(err)
	}

	err = watcher.Add(path)
	if err != nil {
		panic(err)
	}
	defer watcher.Close()

	goFiles, mainFiles := goFiles(path)

	// pre run gofmt on found Go files
	for _, f := range goFiles {
		format(f)
	}

	if len(*watchedRun) == 0 {
		if len(mainFiles) == 1 {
			*watchedRun = mainFiles[0]
			log.Println("found a main Go file, watch and run", mainFiles[0])
		} else if len(mainFiles) > 1 {
			*watchedRun = mainFiles[0]
			log.Println("found more than one main Go files, watch and run", mainFiles[0])
		} else {
			*watchedRun = ""
			log.Println("main Go files not found")
		}
	} else {
		*watchedRun, _ = filepath.Abs(*watchedRun)
	}

	if len(*watchedRun) > 0 && !*noRun {
		go runGoRun(*watchedRun)
		defer killGoRun()
	}

	log.Println("watching", path)

	fi, _ := os.Stdout.Stat()
	isPipe = fi.Mode()&os.ModeNamedPipe != 0

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt, os.Kill)

	for {
		select {
		case <-sig:
			return
		case event, more := <-watcher.Events:
			if more {
				if event.Op == fsnotify.Create || event.Op == fsnotify.Write {
					if time.Since(lastReport[event.Name]) < time.Duration(*delay)*time.Second {
						continue
					}
					lastReport[event.Name] = time.Now()
					relPath, _ := filepath.Rel(path, event.Name)
					if isPipe {
						io.WriteString(os.Stdout, relPath+"\n")
					} else {
						if f, _ := os.Stat(event.Name); isGoFile(f) {
							format(event.Name)
							if event.Name == *watchedRun && !*noRun {
								go runGoRun(*watchedRun)
							}
						}
					}
				}
			} else {
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>// package tlsdialer contains a customized version of crypto/tls.Dial that
// allows control over whether or not to send the ServerName extension in the
// client handshake.
package tlsdialer

import (
	"crypto/x509"
	"fmt"
	"net"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/mtime"
	"github.com/getlantern/netx"
	"github.com/getlantern/ops"

	"github.com/refraction-networking/utls"
)

var (
	log = golog.LoggerFor("tlsdialer")
)

type timeoutError struct{}

func (timeoutError) Error() string   { return "tlsdialer: DialWithDialer timed out" }
func (timeoutError) Timeout() bool   { return true }
func (timeoutError) Temporary() bool { return true }

// Dialer is a configurable dialer that dials using tls
type Dialer struct {
	DoDial         func(net string, addr string, timeout time.Duration) (net.Conn, error)
	Timeout        time.Duration
	Network        string
	SendServerName bool
	ClientHelloID  tls.ClientHelloID
	Config         *tls.Config
}

// A tls.Conn along with timings for key steps in establishing that Conn
type ConnWithTimings struct {
	// Conn: the conn resulting from dialing
	Conn *tls.Conn
	// ResolutionTime: the amount of time it took to resolve the address
	ResolutionTime time.Duration
	// ConnectTime: the amount of time that it took to connect the socket
	ConnectTime time.Duration
	// HandshakeTime: the amount of time that it took to complete the TLS
	// handshake
	HandshakeTime time.Duration
	// ResolvedAddr: the address to which our dns lookup resolved
	ResolvedAddr *net.TCPAddr
	// VerifiedChains: like tls.ConnectionState.VerifiedChains
	VerifiedChains [][]*x509.Certificate
}

// Like crypto/tls.Dial, but with the ability to control whether or not to
// send the ServerName extension in client handshakes through the sendServerName
// flag.
//
// Note - if sendServerName is false, the VerifiedChains field on the
// connection's ConnectionState will never get populated. Use DialForTimings to
// get back a data structure that includes the verified chains.
func Dial(network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	return DialTimeout(netx.DialTimeout, 1*time.Minute, network, addr, sendServerName, config)
}

// Like Dial, but timing out after the given timeout.
func DialTimeout(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.Dial(network, addr)
}

// Like DialWithDialer but returns a data structure including timings and the
// verified chains.
func DialForTimings(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*ConnWithTimings, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.DialForTimings(network, addr)
}

// Dial dials the given network and address.
func (d *Dialer) Dial(network, addr string) (*tls.Conn, error) {
	cwt, err := d.DialForTimings(network, addr)
	if err != nil {
		return nil, err
	}
	return cwt.Conn, nil
}

// DialForTimings dials the given network and address and returns a ConnWithTimings.
func (d *Dialer) DialForTimings(network, addr string) (*ConnWithTimings, error) {
	result := &ConnWithTimings{}

	var errCh chan error

	if d.Timeout != 0 {
		// We want the Timeout and Deadline values to cover the whole process: TCP
		// connection and TLS handshake. This means that we also need to start our own
		// timers now.
		errCh = make(chan error, 10)
		time.AfterFunc(d.Timeout, func() {
			errCh <- timeoutError{}
		})
	}

	log.Tracef("Resolving addr: %s", addr)
	elapsed := mtime.Stopwatch()
	var err error
	if d.Timeout == 0 {
		log.Tracef("Resolving immediately")
		result.ResolvedAddr, err = netx.Resolve("tcp", addr)
	} else {
		log.Tracef("Resolving on goroutine")
		resolvedCh := make(chan *net.TCPAddr, 10)
		ops.Go(func() {
			resolved, resolveErr := netx.Resolve("tcp", addr)
			log.Tracef("Resolution resulted in %s : %s", resolved, resolveErr)
			resolvedCh <- resolved
			errCh <- resolveErr
		})
		err = <-errCh
		if err == nil {
			log.Tracef("No error, looking for resolved")
			result.ResolvedAddr = <-resolvedCh
		}
	}

	if err != nil {
		return result, err
	}
	result.ResolutionTime = elapsed()
	log.Tracef("Resolved addr %s to %s in %s", addr, result.ResolvedAddr, result.ResolutionTime)

	hostname, _, err := net.SplitHostPort(addr)
	if err != nil {
		return result, fmt.Errorf("Unable to split host and port for %v: %v", addr, err)
	}

	log.Tracef("Dialing %s %s (%s)", network, addr, result.ResolvedAddr)
	elapsed = mtime.Stopwatch()
	resolvedAddr := result.ResolvedAddr.String()
	rawConn, err := d.DoDial(network, resolvedAddr, d.Timeout)
	if err != nil {
		return result, err
	}
	result.ConnectTime = elapsed()
	log.Tracef("Dialed in %s", result.ConnectTime)

	config := d.Config
	if config == nil {
		config = &tls.Config{}
	}

	serverName := config.ServerName

	if serverName == "" {
		log.Trace("No ServerName set, inferring from the hostname to which we're connecting")
		serverName = hostname
	}
	log.Tracef("ServerName is: %s", serverName)

	log.Trace("Copying config so that we can tweak it")
	configCopy := new(tls.Config)
	*configCopy = *config

	if d.SendServerName {
		log.Tracef("Setting ServerName to %s and relying on the usual logic in tls.Conn.Handshake() to do verification", serverName)
		configCopy.ServerName = serverName
	} else {
		log.Trace("Clearing ServerName and disabling verification in tls.Conn.Handshake(). We'll verify manually after handshaking.")
		configCopy.ServerName = ""
		configCopy.InsecureSkipVerify = true
	}

	chid := d.ClientHelloID
	if chid.Browser == "" {
		log.Trace("Defaulting to typical Golang client hello")
		chid = tls.HelloGolang
	}
	conn := tls.UClient(rawConn, configCopy, chid)

	elapsed = mtime.Stopwatch()
	if d.Timeout == 0 {
		log.Trace("Handshaking immediately")
		err = conn.Handshake()
	} else {
		log.Trace("Handshaking on goroutine")
		ops.Go(func() {
			errCh <- conn.Handshake()
		})
		err = <-errCh
	}
	if err == nil {
		result.HandshakeTime = elapsed()
	}
	log.Tracef("Finished handshaking in: %s", result.HandshakeTime)

	if err == nil && !config.InsecureSkipVerify {
		if d.SendServerName {
			log.Trace("Depending on certificate verification in tls.Conn.Handshake()")
			result.VerifiedChains = conn.ConnectionState().VerifiedChains
		} else {
			log.Trace("Manually verifying certificates")
			configCopy.ServerName = ""
			result.VerifiedChains, err = verifyServerCerts(conn.Conn, serverName, configCopy)
		}
	}

	if err != nil {
		log.Tracef("Handshake or verification error, closing underlying connection: %v", err)
		if closeErr := rawConn.Close(); closeErr != nil {
			log.Debugf("Unable to close connection: %v", closeErr)
		}
		log.Errorf("Error establishing TLS connection to %v: %v", rawConn.RemoteAddr(), err)
		return result, err
	}

	result.Conn = conn.Conn
	return result, nil
}

func verifyServerCerts(conn *tls.Conn, serverName string, config *tls.Config) ([][]*x509.Certificate, error) {
	certs := conn.ConnectionState().PeerCertificates

	opts := x509.VerifyOptions{
		Roots:         config.RootCAs,
		CurrentTime:   time.Now(),
		DNSName:       serverName,
		Intermediates: x509.NewCertPool(),
	}

	for i, cert := range certs {
		if i == 0 {
			continue
		}
		opts.Intermediates.AddCert(cert)
	}
	return certs[0].Verify(opts)
}
<commit_msg>omit SNI extension if no servername is present, small fixes for updated utls library<commit_after>// package tlsdialer contains a customized version of crypto/tls.Dial that
// allows control over whether or not to send the ServerName extension in the
// client handshake.
package tlsdialer

import (
	"crypto/x509"
	"fmt"
	"net"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/mtime"
	"github.com/getlantern/netx"
	"github.com/getlantern/ops"

	"github.com/refraction-networking/utls"
)

var (
	log = golog.LoggerFor("tlsdialer")
)

type timeoutError struct{}

func (timeoutError) Error() string   { return "tlsdialer: DialWithDialer timed out" }
func (timeoutError) Timeout() bool   { return true }
func (timeoutError) Temporary() bool { return true }

// Dialer is a configurable dialer that dials using tls
type Dialer struct {
	DoDial         func(net string, addr string, timeout time.Duration) (net.Conn, error)
	Timeout        time.Duration
	Network        string
	SendServerName bool
	ClientHelloID  tls.ClientHelloID
	Config         *tls.Config
}

// A tls.Conn along with timings for key steps in establishing that Conn
type ConnWithTimings struct {
	// Conn: the conn resulting from dialing
	Conn *tls.Conn
	// ResolutionTime: the amount of time it took to resolve the address
	ResolutionTime time.Duration
	// ConnectTime: the amount of time that it took to connect the socket
	ConnectTime time.Duration
	// HandshakeTime: the amount of time that it took to complete the TLS
	// handshake
	HandshakeTime time.Duration
	// ResolvedAddr: the address to which our dns lookup resolved
	ResolvedAddr *net.TCPAddr
	// VerifiedChains: like tls.ConnectionState.VerifiedChains
	VerifiedChains [][]*x509.Certificate
}

// Like crypto/tls.Dial, but with the ability to control whether or not to
// send the ServerName extension in client handshakes through the sendServerName
// flag.
//
// Note - if sendServerName is false, the VerifiedChains field on the
// connection's ConnectionState will never get populated. Use DialForTimings to
// get back a data structure that includes the verified chains.
func Dial(network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	return DialTimeout(netx.DialTimeout, 1*time.Minute, network, addr, sendServerName, config)
}

// Like Dial, but timing out after the given timeout.
func DialTimeout(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*tls.Conn, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.Dial(network, addr)
}

// Like DialWithDialer but returns a data structure including timings and the
// verified chains.
func DialForTimings(dial func(net string, addr string, timeout time.Duration) (net.Conn, error), timeout time.Duration, network, addr string, sendServerName bool, config *tls.Config) (*ConnWithTimings, error) {
	d := &Dialer{
		DoDial:         dial,
		Timeout:        timeout,
		SendServerName: sendServerName,
		Config:         config,
	}
	return d.DialForTimings(network, addr)
}

// Dial dials the given network and address.
func (d *Dialer) Dial(network, addr string) (*tls.Conn, error) {
	cwt, err := d.DialForTimings(network, addr)
	if err != nil {
		return nil, err
	}
	return cwt.Conn, nil
}

// DialForTimings dials the given network and address and returns a ConnWithTimings.
func (d *Dialer) DialForTimings(network, addr string) (*ConnWithTimings, error) {
	result := &ConnWithTimings{}

	var errCh chan error

	if d.Timeout != 0 {
		// We want the Timeout and Deadline values to cover the whole process: TCP
		// connection and TLS handshake. This means that we also need to start our own
		// timers now.
		errCh = make(chan error, 10)
		time.AfterFunc(d.Timeout, func() {
			errCh <- timeoutError{}
		})
	}

	log.Tracef("Resolving addr: %s", addr)
	elapsed := mtime.Stopwatch()
	var err error
	if d.Timeout == 0 {
		log.Tracef("Resolving immediately")
		result.ResolvedAddr, err = netx.Resolve("tcp", addr)
	} else {
		log.Tracef("Resolving on goroutine")
		resolvedCh := make(chan *net.TCPAddr, 10)
		ops.Go(func() {
			resolved, resolveErr := netx.Resolve("tcp", addr)
			log.Tracef("Resolution resulted in %s : %s", resolved, resolveErr)
			resolvedCh <- resolved
			errCh <- resolveErr
		})
		err = <-errCh
		if err == nil {
			log.Tracef("No error, looking for resolved")
			result.ResolvedAddr = <-resolvedCh
		}
	}

	if err != nil {
		return result, err
	}
	result.ResolutionTime = elapsed()
	log.Tracef("Resolved addr %s to %s in %s", addr, result.ResolvedAddr, result.ResolutionTime)

	hostname, _, err := net.SplitHostPort(addr)
	if err != nil {
		return result, fmt.Errorf("Unable to split host and port for %v: %v", addr, err)
	}

	log.Tracef("Dialing %s %s (%s)", network, addr, result.ResolvedAddr)
	elapsed = mtime.Stopwatch()
	resolvedAddr := result.ResolvedAddr.String()
	rawConn, err := d.DoDial(network, resolvedAddr, d.Timeout)
	if err != nil {
		return result, err
	}
	result.ConnectTime = elapsed()
	log.Tracef("Dialed in %s", result.ConnectTime)

	config := d.Config
	if config == nil {
		config = &tls.Config{}
	}

	serverName := config.ServerName

	if serverName == "" {
		log.Trace("No ServerName set, inferring from the hostname to which we're connecting")
		serverName = hostname
	}
	log.Tracef("ServerName is: %s", serverName)

	log.Trace("Copying config so that we can tweak it")
	configCopy := new(tls.Config)
	*configCopy = *config

	if d.SendServerName {
		log.Tracef("Setting ServerName to %s and relying on the usual logic in tls.Conn.Handshake() to do verification", serverName)
		configCopy.ServerName = serverName
	} else {
		log.Trace("Clearing ServerName and disabling verification in tls.Conn.Handshake(). We'll verify manually after handshaking.")
		configCopy.ServerName = ""
		configCopy.InsecureSkipVerify = true
	}

	chid := d.ClientHelloID
	if chid.Client == "" {
		log.Trace("Defaulting to typical Golang client hello")
		chid = tls.HelloGolang
	}
	conn := tls.UClient(rawConn, configCopy, chid)
	if !d.SendServerName {
		// Actually omit the SNI Extension.
		// blank ServerName values appear to confuse some server software
		// and real browsers appear to omit the extension in this
		// case. XXX upstream fix?
		filteredExts := make([]tls.TLSExtension, 0, len(conn.Extensions))
		for _, e := range conn.Extensions {
			if _, ok := e.(*tls.SNIExtension); !ok {
				filteredExts = append(filteredExts, e)
			}
		}
		conn.Extensions = filteredExts
	}
	elapsed = mtime.Stopwatch()
	if d.Timeout == 0 {
		log.Trace("Handshaking immediately")
		err = conn.Handshake()
	} else {
		log.Trace("Handshaking on goroutine")
		ops.Go(func() {
			errCh <- conn.Handshake()
		})
		err = <-errCh
	}
	if err == nil {
		result.HandshakeTime = elapsed()
	}
	log.Tracef("Finished handshaking in: %s", result.HandshakeTime)

	if err == nil && !config.InsecureSkipVerify {
		if d.SendServerName {
			log.Trace("Depending on certificate verification in tls.Conn.Handshake()")
			result.VerifiedChains = conn.ConnectionState().VerifiedChains
		} else {
			log.Trace("Manually verifying certificates")
			configCopy.ServerName = ""
			result.VerifiedChains, err = verifyServerCerts(conn.Conn, serverName, configCopy)
		}
	}

	if err != nil {
		log.Tracef("Handshake or verification error, closing underlying connection: %v", err)
		if closeErr := rawConn.Close(); closeErr != nil {
			log.Debugf("Unable to close connection: %v", closeErr)
		}
		log.Errorf("Error establishing TLS connection to %v: %v", rawConn.RemoteAddr(), err)
		return result, err
	}

	result.Conn = conn.Conn
	return result, nil
}

func verifyServerCerts(conn *tls.Conn, serverName string, config *tls.Config) ([][]*x509.Certificate, error) {
	certs := conn.ConnectionState().PeerCertificates

	opts := x509.VerifyOptions{
		Roots:         config.RootCAs,
		CurrentTime:   time.Now(),
		DNSName:       serverName,
		Intermediates: x509.NewCertPool(),
	}

	for i, cert := range certs {
		if i == 0 {
			continue
		}
		opts.Intermediates.AddCert(cert)
	}
	return certs[0].Verify(opts)
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	"strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	// flag.IntVar(&year, "year", 2014, "2014 year")
	flag.Parse()

	setinitialconnection()

	prepdatabranch()
	// prepdatacostcenter()
	// prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	// arrfilter := []*dbox.Filter{}
	// for i := 4; i < 10; i++ {
	// 	toolkit.Printfn("%d - 2014", i)
	// 	f := dbox.And(dbox.Eq("year", 2014), dbox.Eq("month", i))
	// 	arrfilter = append(arrfilter, f)
	// }

	// f := dbox.Or(arrfilter...)

	csr, _ := workerconn.NewQuery().Select().From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		// cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		// key.Set("customer_branchgroup", "OTHER")

		// trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		// if mastercostcenter.Has(cc) {
		// 	mcc := mastercostcenter[cc].(toolkit.M)
		// 	brid := mcc.GetString("branchid")

		// 	trx.Set("branchid", brid)
		// 	trx.Set("costgroup", mcc.GetString("costgroup01"))

		// 	if masterbranch.Has(brid) {
		// 		trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
		// 		trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
		// 	}
		// }

		// branchid := trx.GetString("branchid")

		// accdesc := trx.GetString("accountdescription")
		// trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		// if trx.GetString("costgroup") == "" {
		// 	trx.Set("costgroup", "OTHER")
		// }

		// if trx.GetString("branchname") == "" {
		// 	trx.Set("branchname", "OTHER")
		// }

		// if trx.GetString("brancharea") == "" {
		// 	trx.Set("brancharea", "OTHER")
		// }

		// if trx.GetString("accountgroup") == "" {
		// 	trx.Set("accountgroup", "OTHER")
		// }

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		//=== For data rawdata mode

		// branchid := trx.GetString("branchid")
		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// trx.Set("branchgroup", branchgroup.GetString("branchgroup"))
		// trx.Set("branchlvl2", branchgroup.GetString("branchlvl2"))

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		// if trx.GetString("branchlvl2") == "" {
		// 	trx.Set("branchlvl2", "OTHER")
		// }

		//===========================================

		//=== For data salespls-summary mode consolidate

		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// branchid := key.GetString("customer_branchid")

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))
		// key.Set("customer_branchlvl2", branchgroup.GetString("branchlvl2"))

		// if key.GetString("customer_branchgroup") == "" {
		// 	key.Set("customer_branchgroup", "OTHER")
		// }

		// if key.GetString("customer_branchlvl2") == "" {
		// 	key.Set("customer_branchlvl2", "OTHER")
		// }

		// /*other fix*/
		// if key.GetString("customer_reportsubchannel") == "R3" {
		// 	key.Set("customer_reportsubchannel", "R3 - Retailer Umum")
		// }

		// if key.GetString("customer_region") == "" || key.GetString("customer_region") == "Other" {
		// 	key.Set("customer_region", "OTHER")
		// }

		// if key.GetString("customer_zone") == "" || key.GetString("customer_zone") == "Other" {
		// 	key.Set("customer_zone", "OTHER")
		// }

		// trx.Set("key", key)
		//============================================

		// For cogs consolidate
		// arrstr := []string{"rm_perunit", "lc_perunit", "pf_perunit", "other_perunit", "fixed_perunit", "depre_perunit", "cogs_perunit"}
		// for _, v := range arrstr {
		// 	xval := trx.GetFloat64(v) * 6
		// 	trx.Set(v, xval)
		// }
		// ====================

		trx.Set("addinfo", "")
		if trx.GetString("branchid") == "HD11" && strings.Contains(trx.GetString("costcentername"), "Jkt") {
			trx.Set("addinfo", "Jakarta")
		}

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<commit_msg>key to id<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	// "strings"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	// flag.IntVar(&year, "year", 2014, "2014 year")
	flag.Parse()

	setinitialconnection()

	prepdatabranch()
	// prepdatacostcenter()
	// prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	// arrfilter := []*dbox.Filter{}
	// for i := 4; i < 10; i++ {
	// 	toolkit.Printfn("%d - 2014", i)
	// 	f := dbox.And(dbox.Eq("year", 2014), dbox.Eq("month", i))
	// 	arrfilter = append(arrfilter, f)
	// }

	// f := dbox.Or(arrfilter...)

	csr, _ := workerconn.NewQuery().Select().From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		key := trx.Get("key", toolkit.M{}).(toolkit.M)
		trx.Set("key", key)

		id := toolkit.Sprintf("%d|%s|%s|%s|%s|%s|%s|%s", key.GetInt("year"), key.GetString("branchid"),
			key.GetString("branchname"), key.GetString("brancharea"), key.GetString("account"),
			key.GetString("accountdescription"), key.GetString("costgroup"), key.GetString("addinfo"))

		trx.Set("_id", id)

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		// cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		// key.Set("customer_branchgroup", "OTHER")

		// trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		// if mastercostcenter.Has(cc) {
		// 	mcc := mastercostcenter[cc].(toolkit.M)
		// 	brid := mcc.GetString("branchid")

		// 	trx.Set("branchid", brid)
		// 	trx.Set("costgroup", mcc.GetString("costgroup01"))

		// 	if masterbranch.Has(brid) {
		// 		trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
		// 		trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
		// 	}
		// }

		// branchid := trx.GetString("branchid")

		// accdesc := trx.GetString("accountdescription")
		// trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		// if trx.GetString("costgroup") == "" {
		// 	trx.Set("costgroup", "OTHER")
		// }

		// if trx.GetString("branchname") == "" {
		// 	trx.Set("branchname", "OTHER")
		// }

		// if trx.GetString("brancharea") == "" {
		// 	trx.Set("brancharea", "OTHER")
		// }

		// if trx.GetString("accountgroup") == "" {
		// 	trx.Set("accountgroup", "OTHER")
		// }

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		//=== For data rawdata mode

		// branchid := trx.GetString("branchid")
		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// trx.Set("branchgroup", branchgroup.GetString("branchgroup"))
		// trx.Set("branchlvl2", branchgroup.GetString("branchlvl2"))

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		// if trx.GetString("branchlvl2") == "" {
		// 	trx.Set("branchlvl2", "OTHER")
		// }

		//===========================================

		//=== For data salespls-summary mode consolidate

		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// branchid := key.GetString("customer_branchid")

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))
		// key.Set("customer_branchlvl2", branchgroup.GetString("branchlvl2"))

		// if key.GetString("customer_branchgroup") == "" {
		// 	key.Set("customer_branchgroup", "OTHER")
		// }

		// if key.GetString("customer_branchlvl2") == "" {
		// 	key.Set("customer_branchlvl2", "OTHER")
		// }

		// /*other fix*/
		// if key.GetString("customer_reportsubchannel") == "R3" {
		// 	key.Set("customer_reportsubchannel", "R3 - Retailer Umum")
		// }

		// if key.GetString("customer_region") == "" || key.GetString("customer_region") == "Other" {
		// 	key.Set("customer_region", "OTHER")
		// }

		// if key.GetString("customer_zone") == "" || key.GetString("customer_zone") == "Other" {
		// 	key.Set("customer_zone", "OTHER")
		// }

		// trx.Set("key", key)
		//============================================

		// For cogs consolidate
		// arrstr := []string{"rm_perunit", "lc_perunit", "pf_perunit", "other_perunit", "fixed_perunit", "depre_perunit", "cogs_perunit"}
		// for _, v := range arrstr {
		// 	xval := trx.GetFloat64(v) * 6
		// 	trx.Set(v, xval)
		// }
		// ====================

		// trx.Set("addinfo", "")
		// if trx.GetString("branchid") == "HD11" && strings.Contains(trx.GetString("costcentername"), "Jkt") {
		// 	trx.Set("addinfo", "Jakarta")
		// }

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<|endoftext|>


<commit_before>package eventmaster

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"

	eventmaster "github.com/ContextLogic/eventmaster/proto"
)

// AnnotationsReq encodes the information provided by Grafana in its requests.
type AnnotationsReq struct {
	Range      Range      `json:"range"`
	Annotation Annotation `json:"annotation"`
}

// Range specifies the time range the request is valid for.
type Range struct {
	From time.Time `json:"from"`
	To   time.Time `json:"to"`
}

// Annotation is the object passed by Grafana when it fetches annotations.
//
// http://docs.grafana.org/plugins/developing/datasources/#annotation-query
type Annotation struct {
	// Name must match in the request and response
	Name string `json:"name"`

	Datasource string `json:"datasource"`
	IconColor  string `json:"iconColor"`
	Enable     bool   `json:"enable"`
	ShowLine   bool   `json:"showLine"`
	Query      string `json:"query"`
}

// AnnotationResponse contains all the information needed to render an
// annotation event.
//
// https://github.com/grafana/simple-json-datasource#annotation-api
type AnnotationResponse struct {
	// The original annotation sent from Grafana.
	Annotation Annotation `json:"annotation"`
	// Time since UNIX Epoch in milliseconds. (required)
	Time int64 `json:"time"`
	// The title for the annotation tooltip. (required)
	Title string `json:"title"`
	// Tags for the annotation. (optional)
	Tags string `json:"tags"`
	// Text for the annotation. (optional)
	Text string `json:"text"`
}

// AnnotationQuery is a collection of possible filters for a Grafana annotation
// request.
//
// These values are set in gear icon > annotations > edit > Query and can have
// such useful values as:
//
//		{"topic": "$topic", "dc": "$dc"}
//
// or if you do not want filtering leave it blank.
type AnnotationQuery struct {
	Topic string `json:"topic"`
	DC    string `json:"dc"`
}

type TemplateRequest struct {
	Target string `json:"target"`
}

// cors adds headers that Grafana requires to work as a direct access data
// source.
//
// forgetting to add these manifests itself as an unintellible error when
// adding a datasource.
//
// These are not required if using "proxy" access.
func cors(h httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		w.Header().Set("Access-Control-Allow-Headers", "accept, content-type")
		w.Header().Set("Access-Control-Allow-Methods", "POST")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		h(w, r, p)
	}
}

func (h *Server) grafanaOK(w http.ResponseWriter, r *http.Request, p httprouter.Params) {}

// grafanaAnnotations handles the POST requests from Grafana asking for
// a window of annotations.
//
// It accepts an AnnotationsReq in the request body, and parses the
// AnnotationsReq.Query for a AnnotationQuery. If one is found it parses it,
// and interprets the value "all" as "do not apply this filter".
func (h *Server) grafanaAnnotations(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	switch r.Method {
	case http.MethodPost:
		ar := AnnotationsReq{}
		if err := json.NewDecoder(r.Body).Decode(&ar); err != nil {
			http.Error(w, fmt.Sprintf("json decode failure: %v", err), http.StatusBadRequest)
			return
		}

		q := &eventmaster.Query{
			StartEventTime: ar.Range.From.Unix(),
			EndEventTime:   ar.Range.To.Unix(),
		}

		if rq := ar.Annotation.Query; rq != "" {
			aq := AnnotationQuery{}
			if err := json.Unmarshal([]byte(rq), &aq); err != nil {
				http.Error(w, fmt.Sprintf("json decode failure: %v", err), http.StatusBadRequest)
				return
			}
			if aq.DC != "all" {
				q.Dc = []string{aq.DC}
			}
			if aq.Topic != "all" {
				q.TopicName = []string{aq.Topic}
			}
		}

		evs, err := h.store.Find(q)
		if err != nil {
			e := errors.Wrapf(err, "grafana search with %v", q)
			http.Error(w, e.Error(), http.StatusInternalServerError)
			return
		}

		ars := []AnnotationResponse{}
		for _, ev := range evs {
			ar, err := FromEvent(h.store, ev)
			if err != nil {
				http.Error(w, errors.Wrap(err, "from event").Error(), http.StatusInternalServerError)
				return
			}
			ars = append(ars, ar)
		}

		if err := json.NewEncoder(w).Encode(ars); err != nil {
			log.Printf("json enc: %+v", err)
		}
	default:
		http.Error(w, "bad method; supported POST", http.StatusBadRequest)
		return
	}
}

func (h *Server) grafanaSearch(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	req := TemplateRequest{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		http.Error(w, errors.Wrap(err, "json decode").Error(), http.StatusBadRequest)
		return
	}

	tags := []string{"all"}
	switch req.Target {
	case "dc":
		dcs, err := h.store.GetDcs()
		if err != nil {
			http.Error(w, errors.Wrap(err, "get dcs").Error(), http.StatusInternalServerError)
			return
		}
		for _, dc := range dcs {
			tags = append(tags, dc.Name)
		}
	case "topic":
		topics, err := h.store.GetTopics()
		if err != nil {
			http.Error(w, errors.Wrap(err, "get topics").Error(), http.StatusInternalServerError)
			return
		}
		for _, topic := range topics {
			tags = append(tags, topic.Name)
		}
	default:
		http.Error(w, fmt.Sprintf("unknown target: got %q, want [%q, %q]", req.Target, "dc", "topic"), http.StatusBadRequest)
		return
	}

	sort.Strings(tags[1:])
	if err := json.NewEncoder(w).Encode(tags); err != nil {
		log.Printf("json encode failure: %+v", err)
	}
}

type topicNamer interface {
	getTopicName(string) string
	getDcName(string) string
}

func FromEvent(store topicNamer, ev *Event) (AnnotationResponse, error) {
	fm := template.FuncMap{
		"trim": strings.TrimSpace,
	}
	t := `

Host: {{ .Host }}
Target Hosts:
{{- range .TargetHosts }}
	{{ trim . -}}
{{ end }}
User: {{ .User }}
Data: {{ .Data }}

`
	tmpl, err := template.New("text").Funcs(fm).Parse(t)
	if err != nil {
		return AnnotationResponse{}, errors.Wrap(err, "making template")
	}
	buf := &bytes.Buffer{}
	tmpl.Execute(buf, ev)
	r := AnnotationResponse{
		Time:  ev.EventTime * 1000,
		Title: fmt.Sprintf("%v in %v", store.getTopicName(ev.TopicID), store.getDcName(ev.DcID)),
		Text:  buf.String(),
		Tags:  strings.Join(ev.Tags, ","),
	}
	return r, nil
}
<commit_msg>Only populate slices if it's required<commit_after>package eventmaster

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"

	eventmaster "github.com/ContextLogic/eventmaster/proto"
)

// AnnotationsReq encodes the information provided by Grafana in its requests.
type AnnotationsReq struct {
	Range      Range      `json:"range"`
	Annotation Annotation `json:"annotation"`
}

// Range specifies the time range the request is valid for.
type Range struct {
	From time.Time `json:"from"`
	To   time.Time `json:"to"`
}

// Annotation is the object passed by Grafana when it fetches annotations.
//
// http://docs.grafana.org/plugins/developing/datasources/#annotation-query
type Annotation struct {
	// Name must match in the request and response
	Name string `json:"name"`

	Datasource string `json:"datasource"`
	IconColor  string `json:"iconColor"`
	Enable     bool   `json:"enable"`
	ShowLine   bool   `json:"showLine"`
	Query      string `json:"query"`
}

// AnnotationResponse contains all the information needed to render an
// annotation event.
//
// https://github.com/grafana/simple-json-datasource#annotation-api
type AnnotationResponse struct {
	// The original annotation sent from Grafana.
	Annotation Annotation `json:"annotation"`
	// Time since UNIX Epoch in milliseconds. (required)
	Time int64 `json:"time"`
	// The title for the annotation tooltip. (required)
	Title string `json:"title"`
	// Tags for the annotation. (optional)
	Tags string `json:"tags"`
	// Text for the annotation. (optional)
	Text string `json:"text"`
}

// AnnotationQuery is a collection of possible filters for a Grafana annotation
// request.
//
// These values are set in gear icon > annotations > edit > Query and can have
// such useful values as:
//
//		{"topic": "$topic", "dc": "$dc"}
//
// or if you do not want filtering leave it blank.
type AnnotationQuery struct {
	Topic string `json:"topic"`
	DC    string `json:"dc"`
}

type TemplateRequest struct {
	Target string `json:"target"`
}

// cors adds headers that Grafana requires to work as a direct access data
// source.
//
// forgetting to add these manifests itself as an unintellible error when
// adding a datasource.
//
// These are not required if using "proxy" access.
func cors(h httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		w.Header().Set("Access-Control-Allow-Headers", "accept, content-type")
		w.Header().Set("Access-Control-Allow-Methods", "POST")
		w.Header().Set("Access-Control-Allow-Origin", "*")
		h(w, r, p)
	}
}

func (h *Server) grafanaOK(w http.ResponseWriter, r *http.Request, p httprouter.Params) {}

// grafanaAnnotations handles the POST requests from Grafana asking for
// a window of annotations.
//
// It accepts an AnnotationsReq in the request body, and parses the
// AnnotationsReq.Query for a AnnotationQuery. If one is found it parses it,
// and interprets the value "all" as "do not apply this filter".
func (h *Server) grafanaAnnotations(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	switch r.Method {
	case http.MethodPost:
		ar := AnnotationsReq{}
		if err := json.NewDecoder(r.Body).Decode(&ar); err != nil {
			http.Error(w, fmt.Sprintf("json decode failure: %v", err), http.StatusBadRequest)
			return
		}

		q := &eventmaster.Query{
			StartEventTime: ar.Range.From.Unix(),
			EndEventTime:   ar.Range.To.Unix(),
		}

		if rq := ar.Annotation.Query; rq != "" {
			aq := AnnotationQuery{}
			if err := json.Unmarshal([]byte(rq), &aq); err != nil {
				http.Error(w, fmt.Sprintf("json decode failure: %v", err), http.StatusBadRequest)
				return
			}
			if aq.DC != "" && aq.DC != "all" {
				q.Dc = []string{aq.DC}
			}
			if aq.Topic != "" && aq.Topic != "all" {
				q.TopicName = []string{aq.Topic}
			}
		}

		evs, err := h.store.Find(q)
		if err != nil {
			e := errors.Wrapf(err, "grafana search with %v", q)
			http.Error(w, e.Error(), http.StatusInternalServerError)
			return
		}

		ars := []AnnotationResponse{}
		for _, ev := range evs {
			ar, err := FromEvent(h.store, ev)
			if err != nil {
				http.Error(w, errors.Wrap(err, "from event").Error(), http.StatusInternalServerError)
				return
			}
			ars = append(ars, ar)
		}

		if err := json.NewEncoder(w).Encode(ars); err != nil {
			log.Printf("json enc: %+v", err)
		}
	default:
		http.Error(w, "bad method; supported POST", http.StatusBadRequest)
		return
	}
}

func (h *Server) grafanaSearch(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	req := TemplateRequest{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		http.Error(w, errors.Wrap(err, "json decode").Error(), http.StatusBadRequest)
		return
	}

	tags := []string{"all"}
	switch req.Target {
	case "dc":
		dcs, err := h.store.GetDcs()
		if err != nil {
			http.Error(w, errors.Wrap(err, "get dcs").Error(), http.StatusInternalServerError)
			return
		}
		for _, dc := range dcs {
			tags = append(tags, dc.Name)
		}
	case "topic":
		topics, err := h.store.GetTopics()
		if err != nil {
			http.Error(w, errors.Wrap(err, "get topics").Error(), http.StatusInternalServerError)
			return
		}
		for _, topic := range topics {
			tags = append(tags, topic.Name)
		}
	default:
		http.Error(w, fmt.Sprintf("unknown target: got %q, want [%q, %q]", req.Target, "dc", "topic"), http.StatusBadRequest)
		return
	}

	sort.Strings(tags[1:])
	if err := json.NewEncoder(w).Encode(tags); err != nil {
		log.Printf("json encode failure: %+v", err)
	}
}

type topicNamer interface {
	getTopicName(string) string
	getDcName(string) string
}

func FromEvent(store topicNamer, ev *Event) (AnnotationResponse, error) {
	fm := template.FuncMap{
		"trim": strings.TrimSpace,
	}
	t := `

Host: {{ .Host }}
Target Hosts:
{{- range .TargetHosts }}
	{{ trim . -}}
{{ end }}
User: {{ .User }}
Data: {{ .Data }}

`
	tmpl, err := template.New("text").Funcs(fm).Parse(t)
	if err != nil {
		return AnnotationResponse{}, errors.Wrap(err, "making template")
	}
	buf := &bytes.Buffer{}
	tmpl.Execute(buf, ev)
	r := AnnotationResponse{
		Time:  ev.EventTime * 1000,
		Title: fmt.Sprintf("%v in %v", store.getTopicName(ev.TopicID), store.getDcName(ev.DcID)),
		Text:  buf.String(),
		Tags:  strings.Join(ev.Tags, ","),
	}
	return r, nil
}
<|endoftext|>