<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// spoof contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package spoof

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/test/ingress"
	"knative.dev/pkg/test/logging"
	"knative.dev/pkg/test/zipkin"
	"knative.dev/pkg/tracing/propagation/tracecontextb3"

	"go.opencensus.io/plugin/ochttp"
	"go.opencensus.io/trace"
)

const (
	// Name of the temporary HTTP header that is added to http.Request to indicate that
	// it is a SpoofClient.Poll request. This header is removed before making call to backend.
	pollReqHeader = "X-Kn-Poll-Request-Do-Not-Trace"
)

// Response is a stripped down subset of http.Response. The is primarily useful
// for ResponseCheckers to inspect the response body without consuming it.
// Notably, Body is a byte slice instead of an io.ReadCloser.
type Response struct {
	Status     string
	StatusCode int
	Header     http.Header
	Body       []byte
}

func (r *Response) String() string {
	return fmt.Sprintf("status: %d, body: %s, headers: %v", r.StatusCode, string(r.Body), r.Header)
}

// Interface defines the actions that can be performed by the spoofing client.
type Interface interface {
	Do(*http.Request, ...ErrorRetryChecker) (*Response, error)
	Poll(*http.Request, ResponseChecker, ...ErrorRetryChecker) (*Response, error)
}

// https://medium.com/stupid-gopher-tricks/ensuring-go-interface-satisfaction-at-compile-time-1ed158e8fa17
var (
	_           Interface = (*SpoofingClient)(nil)
	dialContext           = (&net.Dialer{}).DialContext
)

// ResponseChecker is used to determine when SpoofinClient.Poll is done polling.
// This allows you to predicate wait.PollImmediate on the request's http.Response.
//
// See the apimachinery wait package:
// https://github.com/kubernetes/apimachinery/blob/cf7ae2f57dabc02a3d215f15ca61ae1446f3be8f/pkg/util/wait/wait.go#L172
type ResponseChecker func(resp *Response) (done bool, err error)

// ErrorRetryChecker is used to determine if an error should be retried or not.
// If an error should be retried, it should return true and the wrapped error to explain why to retry.
type ErrorRetryChecker func(e error) (retry bool, err error)

// SpoofingClient is a minimal HTTP client wrapper that spoofs the domain of requests
// for non-resolvable domains.
type SpoofingClient struct {
	Client          *http.Client
	RequestInterval time.Duration
	RequestTimeout  time.Duration
	Logf            logging.FormatLogger
}

// TransportOption allows callers to customize the http.Transport used by a SpoofingClient
type TransportOption func(transport *http.Transport) *http.Transport

// New returns a SpoofingClient that rewrites requests if the target domain is not `resolvable`.
// It does this by looking up the ingress at construction time, so reusing a client will not
// follow the ingress if it moves (or if there are multiple ingresses).
//
// If that's a problem, see test/request.go#WaitForEndpointState for oneshot spoofing.
func New(
	ctx context.Context,
	kubeClientset *kubernetes.Clientset,
	logf logging.FormatLogger,
	domain string,
	resolvable bool,
	endpointOverride string,
	requestInterval time.Duration,
	requestTimeout time.Duration,
	opts ...TransportOption) (*SpoofingClient, error) {
	endpoint, mapper, err := ResolveEndpoint(ctx, kubeClientset, domain, resolvable, endpointOverride)
	if err != nil {
		return nil, fmt.Errorf("failed get the cluster endpoint: %w", err)
	}

	// Spoof the hostname at the resolver level
	logf("Spoofing %s -> %s", domain, endpoint)
	transport := &http.Transport{
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			_, port, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}
			// The original hostname:port is spoofed by replacing the hostname by the value
			// returned by ResolveEndpoint.
			return dialContext(ctx, network, net.JoinHostPort(endpoint, mapper(port)))
		},
	}

	for _, opt := range opts {
		transport = opt(transport)
	}

	// Enable Zipkin tracing
	roundTripper := &ochttp.Transport{
		Base:        transport,
		Propagation: tracecontextb3.TraceContextB3Egress,
	}

	sc := SpoofingClient{
		Client:          &http.Client{Transport: roundTripper},
		RequestInterval: requestInterval,
		RequestTimeout:  requestTimeout,
		Logf:            logf,
	}
	return &sc, nil
}

// ResolveEndpoint resolves the endpoint address considering whether the domain is resolvable and taking into
// account whether the user overrode the endpoint address externally
func ResolveEndpoint(ctx context.Context, kubeClientset *kubernetes.Clientset, domain string, resolvable bool, endpointOverride string) (string, func(string) string, error) {
	id := func(in string) string { return in }
	// If the domain is resolvable, it can be used directly
	if resolvable {
		return domain, id, nil
	}
	// Otherwise, use the actual cluster endpoint
	return ingress.GetIngressEndpoint(ctx, kubeClientset, endpointOverride)
}

// Do dispatches to the underlying http.Client.Do, spoofing domains as needed
// and transforming the http.Response into a spoof.Response.
// Each response is augmented with "ZipkinTraceID" header that identifies the zipkin trace corresponding to the request.
func (sc *SpoofingClient) Do(req *http.Request, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
	return sc.Poll(req, func(*Response) (bool, error) { return true, nil }, errorRetryCheckers...)
}

// Poll executes an http request until it satisfies the inState condition or, if there's an error,
// none of the error retry checkers permit a retry.
// If no retry checkers are specified `DefaultErrorRetryChecker` will be used.
func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
	if len(errorRetryCheckers) == 0 {
		errorRetryCheckers = []ErrorRetryChecker{DefaultErrorRetryChecker}
	}

	var resp *Response
	err := wait.PollImmediate(sc.RequestInterval, sc.RequestTimeout, func() (bool, error) {
		// Starting span to capture zipkin trace.
		traceContext, span := trace.StartSpan(req.Context(), "SpoofingClient-Trace")
		defer span.End()
		rawResp, err := sc.Client.Do(req.WithContext(traceContext))
		if err != nil {
			for _, checker := range errorRetryCheckers {
				retry, newErr := checker(err)
				if retry {
					sc.Logf("Retrying %s: %v", req.URL.String(), newErr)
					return false, nil
				}
			}
			sc.Logf("NOT Retrying %s: %v", req.URL.String(), err)
			return true, err
		}
		defer rawResp.Body.Close()

		body, err := ioutil.ReadAll(rawResp.Body)
		if err != nil {
			return true, err
		}
		rawResp.Header.Add(zipkin.ZipkinTraceIDHeader, span.SpanContext().TraceID.String())

		resp = &Response{
			Status:     rawResp.Status,
			StatusCode: rawResp.StatusCode,
			Header:     rawResp.Header,
			Body:       body,
		}
		return inState(resp)
	})

	if resp != nil {
		sc.logZipkinTrace(resp)
	}

	if err != nil {
		return resp, fmt.Errorf("response: %s did not pass checks: %w", resp, err)
	}
	return resp, nil
}

// DefaultErrorRetryChecker implements the defaults for retrying on error.
func DefaultErrorRetryChecker(err error) (bool, error) {
	if isTCPTimeout(err) {
		return true, fmt.Errorf("retrying for TCP timeout: %w", err)
	}
	// Retrying on DNS error, since we may be using xip.io or nip.io in tests.
	if isDNSError(err) {
		return true, fmt.Errorf("retrying for DNS error: %w", err)
	}
	// Repeat the poll on `connection refused` errors, which are usually transient Istio errors.
	if isConnectionRefused(err) {
		return true, fmt.Errorf("retrying for connection refused: %w", err)
	}
	if isConnectionReset(err) {
		return true, fmt.Errorf("retrying for connection reset: %w", err)
	}
	// Retry on connection/network errors.
	if errors.Is(err, io.EOF) {
		return true, fmt.Errorf("retrying for: %w", err)
	}
	return false, err
}

// logZipkinTrace provides support to log Zipkin Trace for param: spoofResponse
// We only log Zipkin trace for HTTP server errors i.e for HTTP status codes between 500 to 600
func (sc *SpoofingClient) logZipkinTrace(spoofResp *Response) {
	if !zipkin.ZipkinTracingEnabled || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
		return
	}

	traceID := spoofResp.Header.Get(zipkin.ZipkinTraceIDHeader)
	sc.Logf("Logging Zipkin Trace for: %s", traceID)

	json, err := zipkin.JSONTrace(traceID /* We don't know the expected number of spans */, -1, 5*time.Second)
	if err != nil {
		if _, ok := err.(*zipkin.TimeoutError); !ok {
			sc.Logf("Error getting zipkin trace: %v", err)
		}
	}

	sc.Logf("%s", json)
}
<commit_msg>Remove the interface that is no longer required (#1707)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// spoof contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package spoof

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"knative.dev/pkg/test/ingress"
	"knative.dev/pkg/test/logging"
	"knative.dev/pkg/test/zipkin"
	"knative.dev/pkg/tracing/propagation/tracecontextb3"

	"go.opencensus.io/plugin/ochttp"
	"go.opencensus.io/trace"
)

const (
	// Name of the temporary HTTP header that is added to http.Request to indicate that
	// it is a SpoofClient.Poll request. This header is removed before making call to backend.
	pollReqHeader = "X-Kn-Poll-Request-Do-Not-Trace"
)

// Response is a stripped down subset of http.Response. The is primarily useful
// for ResponseCheckers to inspect the response body without consuming it.
// Notably, Body is a byte slice instead of an io.ReadCloser.
type Response struct {
	Status     string
	StatusCode int
	Header     http.Header
	Body       []byte
}

func (r *Response) String() string {
	return fmt.Sprintf("status: %d, body: %s, headers: %v", r.StatusCode, string(r.Body), r.Header)
}

// https://medium.com/stupid-gopher-tricks/ensuring-go-interface-satisfaction-at-compile-time-1ed158e8fa17
var dialContext = (&net.Dialer{}).DialContext

// ResponseChecker is used to determine when SpoofinClient.Poll is done polling.
// This allows you to predicate wait.PollImmediate on the request's http.Response.
//
// See the apimachinery wait package:
// https://github.com/kubernetes/apimachinery/blob/cf7ae2f57dabc02a3d215f15ca61ae1446f3be8f/pkg/util/wait/wait.go#L172
type ResponseChecker func(resp *Response) (done bool, err error)

// ErrorRetryChecker is used to determine if an error should be retried or not.
// If an error should be retried, it should return true and the wrapped error to explain why to retry.
type ErrorRetryChecker func(e error) (retry bool, err error)

// SpoofingClient is a minimal HTTP client wrapper that spoofs the domain of requests
// for non-resolvable domains.
type SpoofingClient struct {
	Client          *http.Client
	RequestInterval time.Duration
	RequestTimeout  time.Duration
	Logf            logging.FormatLogger
}

// TransportOption allows callers to customize the http.Transport used by a SpoofingClient
type TransportOption func(transport *http.Transport) *http.Transport

// New returns a SpoofingClient that rewrites requests if the target domain is not `resolvable`.
// It does this by looking up the ingress at construction time, so reusing a client will not
// follow the ingress if it moves (or if there are multiple ingresses).
//
// If that's a problem, see test/request.go#WaitForEndpointState for oneshot spoofing.
func New(
	ctx context.Context,
	kubeClientset *kubernetes.Clientset,
	logf logging.FormatLogger,
	domain string,
	resolvable bool,
	endpointOverride string,
	requestInterval, requestTimeout time.Duration,
	opts ...TransportOption) (*SpoofingClient, error) {
	endpoint, mapper, err := ResolveEndpoint(ctx, kubeClientset, domain, resolvable, endpointOverride)
	if err != nil {
		return nil, fmt.Errorf("failed get the cluster endpoint: %w", err)
	}

	// Spoof the hostname at the resolver level
	logf("Spoofing %s -> %s", domain, endpoint)
	transport := &http.Transport{
		DialContext: func(ctx context.Context, network, addr string) (conn net.Conn, e error) {
			_, port, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}
			// The original hostname:port is spoofed by replacing the hostname by the value
			// returned by ResolveEndpoint.
			return dialContext(ctx, network, net.JoinHostPort(endpoint, mapper(port)))
		},
	}

	for _, opt := range opts {
		transport = opt(transport)
	}

	// Enable Zipkin tracing
	roundTripper := &ochttp.Transport{
		Base:        transport,
		Propagation: tracecontextb3.TraceContextB3Egress,
	}

	sc := &SpoofingClient{
		Client:          &http.Client{Transport: roundTripper},
		RequestInterval: requestInterval,
		RequestTimeout:  requestTimeout,
		Logf:            logf,
	}
	return sc, nil
}

// ResolveEndpoint resolves the endpoint address considering whether the domain is resolvable and taking into
// account whether the user overrode the endpoint address externally
func ResolveEndpoint(ctx context.Context, kubeClientset *kubernetes.Clientset, domain string, resolvable bool, endpointOverride string) (string, func(string) string, error) {
	id := func(in string) string { return in }
	// If the domain is resolvable, it can be used directly
	if resolvable {
		return domain, id, nil
	}
	// Otherwise, use the actual cluster endpoint
	return ingress.GetIngressEndpoint(ctx, kubeClientset, endpointOverride)
}

// Do dispatches to the underlying http.Client.Do, spoofing domains as needed
// and transforming the http.Response into a spoof.Response.
// Each response is augmented with "ZipkinTraceID" header that identifies the zipkin trace corresponding to the request.
func (sc *SpoofingClient) Do(req *http.Request, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
	return sc.Poll(req, func(*Response) (bool, error) { return true, nil }, errorRetryCheckers...)
}

// Poll executes an http request until it satisfies the inState condition or, if there's an error,
// none of the error retry checkers permit a retry.
// If no retry checkers are specified `DefaultErrorRetryChecker` will be used.
func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker, errorRetryCheckers ...ErrorRetryChecker) (*Response, error) {
	if len(errorRetryCheckers) == 0 {
		errorRetryCheckers = []ErrorRetryChecker{DefaultErrorRetryChecker}
	}

	var resp *Response
	err := wait.PollImmediate(sc.RequestInterval, sc.RequestTimeout, func() (bool, error) {
		// Starting span to capture zipkin trace.
		traceContext, span := trace.StartSpan(req.Context(), "SpoofingClient-Trace")
		defer span.End()
		rawResp, err := sc.Client.Do(req.WithContext(traceContext))
		if err != nil {
			for _, checker := range errorRetryCheckers {
				retry, newErr := checker(err)
				if retry {
					sc.Logf("Retrying %s: %v", req.URL.String(), newErr)
					return false, nil
				}
			}
			sc.Logf("NOT Retrying %s: %v", req.URL.String(), err)
			return true, err
		}
		defer rawResp.Body.Close()

		body, err := ioutil.ReadAll(rawResp.Body)
		if err != nil {
			return true, err
		}
		rawResp.Header.Add(zipkin.ZipkinTraceIDHeader, span.SpanContext().TraceID.String())

		resp = &Response{
			Status:     rawResp.Status,
			StatusCode: rawResp.StatusCode,
			Header:     rawResp.Header,
			Body:       body,
		}
		return inState(resp)
	})

	if resp != nil {
		sc.logZipkinTrace(resp)
	}

	if err != nil {
		return resp, fmt.Errorf("response: %s did not pass checks: %w", resp, err)
	}
	return resp, nil
}

// DefaultErrorRetryChecker implements the defaults for retrying on error.
func DefaultErrorRetryChecker(err error) (bool, error) {
	if isTCPTimeout(err) {
		return true, fmt.Errorf("retrying for TCP timeout: %w", err)
	}
	// Retrying on DNS error, since we may be using xip.io or nip.io in tests.
	if isDNSError(err) {
		return true, fmt.Errorf("retrying for DNS error: %w", err)
	}
	// Repeat the poll on `connection refused` errors, which are usually transient Istio errors.
	if isConnectionRefused(err) {
		return true, fmt.Errorf("retrying for connection refused: %w", err)
	}
	if isConnectionReset(err) {
		return true, fmt.Errorf("retrying for connection reset: %w", err)
	}
	// Retry on connection/network errors.
	if errors.Is(err, io.EOF) {
		return true, fmt.Errorf("retrying for: %w", err)
	}
	return false, err
}

// logZipkinTrace provides support to log Zipkin Trace for param: spoofResponse
// We only log Zipkin trace for HTTP server errors i.e for HTTP status codes between 500 to 600
func (sc *SpoofingClient) logZipkinTrace(spoofResp *Response) {
	if !zipkin.ZipkinTracingEnabled || spoofResp.StatusCode < http.StatusInternalServerError || spoofResp.StatusCode >= 600 {
		return
	}

	traceID := spoofResp.Header.Get(zipkin.ZipkinTraceIDHeader)
	sc.Logf("Logging Zipkin Trace for: %s", traceID)

	json, err := zipkin.JSONTrace(traceID /* We don't know the expected number of spans */, -1, 5*time.Second)
	if err != nil {
		if _, ok := err.(*zipkin.TimeoutError); !ok {
			sc.Logf("Error getting zipkin trace: %v", err)
		}
	}

	sc.Logf("%s", json)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"sort"

	"github.com/aead/siphash"
	"github.com/kkdai/bstream"
)

// Inspired by https://github.com/rasky/gcs

var (
	// ErrNTooBig signifies that the filter can't handle N items.
	ErrNTooBig = fmt.Errorf("N is too big to fit in uint32")

	// ErrPTooBig signifies that the filter can't handle `1/2**P`
	// collision probability.
	ErrPTooBig = fmt.Errorf("P is too big to fit in uint32")

	// ErrNoData signifies that an empty slice was passed.
	ErrNoData = fmt.Errorf("No data provided")
)

const (
	// KeySize is the size of the byte array required for key material for
	// the SipHash keyed hash function.
	KeySize = 16
)

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type Filter struct {
	n          uint32
	p          uint8
	modulusP   uint64
	modulusNP  uint64
	filterData []byte
}

// BuildGCSFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
	// Some initial parameter checks: make sure we have data from which to
	// build the filter, and make sure our parameters will fit the hash
	// function we're using.
	if len(data) == 0 {
		return nil, ErrNoData
	}
	if len(data) > math.MaxInt32 {
		return nil, ErrNTooBig
	}
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := Filter{
		n: uint32(len(data)),
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Build the filter.
	values := make(uint64Slice, 0, len(data))
	b := bstream.NewBStreamWriter(0)

	// Insert the hash (modulo N*P) of each data element into a slice and
	// sort the slice.
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & (f.modulusP - 1)

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.WriteBit(true)
			value--
		}
		b.WriteBit(false)

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.WriteBits(remainder, int(f.p))
	}

	// Copy the bitstream into the filter object and return the object.
	f.filterData = b.Bytes()

	return &f, nil
}

// FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {

	// Basic sanity check.
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := &Filter{
		n: N,
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Copy the filter.
	f.filterData = make([]byte, len(d))
	copy(f.filterData, d)

	return f, nil
}

// FromNBytes deserializes a GCS filter from a known P, and serialized N and
// filter as returned by NBytes().
func FromNBytes(P uint8, d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), P, d[4:])
}

// FromPBytes deserializes a GCS filter from a known N, and serialized P and
// filter as returned by NBytes().
func FromPBytes(N uint32, d []byte) (*Filter, error) {
	return FromBytes(N, d[0], d[1:])
}

// FromNPBytes deserializes a GCS filter from a serialized N, P, and filter as
// returned by NPBytes().
func FromNPBytes(d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), d[4], d[5:])
}

// Bytes returns the serialized format of the GCS filter, which does not
// include N or P (returned by separate methods) or the key used by SipHash.
func (f *Filter) Bytes() []byte {
	filterData := make([]byte, len(f.filterData))
	copy(filterData, f.filterData)
	return filterData
}

// NBytes returns the serialized format of the GCS filter with N, which does
// not include P (returned by a separate method) or the key used by SipHash.
func (f *Filter) NBytes() []byte {
	filterData := make([]byte, len(f.filterData)+4)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	copy(filterData[4:], f.filterData)
	return filterData
}

// PBytes returns the serialized format of the GCS filter with P, which does
// not include N (returned by a separate method) or the key used by SipHash.
func (f *Filter) PBytes() []byte {
	filterData := make([]byte, len(f.filterData)+1)
	filterData[0] = f.p
	copy(filterData[1:], f.filterData)
	return filterData
}

// NPBytes returns the serialized format of the GCS filter with N and P, which
// does not include the key used by SipHash.
func (f *Filter) NPBytes() []byte {
	filterData := make([]byte, len(f.filterData)+5)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	filterData[4] = f.p
	copy(filterData[5:], f.filterData)
	return filterData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *Filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *Filter) N() uint32 {
	return f.n
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Hash our search term with the same parameters as the filter.
	term := siphash.Sum64(data, &key) % f.modulusNP

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue < term {

		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(b)
		if err != nil {
			if err == io.EOF {
				return false, nil
			}
			return false, err
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true, nil
		}

		lastValue = value
	}

	return false, nil
}

// MatchAny returns checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {

	// Basic sanity check.
	if len(data) == 0 {
		return false, ErrNoData
	}

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Create an uncompressed filter of the search values.
	values := make(uint64Slice, 0, len(data))
	for _, d := range data {
		v := siphash.Sum64(d, &key) % f.modulusNP
		values = append(values, v)
	}
	sort.Sort(values)

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	var lastValue1, lastValue2 uint64
	lastValue2 = values[0]
	i := 1
	for lastValue1 != lastValue2 {
		// Check which filter to advance to make sure we're comparing
		// the right values.
		switch {
		case lastValue1 > lastValue2:
			// Advance filter created from search terms or return
			// false if we're at the end because nothing matched.
			if i < len(values) {
				lastValue2 = values[i]
				i++
			} else {
				return false, nil
			}
		case lastValue2 > lastValue1:
			// Advance filter we're searching or return false if
			// we're at the end because nothing matched.
			value, err := f.readFullUint64(b)
			if err != nil {
				if err == io.EOF {
					return false, nil
				}
				return false, err
			}
			lastValue1 += value
		}
	}

	// If we've made it this far, an element matched between filters so we
	// return true.
	return true, nil
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) {
	var v uint64

	// Count the 1s until we reach a 0.
	c, err := b.ReadBit()
	if err != nil {
		return 0, err
	}
	for c {
		v++
		c, err = b.ReadBit()
		if err != nil {
			return 0, err
		}
	}

	// Read P bits.
	remainder, err := b.ReadBits(int(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	v = v*f.modulusP + remainder
	return v, nil
}
<commit_msg>gcs: update modulo algorithm to fast range per gmaxwell's suggestion<commit_after>// Copyright (c) 2016-2017 The btcsuite developers
// Copyright (c) 2016-2017 The Lightning Network Developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package gcs

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"
	"sort"

	"github.com/aead/siphash"
	"github.com/kkdai/bstream"
)

// Inspired by https://github.com/rasky/gcs

var (
	// ErrNTooBig signifies that the filter can't handle N items.
	ErrNTooBig = fmt.Errorf("N is too big to fit in uint32")

	// ErrPTooBig signifies that the filter can't handle `1/2**P`
	// collision probability.
	ErrPTooBig = fmt.Errorf("P is too big to fit in uint32")

	// ErrNoData signifies that an empty slice was passed.
	ErrNoData = fmt.Errorf("No data provided")
)

const (
	// KeySize is the size of the byte array required for key material for
	// the SipHash keyed hash function.
	KeySize = 16
)

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
// include N or P to allow the user to encode the metadata separately if
// necessary. The hash function used is SipHash, a keyed function; the key used
// in building the filter is required in order to match filter values and is
// not included in the serialized form.
type Filter struct {
	n          uint32
	p          uint8
	modulusP   uint64
	modulusNP  uint64
	filterData []byte
}

// BuildGCSFilter builds a new GCS filter with the collision probability of
// `1/(2**P)`, key `key`, and including every `[]byte` in `data` as a member of
// the set.
func BuildGCSFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
	// Some initial parameter checks: make sure we have data from which to
	// build the filter, and make sure our parameters will fit the hash
	// function we're using.
	if len(data) == 0 {
		return nil, ErrNoData
	}
	if len(data) > math.MaxInt32 {
		return nil, ErrNTooBig
	}
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := Filter{
		n: uint32(len(data)),
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Build the filter.
	values := make(uint64Slice, 0, len(data))
	b := bstream.NewBStreamWriter(0)
	// Insert the hash (fast-ranged over a space of N*P) of each data
	// element into a slice and sort the slice. This can be greatly
	// optimized with native 128-bit multiplication, but we're going to be
	// fully portable for now.
	var v, vhi, vlo, nphi, nplo, vnphi, vnpmid, npvmid, vnplo, carry uint64
	// First, we cache the high and low bits of modulusNP for the
	// multiplication of 2 64-bit integers into a 128-bit integer.
	nphi = f.modulusNP >> 32
	nplo = uint64(uint32(f.modulusNP))
	for _, d := range data {
		// For each datum, we assign the initial hash to a uint64.
		v = siphash.Sum64(d, &key)
		// Then, we split it into high bits and low bits.
		vhi = v >> 32
		vlo = uint64(uint32(v))
		// Then, we distribute multiplication over each part.
		vnphi = vhi * nphi
		vnpmid = vhi * nplo
		npvmid = nphi * vlo
		vnplo = vlo * nplo
		// We calculate the carry bit.
		carry = (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
			(vnplo >> 32)) >> 32
		// Last, we add the high bits, the middle bits, and the carry.
		v = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry
		values = append(values, v)
	}
	sort.Sort(values)

	// Write the sorted list of values into the filter bitstream,
	// compressing it using Golomb coding.
	var value, lastValue, remainder uint64
	for _, v := range values {
		// Calculate the difference between this value and the last,
		// modulo P.
		remainder = (v - lastValue) & (f.modulusP - 1)

		// Calculate the difference between this value and the last,
		// divided by P.
		value = (v - lastValue - remainder) >> f.p
		lastValue = v

		// Write the P multiple into the bitstream in unary; the
		// average should be around 1 (2 bits - 0b10).
		for value > 0 {
			b.WriteBit(true)
			value--
		}
		b.WriteBit(false)

		// Write the remainder as a big-endian integer with enough bits
		// to represent the appropriate collision probability.
		b.WriteBits(remainder, int(f.p))
	}

	// Copy the bitstream into the filter object and return the object.
	f.filterData = b.Bytes()

	return &f, nil
}

// FromBytes deserializes a GCS filter from a known N, P, and serialized filter
// as returned by Bytes().
func FromBytes(N uint32, P uint8, d []byte) (*Filter, error) {

	// Basic sanity check.
	if P > 32 {
		return nil, ErrPTooBig
	}

	// Create the filter object and insert metadata.
	f := &Filter{
		n: N,
		p: P,
	}
	f.modulusP = uint64(1 << f.p)
	f.modulusNP = uint64(f.n) * f.modulusP

	// Copy the filter.
	f.filterData = make([]byte, len(d))
	copy(f.filterData, d)

	return f, nil
}

// FromNBytes deserializes a GCS filter from a known P, and serialized N and
// filter as returned by NBytes().
func FromNBytes(P uint8, d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), P, d[4:])
}

// FromPBytes deserializes a GCS filter from a known N, and serialized P and
// filter as returned by NBytes().
func FromPBytes(N uint32, d []byte) (*Filter, error) {
	return FromBytes(N, d[0], d[1:])
}

// FromNPBytes deserializes a GCS filter from a serialized N, P, and filter as
// returned by NPBytes().
func FromNPBytes(d []byte) (*Filter, error) {
	return FromBytes(binary.BigEndian.Uint32(d[:4]), d[4], d[5:])
}

// Bytes returns the serialized format of the GCS filter, which does not
// include N or P (returned by separate methods) or the key used by SipHash.
func (f *Filter) Bytes() []byte {
	filterData := make([]byte, len(f.filterData))
	copy(filterData, f.filterData)
	return filterData
}

// NBytes returns the serialized format of the GCS filter with N, which does
// not include P (returned by a separate method) or the key used by SipHash.
func (f *Filter) NBytes() []byte {
	filterData := make([]byte, len(f.filterData)+4)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	copy(filterData[4:], f.filterData)
	return filterData
}

// PBytes returns the serialized format of the GCS filter with P, which does
// not include N (returned by a separate method) or the key used by SipHash.
func (f *Filter) PBytes() []byte {
	filterData := make([]byte, len(f.filterData)+1)
	filterData[0] = f.p
	copy(filterData[1:], f.filterData)
	return filterData
}

// NPBytes returns the serialized format of the GCS filter with N and P, which
// does not include the key used by SipHash.
func (f *Filter) NPBytes() []byte {
	filterData := make([]byte, len(f.filterData)+5)
	binary.BigEndian.PutUint32(filterData[:4], f.n)
	filterData[4] = f.p
	copy(filterData[5:], f.filterData)
	return filterData
}

// P returns the filter's collision probability as a negative power of 2 (that
// is, a collision probability of `1/2**20` is represented as 20).
func (f *Filter) P() uint8 {
	return f.p
}

// N returns the size of the data set used to build the filter.
func (f *Filter) N() uint32 {
	return f.n
}

// Match checks whether a []byte value is likely (within collision probability)
// to be a member of the set represented by the filter.
func (f *Filter) Match(key [KeySize]byte, data []byte) (bool, error) {

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// We take the high and low bits of modulusNP for the multiplication
	// of 2 64-bit integers into a 128-bit integer.
	nphi := f.modulusNP >> 32
	nplo := uint64(uint32(f.modulusNP))
	// Then we hash our search term with the same parameters as the filter.
	term := siphash.Sum64(data, &key)
	// Then, we split it into high bits and low bits.
	vhi := term >> 32
	vlo := uint64(uint32(term))
	// Then, we distribute multiplication over each part.
	vnphi := vhi * nphi
	vnpmid := vhi * nplo
	npvmid := nphi * vlo
	vnplo := vlo * nplo
	// We calculate the carry bit.
	carry := (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
		(vnplo >> 32)) >> 32
	// Last, we add the high bits, the middle bits, and the carry.
	term = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry

	// Go through the search filter and look for the desired value.
	var lastValue uint64
	for lastValue < term {

		// Read the difference between previous and new value from
		// bitstream.
		value, err := f.readFullUint64(b)
		if err != nil {
			if err == io.EOF {
				return false, nil
			}
			return false, err
		}

		// Add the previous value to it.
		value += lastValue
		if value == term {
			return true, nil
		}

		lastValue = value
	}

	return false, nil
}

// MatchAny returns checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) (bool, error) {

	// Basic sanity check.
	if len(data) == 0 {
		return false, ErrNoData
	}

	// Create a filter bitstream.
	filterData := f.Bytes()
	b := bstream.NewBStreamReader(filterData)

	// Create an uncompressed filter of the search values.
	values := make(uint64Slice, 0, len(data))
	var v, vhi, vlo, nphi, nplo, vnphi, vnpmid, npvmid, vnplo, carry uint64
	// First, we cache the high and low bits of modulusNP for the
	// multiplication of 2 64-bit integers into a 128-bit integer.
	nphi = f.modulusNP >> 32
	nplo = uint64(uint32(f.modulusNP))
	for _, d := range data {
		// For each datum, we assign the initial hash to a uint64.
		v = siphash.Sum64(d, &key)
		// Then, we split it into high bits and low bits.
		vhi = v >> 32
		vlo = uint64(uint32(v))
		// Then, we distribute multiplication over each part.
		vnphi = vhi * nphi
		vnpmid = vhi * nplo
		npvmid = nphi * vlo
		vnplo = vlo * nplo
		// We calculate the carry bit.
		carry = (uint64(uint32(vnpmid)) + uint64(uint32(npvmid)) +
			(vnplo >> 32)) >> 32
		// Last, we add the high bits, the middle bits, and the carry.
		v = vnphi + (vnpmid >> 32) + (npvmid >> 32) + carry
		values = append(values, v)
	}
	sort.Sort(values)

	// Zip down the filters, comparing values until we either run out of
	// values to compare in one of the filters or we reach a matching
	// value.
	var lastValue1, lastValue2 uint64
	lastValue2 = values[0]
	i := 1
	for lastValue1 != lastValue2 {
		// Check which filter to advance to make sure we're comparing
		// the right values.
		switch {
		case lastValue1 > lastValue2:
			// Advance filter created from search terms or return
			// false if we're at the end because nothing matched.
			if i < len(values) {
				lastValue2 = values[i]
				i++
			} else {
				return false, nil
			}
		case lastValue2 > lastValue1:
			// Advance filter we're searching or return false if
			// we're at the end because nothing matched.
			value, err := f.readFullUint64(b)
			if err != nil {
				if err == io.EOF {
					return false, nil
				}
				return false, err
			}
			lastValue1 += value
		}
	}

	// If we've made it this far, an element matched between filters so we
	// return true.
	return true, nil
}

// readFullUint64 reads a value represented by the sum of a unary multiple of
// the filter's P modulus (`2**P`) and a big-endian P-bit remainder.
func (f *Filter) readFullUint64(b *bstream.BStream) (uint64, error) {
	var v uint64

	// Count the 1s until we reach a 0.
	c, err := b.ReadBit()
	if err != nil {
		return 0, err
	}
	for c {
		v++
		c, err = b.ReadBit()
		if err != nil {
			return 0, err
		}
	}

	// Read P bits.
	remainder, err := b.ReadBits(int(f.p))
	if err != nil {
		return 0, err
	}

	// Add the multiple and the remainder.
	v = v*f.modulusP + remainder
	return v, nil
}
<|endoftext|>


<commit_before>package scanner

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
	// If IgnorePerms is true, changes to permission bits will not be
	// detected. Scanned files will get zero permission bits and the
	// NoPermissionBits flag set.
	IgnorePerms bool
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := strings.Trim(pn, "/")
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				if len(line) > 0 {
					patterns = append(patterns, string(line))
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
			for _, pattern := range pats {
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Stat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	}
	return nil
}

func PermsEqual(a, b uint32) bool {
	switch runtime.GOOS {
	case "windows":
		// There is only writeable and read only, represented for user, group
		// and other equally. We only compare against user.
		return a&0600 == b&0600
	default:
		// All bits count
		return a&0777 == b&0777
	}
}
<commit_msg>Send default permissions 0777 on directories when IgnorePerms set (ref #284)<commit_after>package scanner

import (
	"bytes"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
)

type Walker struct {
	// Dir is the base directory for the walk
	Dir string
	// BlockSize controls the size of the block used when hashing.
	BlockSize int
	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
	IgnoreFile string
	// If TempNamer is not nil, it is used to ignore tempory files when walking.
	TempNamer TempNamer
	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
	CurrentFiler CurrentFiler
	// If Suppressor is not nil, it is queried for supression of modified files.
	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
	// Requires CurrentFiler to be set.
	Suppressor Suppressor
	// If IgnorePerms is true, changes to permission bits will not be
	// detected. Scanned files will get zero permission bits and the
	// NoPermissionBits flag set.
	IgnorePerms bool
}

type TempNamer interface {
	// Temporary returns a temporary name for the filed referred to by filepath.
	TempName(path string) string
	// IsTemporary returns true if path refers to the name of temporary file.
	IsTemporary(path string) bool
}

type Suppressor interface {
	// Supress returns true if the update to the named file should be ignored.
	Suppress(name string, fi os.FileInfo) (bool, bool)
}

type CurrentFiler interface {
	// CurrentFile returns the file as seen at last scan.
	CurrentFile(name string) File
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
	if debug {
		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
	}

	err = checkDir(w.Dir)
	if err != nil {
		return
	}

	t0 := time.Now()

	ignore = make(map[string][]string)
	hashFiles := w.walkAndHashFiles(&files, ignore)

	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
	filepath.Walk(w.Dir, hashFiles)

	if debug {
		t1 := time.Now()
		d := t1.Sub(t0).Seconds()
		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
	}

	err = checkDir(w.Dir)
	return
}

// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
	filepath.Walk(w.Dir, w.cleanTempFile)
}

func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		rn, err := filepath.Rel(dir, p)
		if err != nil {
			return nil
		}

		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
			pn := strings.Trim(pn, "/")
			bs, _ := ioutil.ReadFile(p)
			lines := bytes.Split(bs, []byte("\n"))
			var patterns []string
			for _, line := range lines {
				if len(line) > 0 {
					patterns = append(patterns, string(line))
				}
			}
			ign[pn] = patterns
		}

		return nil
	}
}

func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
	return func(p string, info os.FileInfo, err error) error {
		if err != nil {
			if debug {
				l.Debugln("error:", p, info, err)
			}
			return nil
		}

		rn, err := filepath.Rel(w.Dir, p)
		if err != nil {
			if debug {
				l.Debugln("rel error:", p, err)
			}
			return nil
		}

		if rn == "." {
			return nil
		}

		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
			// A temporary file
			if debug {
				l.Debugln("temporary:", rn)
			}
			return nil
		}

		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
			// An ignored file
			if debug {
				l.Debugln("ignored:", rn)
			}
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if info.Mode().IsDir() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if cf.Modified == info.ModTime().Unix() && protocol.IsDirectory(cf.Flags) && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
				} else {
					var flags uint32 = protocol.FlagDirectory
					if w.IgnorePerms {
						flags |= protocol.FlagNoPermBits | 0777
					} else {
						flags |= uint32(info.Mode() & os.ModePerm)
					}
					f := File{
						Name:     rn,
						Version:  lamport.Default.Tick(0),
						Flags:    flags,
						Modified: info.ModTime().Unix(),
					}
					if debug {
						l.Debugln("dir:", cf, f)
					}
					*res = append(*res, f)
				}
				return nil
			}
		}

		if info.Mode().IsRegular() {
			if w.CurrentFiler != nil {
				cf := w.CurrentFiler.CurrentFile(rn)
				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
					if debug {
						l.Debugln("unchanged:", cf)
					}
					*res = append(*res, cf)
					return nil
				}

				if w.Suppressor != nil {
					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
						cf.Suppressed = true
						cf.Version++
						if debug {
							l.Debugln("suppressed:", cf)
						}
						*res = append(*res, cf)
						return nil
					} else if prev && !cur {
						l.Infof("Changes to %q are no longer suppressed.", p)
					}
				}

				if debug {
					l.Debugln("rescan:", cf, info.ModTime().Unix(), info.Mode()&os.ModePerm)
				}
			}

			fd, err := os.Open(p)
			if err != nil {
				if debug {
					l.Debugln("open:", p, err)
				}
				return nil
			}
			defer fd.Close()

			t0 := time.Now()
			blocks, err := Blocks(fd, w.BlockSize)
			if err != nil {
				if debug {
					l.Debugln("hash error:", rn, err)
				}
				return nil
			}
			if debug {
				t1 := time.Now()
				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
			}

			var flags = uint32(info.Mode() & os.ModePerm)
			if w.IgnorePerms {
				flags = protocol.FlagNoPermBits | 0666
			}
			f := File{
				Name:     rn,
				Version:  lamport.Default.Tick(0),
				Size:     info.Size(),
				Flags:    flags,
				Modified: info.ModTime().Unix(),
				Blocks:   blocks,
			}
			*res = append(*res, f)
		}

		return nil
	}
}

func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
		os.Remove(path)
	}
	return nil
}

func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
	first, last := filepath.Split(file)
	for prefix, pats := range patterns {
		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
			for _, pattern := range pats {
				if match, _ := filepath.Match(pattern, last); match {
					return true
				}
			}
		}
	}
	return false
}

func checkDir(dir string) error {
	if info, err := os.Stat(dir); err != nil {
		return err
	} else if !info.IsDir() {
		return errors.New(dir + ": not a directory")
	}
	return nil
}

func PermsEqual(a, b uint32) bool {
	switch runtime.GOOS {
	case "windows":
		// There is only writeable and read only, represented for user, group
		// and other equally. We only compare against user.
		return a&0600 == b&0600
	default:
		// All bits count
		return a&0777 == b&0777
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"gesture/plugin"
	"gesture/plugin/identity"
	"gesture/plugin/gis"
	"gesture/plugin/twitter"
	"gesture/rewrite"
	irc "github.com/fluffle/goirc/client"
	"log"
	"os"
	"strings"
	"io/ioutil"
)

var (
	plugins  []plugin.Plugin
)

// gesture config
type Config struct {
	BotName  string
	Hostname string
	SSL bool
	Channels []string
}

// readsConfig unmarshals the config from a file and returns the struct
func readConfig(filename string) (*Config, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var config Config
	b, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, &config); err != nil {
		return nil, err
	}
	return &config, nil

}

// a Plugin is something that can respond to messages
func main() {
	if len(os.Args) < 2 {
		log.Println("usage: gesture [conf_file]")
		os.Exit(1)
	}

	config, err := readConfig(os.Args[1])
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	plugins = []plugin.Plugin{
		twitter.NewPlugin(),
		gis.NewPlugin(),
		identity.NewPlugin(config.BotName)}

	flag.Parse()
	c := irc.SimpleClient(config.BotName)
	c.SSL = config.SSL
	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				conn.Join(channel)
			}
		})
	quit := make(chan bool)
	c.AddHandler(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) { quit <- true })
	c.AddHandler("PRIVMSG", func(conn *irc.Conn, line *irc.Line) {
		messageReceived(conn, line)
	})
	if err := c.Connect(config.Hostname); err != nil {
		fmt.Printf("Connection error: %s\n", err)
	}
	// Wait for disconnect
	<-quit
}

// When a message comes in on a channel gesture has joined, this method will be called.
func messageReceived(conn *irc.Conn, line *irc.Line) {
	if len(line.Args) > 1 {
		channel := line.Args[0]
		message := line.Args[1]

		mc := &messageContext{conn, line}

		log.Printf(">> %s (%s): %s\n", line.Nick, channel, message)

		handled := false
		for _, plugin := range plugins {
			success, err := plugin.Call(mc)
			if err != nil {
				log.Print(err)
			}
			if success {
				handled = true
				break
			}
		}
		if !handled {
			// try to expand any links
			for _, token := range rewrite.GetRewrittenLinks(mc.Message()) {
				mc.Reply(token)
			}
		}
	}
}

// when an error occurs, calling this method will send the error back to the irc channel
func sendError(conn *irc.Conn, channel string, nick string, err error) {
	log.Print(err)
	conn.Privmsg(channel, fmt.Sprintf("%s: oops: %v", nick, err))
}

type messageContext struct {
	conn *irc.Conn
	line *irc.Line
}

func (mc *messageContext) Message() string {
	if len(mc.line.Args) > 1 {
		return mc.line.Args[1]
	}
	return ""
}

func (mc *messageContext) Command() string {
	sliced := strings.Split(mc.Message(), " ")
	return sliced[0]
}

func (mc *messageContext) CommandArgs() []string {
	sliced := strings.Split(mc.Message(), " ")
	return sliced[1:]
}

func (mc *messageContext) Reply(message string) {
	channel := mc.line.Args[0]
	mc.conn.Privmsg(channel, fmt.Sprintf("%s: ftfy -> %s", mc.line.Nick, rewrite.Rewrite(message)))
}

func (mc *messageContext) Send(message string) {
	channel := mc.line.Args[0]
	mc.conn.Privmsg(channel, rewrite.Rewrite(message))
}
<commit_msg>Panic on connect failure<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"gesture/plugin"
	"gesture/plugin/identity"
	"gesture/plugin/gis"
	"gesture/plugin/twitter"
	"gesture/rewrite"
	irc "github.com/fluffle/goirc/client"
	"log"
	"os"
	"strings"
	"io/ioutil"
)

var (
	plugins  []plugin.Plugin
)

// gesture config
type Config struct {
	BotName  string
	Hostname string
	SSL bool
	Channels []string
}

// readsConfig unmarshals the config from a file and returns the struct
func readConfig(filename string) (*Config, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	var config Config
	b, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}
	if err = json.Unmarshal(b, &config); err != nil {
		return nil, err
	}
	return &config, nil

}

// a Plugin is something that can respond to messages
func main() {
	if len(os.Args) < 2 {
		log.Println("usage: gesture [conf_file]")
		os.Exit(1)
	}

	config, err := readConfig(os.Args[1])
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	plugins = []plugin.Plugin{
		twitter.NewPlugin(),
		gis.NewPlugin(),
		identity.NewPlugin(config.BotName),
	}

	flag.Parse()
	c := irc.SimpleClient(config.BotName)
	c.SSL = config.SSL
	c.AddHandler(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				conn.Join(channel)
			}
		})
	quit := make(chan bool)
	c.AddHandler(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) { quit <- true })
	c.AddHandler("PRIVMSG", func(conn *irc.Conn, line *irc.Line) {
		messageReceived(conn, line)
	})
	if err := c.Connect(config.Hostname); err != nil {
		log.Fatalf("Connection error: %s\n", err)
	}
	// Wait for disconnect
	<-quit
}

// When a message comes in on a channel gesture has joined, this method will be called.
func messageReceived(conn *irc.Conn, line *irc.Line) {
	if len(line.Args) > 1 {
		channel := line.Args[0]
		message := line.Args[1]

		mc := &messageContext{conn, line}

		log.Printf(">> %s (%s): %s\n", line.Nick, channel, message)

		handled := false
		for _, plugin := range plugins {
			success, err := plugin.Call(mc)
			if err != nil {
				log.Print(err)
			}
			if success {
				handled = true
				break
			}
		}
		if !handled {
			// try to expand any links
			for _, token := range rewrite.GetRewrittenLinks(mc.Message()) {
				mc.Reply(token)
			}
		}
	}
}

// when an error occurs, calling this method will send the error back to the irc channel
func sendError(conn *irc.Conn, channel string, nick string, err error) {
	log.Print(err)
	conn.Privmsg(channel, fmt.Sprintf("%s: oops: %v", nick, err))
}

type messageContext struct {
	conn *irc.Conn
	line *irc.Line
}

func (mc *messageContext) Message() string {
	if len(mc.line.Args) > 1 {
		return mc.line.Args[1]
	}
	return ""
}

func (mc *messageContext) Command() string {
	sliced := strings.Split(mc.Message(), " ")
	return sliced[0]
}

func (mc *messageContext) CommandArgs() []string {
	sliced := strings.Split(mc.Message(), " ")
	return sliced[1:]
}

func (mc *messageContext) Reply(message string) {
	channel := mc.line.Args[0]
	mc.conn.Privmsg(channel, fmt.Sprintf("%s: ftfy -> %s", mc.line.Nick, rewrite.Rewrite(message)))
}

func (mc *messageContext) Send(message string) {
	channel := mc.line.Args[0]
	mc.conn.Privmsg(channel, rewrite.Rewrite(message))
}
<|endoftext|>


<commit_before>package gotwilio

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"bytes"
)

// These are the paramters to use when you want Twilio to use callback urls.
// See http://www.twilio.com/docs/api/rest/making-calls for more info.
type CallbackParameters struct {
	Url                           string   // Required
	Method                        string   // Optional
	FallbackUrl                   string   // Optional
	FallbackMethod                string   // Optional
	StatusCallback                string   // Optional
	StatusCallbackMethod          string   // Optional
	StatusCallbackEvent           []string // Optional
	SendDigits                    string   // Optional
	IfMachine                     string   // False, Continue or Hangup; http://www.twilio.com/docs/errors/21207
	Timeout                       int      // Optional
	Record                        bool     // Optional
	RecordingChannels             string   // Optional
	RecordingStatusCallback       string   // Optional
	RecordingStatusCallbackMethod string   // Optional
	MachineDetection              string   // Optional
	MachineDetectionTimeout       int      // Optional
}

// VoiceResponse contains the details about successful voice calls.
type VoiceResponse struct {
	Sid            string   `json:"sid"`
	DateCreated    string   `json:"date_created"`
	DateUpdated    string   `json:"date_updated"`
	ParentCallSid  string   `json:"parent_call_sid"`
	AccountSid     string   `json:"account_sid"`
	To             string   `json:"to"`
	ToFormatted    string   `json:"to_formatted"`
	From           string   `json:"from"`
	FromFormatted  string   `json:"from_formatted"`
	PhoneNumberSid string   `json:"phone_number_sid"`
	Status         string   `json:"status"`
	StartTime      string   `json:"start_time"`
	EndTime        string   `json:"end_time"`
	Duration       int      `json:"duration,string"`
	Price          *float32 `json:"price,omitempty"`
	Direction      string   `json:"direction"`
	AnsweredBy     string   `json:"answered_by"`
	ApiVersion     string   `json:"api_version"`
	Annotation     string   `json:"annotation"`
	ForwardedFrom  string   `json:"forwarded_from"`
	GroupSid       string   `json:"group_sid"`
	CallerName     string   `json:"caller_name"`
	Uri            string   `json:"uri"`
	// TODO: handle SubresourceUris
	// TODO: handle annotation
	// TODO: handle price_unit
}

// Returns VoiceResponse.DateCreated as a time.Time object
// instead of a string.
func (vr *VoiceResponse) DateCreatedAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.DateCreated)
}

// Returns VoiceResponse.DateUpdated as a time.Time object
// instead of a string.
func (vr *VoiceResponse) DateUpdatedAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.DateUpdated)
}

// Returns VoiceResponse.StartTime as a time.Time object
// instead of a string.
func (vr *VoiceResponse) StartTimeAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.StartTime)
}

// Returns VoiceResponse.EndTime as a time.Time object
// instead of a string.
func (vr *VoiceResponse) EndTimeAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.EndTime)
}

// Returns a CallbackParameters type with the specified url and
// CallbackParameters.Timeout set to 60.
func NewCallbackParameters(url string) *CallbackParameters {
	return &CallbackParameters{Url: url, Timeout: 60}
}

// GetCall uses Twilio to get information about a voice call.
// See https://www.twilio.com/docs/voice/api/call
func (twilio *Twilio) GetCall(sid string) (*VoiceResponse, *Exception, error) {
	var voiceResponse *VoiceResponse
	var exception *Exception
	twilioUrl := twilio.BaseUrl + "/Accounts/" + twilio.AccountSid + "/Calls/" + sid + ".json"

	res, err := twilio.get(twilioUrl)
	if err != nil {
		return nil, nil, err
	}
	defer res.Body.Close()

	responseBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode != http.StatusOK {
		exception = new(Exception)
		err = json.Unmarshal(responseBody, exception)
		return nil, exception, err
	}

	voiceResponse = new(VoiceResponse)
	err = json.Unmarshal(responseBody, voiceResponse)
	return voiceResponse, nil, err
}

// Place a voice call with a list of callbacks specified.
func (twilio *Twilio) CallWithUrlCallbacks(from, to string, callbackParameters *CallbackParameters) (*VoiceResponse, *Exception, error) {
	formValues := url.Values{}
	formValues.Set("From", from)
	formValues.Set("To", to)
	formValues.Set("Url", callbackParameters.Url)

	// Optional values
	if callbackParameters.Method != "" {
		formValues.Set("Method", callbackParameters.Method)
	}
	if callbackParameters.FallbackUrl != "" {
		formValues.Set("FallbackUrl", callbackParameters.FallbackUrl)
	}
	if callbackParameters.FallbackMethod != "" {
		formValues.Set("FallbackMethod", callbackParameters.FallbackMethod)
	}
	if callbackParameters.StatusCallback != "" {
		formValues.Set("StatusCallback", callbackParameters.StatusCallback)
	}
	if callbackParameters.StatusCallbackMethod != "" {
		formValues.Set("StatusCallbackMethod", callbackParameters.StatusCallbackMethod)
	}
	for _, event := range callbackParameters.StatusCallbackEvent {
		formValues.Add("StatusCallbackEvent", event)
	}
	if callbackParameters.SendDigits != "" {
		formValues.Set("SendDigits", callbackParameters.SendDigits)
	}
	if callbackParameters.IfMachine != "" {
		formValues.Set("IfMachine", callbackParameters.IfMachine)
	}
	if callbackParameters.Timeout != 0 {
		formValues.Set("Timeout", strconv.Itoa(callbackParameters.Timeout))
	}
	if callbackParameters.MachineDetection != "" {
		formValues.Set("MachineDetection", callbackParameters.MachineDetection)
	}
	if callbackParameters.MachineDetectionTimeout != 0 {
		formValues.Set(
			"MachineDetectionTimeout",
			strconv.Itoa(callbackParameters.MachineDetectionTimeout),
		)
	}

	if callbackParameters.Record {
		formValues.Set("Record", "true")

		if callbackParameters.RecordingChannels != "" {
			formValues.Set("RecordingChannels", callbackParameters.RecordingChannels)
		}
		if callbackParameters.RecordingStatusCallback != "" {
			formValues.Set("RecordingStatusCallback", callbackParameters.RecordingStatusCallback)
		}
		if callbackParameters.RecordingStatusCallbackMethod != "" {
			formValues.Set("RecordingStatusCallbackMethod", callbackParameters.RecordingStatusCallbackMethod)
		}
	} else {
		formValues.Set("Record", "false")
	}

	return twilio.voicePost("Calls.json", formValues)
}

// Place a voice call with an ApplicationSid specified.
func (twilio *Twilio) CallWithApplicationCallbacks(from, to, applicationSid string) (*VoiceResponse, *Exception, error) {
	formValues := url.Values{}
	formValues.Set("From", from)
	formValues.Set("To", to)
	formValues.Set("ApplicationSid", applicationSid)

	return twilio.voicePost("Calls.json", formValues)
}

// Update an existing call
func (twilio *Twilio) CallUpdate(callSid string, formValues url.Values) (*VoiceResponse, *Exception, error) {
	return twilio.voicePost("Calls/"+callSid+".json", formValues)
}

// This is a private method that has the common bits for placing or updating a voice call.
func (twilio *Twilio) voicePost(resourcePath string, formValues url.Values) (*VoiceResponse, *Exception, error) {
	var voiceResponse *VoiceResponse
	var exception *Exception

	twilioUrl := twilio.buildUrl(resourcePath)
	res, err := twilio.post(formValues, twilioUrl)
	if err != nil {
		return voiceResponse, exception, err
	}
	defer res.Body.Close()

	body := new(bytes.Buffer)
	_, err = body.ReadFrom(res.Body)
	if err != nil {
		return voiceResponse, exception, err
	}

	responseStr := body.String()

	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted && res.StatusCode != http.StatusCreated {
		exception = new(Exception)
		err = json.Unmarshal([]byte(responseStr), exception)

		// We aren't checking the error because we don't actually care.
		// It's going to be passed to the client either way.
		return voiceResponse, exception, err
	}

	voiceResponse = new(VoiceResponse)
	err = json.Unmarshal([]byte(responseStr), voiceResponse)
	return voiceResponse, exception, err
}
<commit_msg>convert back to JSON decoder<commit_after>package gotwilio

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

// These are the paramters to use when you want Twilio to use callback urls.
// See http://www.twilio.com/docs/api/rest/making-calls for more info.
type CallbackParameters struct {
	Url                           string   // Required
	Method                        string   // Optional
	FallbackUrl                   string   // Optional
	FallbackMethod                string   // Optional
	StatusCallback                string   // Optional
	StatusCallbackMethod          string   // Optional
	StatusCallbackEvent           []string // Optional
	SendDigits                    string   // Optional
	IfMachine                     string   // False, Continue or Hangup; http://www.twilio.com/docs/errors/21207
	Timeout                       int      // Optional
	Record                        bool     // Optional
	RecordingChannels             string   // Optional
	RecordingStatusCallback       string   // Optional
	RecordingStatusCallbackMethod string   // Optional
	MachineDetection              string   // Optional
	MachineDetectionTimeout       int      // Optional
}

// VoiceResponse contains the details about successful voice calls.
type VoiceResponse struct {
	Sid            string   `json:"sid"`
	DateCreated    string   `json:"date_created"`
	DateUpdated    string   `json:"date_updated"`
	ParentCallSid  string   `json:"parent_call_sid"`
	AccountSid     string   `json:"account_sid"`
	To             string   `json:"to"`
	ToFormatted    string   `json:"to_formatted"`
	From           string   `json:"from"`
	FromFormatted  string   `json:"from_formatted"`
	PhoneNumberSid string   `json:"phone_number_sid"`
	Status         string   `json:"status"`
	StartTime      string   `json:"start_time"`
	EndTime        string   `json:"end_time"`
	Duration       int      `json:"duration,string"`
	Price          *float32 `json:"price,omitempty"`
	Direction      string   `json:"direction"`
	AnsweredBy     string   `json:"answered_by"`
	ApiVersion     string   `json:"api_version"`
	Annotation     string   `json:"annotation"`
	ForwardedFrom  string   `json:"forwarded_from"`
	GroupSid       string   `json:"group_sid"`
	CallerName     string   `json:"caller_name"`
	Uri            string   `json:"uri"`
	// TODO: handle SubresourceUris
	// TODO: handle annotation
	// TODO: handle price_unit
}

// Returns VoiceResponse.DateCreated as a time.Time object
// instead of a string.
func (vr *VoiceResponse) DateCreatedAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.DateCreated)
}

// Returns VoiceResponse.DateUpdated as a time.Time object
// instead of a string.
func (vr *VoiceResponse) DateUpdatedAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.DateUpdated)
}

// Returns VoiceResponse.StartTime as a time.Time object
// instead of a string.
func (vr *VoiceResponse) StartTimeAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.StartTime)
}

// Returns VoiceResponse.EndTime as a time.Time object
// instead of a string.
func (vr *VoiceResponse) EndTimeAsTime() (time.Time, error) {
	return time.Parse(time.RFC1123Z, vr.EndTime)
}

// Returns a CallbackParameters type with the specified url and
// CallbackParameters.Timeout set to 60.
func NewCallbackParameters(url string) *CallbackParameters {
	return &CallbackParameters{Url: url, Timeout: 60}
}

// GetCall uses Twilio to get information about a voice call.
// See https://www.twilio.com/docs/voice/api/call
func (twilio *Twilio) GetCall(sid string) (*VoiceResponse, *Exception, error) {
	var voiceResponse *VoiceResponse
	var exception *Exception
	twilioUrl := twilio.BaseUrl + "/Accounts/" + twilio.AccountSid + "/Calls/" + sid + ".json"

	res, err := twilio.get(twilioUrl)
	if err != nil {
		return nil, nil, err
	}
	defer res.Body.Close()

	responseBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, nil, err
	}
	if res.StatusCode != http.StatusOK {
		exception = new(Exception)
		err = json.Unmarshal(responseBody, exception)
		return nil, exception, err
	}

	voiceResponse = new(VoiceResponse)
	err = json.Unmarshal(responseBody, voiceResponse)
	return voiceResponse, nil, err
}

// Place a voice call with a list of callbacks specified.
func (twilio *Twilio) CallWithUrlCallbacks(from, to string, callbackParameters *CallbackParameters) (*VoiceResponse, *Exception, error) {
	formValues := url.Values{}
	formValues.Set("From", from)
	formValues.Set("To", to)
	formValues.Set("Url", callbackParameters.Url)

	// Optional values
	if callbackParameters.Method != "" {
		formValues.Set("Method", callbackParameters.Method)
	}
	if callbackParameters.FallbackUrl != "" {
		formValues.Set("FallbackUrl", callbackParameters.FallbackUrl)
	}
	if callbackParameters.FallbackMethod != "" {
		formValues.Set("FallbackMethod", callbackParameters.FallbackMethod)
	}
	if callbackParameters.StatusCallback != "" {
		formValues.Set("StatusCallback", callbackParameters.StatusCallback)
	}
	if callbackParameters.StatusCallbackMethod != "" {
		formValues.Set("StatusCallbackMethod", callbackParameters.StatusCallbackMethod)
	}
	for _, event := range callbackParameters.StatusCallbackEvent {
		formValues.Add("StatusCallbackEvent", event)
	}
	if callbackParameters.SendDigits != "" {
		formValues.Set("SendDigits", callbackParameters.SendDigits)
	}
	if callbackParameters.IfMachine != "" {
		formValues.Set("IfMachine", callbackParameters.IfMachine)
	}
	if callbackParameters.Timeout != 0 {
		formValues.Set("Timeout", strconv.Itoa(callbackParameters.Timeout))
	}
	if callbackParameters.MachineDetection != "" {
		formValues.Set("MachineDetection", callbackParameters.MachineDetection)
	}
	if callbackParameters.MachineDetectionTimeout != 0 {
		formValues.Set(
			"MachineDetectionTimeout",
			strconv.Itoa(callbackParameters.MachineDetectionTimeout),
		)
	}

	if callbackParameters.Record {
		formValues.Set("Record", "true")

		if callbackParameters.RecordingChannels != "" {
			formValues.Set("RecordingChannels", callbackParameters.RecordingChannels)
		}
		if callbackParameters.RecordingStatusCallback != "" {
			formValues.Set("RecordingStatusCallback", callbackParameters.RecordingStatusCallback)
		}
		if callbackParameters.RecordingStatusCallbackMethod != "" {
			formValues.Set("RecordingStatusCallbackMethod", callbackParameters.RecordingStatusCallbackMethod)
		}
	} else {
		formValues.Set("Record", "false")
	}

	return twilio.voicePost("Calls.json", formValues)
}

// Place a voice call with an ApplicationSid specified.
func (twilio *Twilio) CallWithApplicationCallbacks(from, to, applicationSid string) (*VoiceResponse, *Exception, error) {
	formValues := url.Values{}
	formValues.Set("From", from)
	formValues.Set("To", to)
	formValues.Set("ApplicationSid", applicationSid)

	return twilio.voicePost("Calls.json", formValues)
}

// Update an existing call
func (twilio *Twilio) CallUpdate(callSid string, formValues url.Values) (*VoiceResponse, *Exception, error) {
	return twilio.voicePost("Calls/"+callSid+".json", formValues)
}

// This is a private method that has the common bits for placing or updating a voice call.
func (twilio *Twilio) voicePost(resourcePath string, formValues url.Values) (*VoiceResponse, *Exception, error) {
	var voiceResponse *VoiceResponse
	var exception *Exception

	twilioUrl := twilio.buildUrl(resourcePath)
	res, err := twilio.post(formValues, twilioUrl)
	if err != nil {
		return voiceResponse, exception, err
	}
	defer res.Body.Close()

	decoder := json.NewDecoder(res.Body)

	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted && res.StatusCode != http.StatusCreated {
		exception = new(Exception)
		err = decoder.Decode(exception)

		// We aren't checking the error because we don't actually care.
		// It's going to be passed to the client either way.
		return voiceResponse, exception, err
	}

	voiceResponse = new(VoiceResponse)
	err = decoder.Decode(voiceResponse)
	return voiceResponse, exception, err
}
<|endoftext|>


<commit_before>package stackoverflow

import (
	"encoding/xml"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// TODO: describe VoteTypeID

// Vote describes a vote
type Vote struct {
	ID           int
	PostID       int
	VoteTypeID   int
	UserID       int
	BountyAmount int
	CreationDate time.Time
}

func decodeVoteAttr(attr xml.Attr, vote *Vote) error {
	var err error
	name := strings.ToLower(attr.Name.Local)
	v := attr.Value
	switch name {
	case "id":
		vote.ID, err = strconv.Atoi(v)
	case "postid":
		vote.PostID, err = strconv.Atoi(v)
	case "userid":
		vote.UserID, err = strconv.Atoi(v)
	case "votetypeid":
		vote.VoteTypeID, err = strconv.Atoi(v)
	case "bountyamount":
		vote.BountyAmount, err = strconv.Atoi(v)
	case "creationdate":
		vote.CreationDate, err = decodeTime(v)
	default:
		err = fmt.Errorf("unknown vote field: '%s'", name)
	}
	return err
}

func decodeVoteRow(t xml.Token, vote *Vote) error {
	// have been checked before that this is "row" element
	e, _ := t.(xml.StartElement)
	for _, attr := range e.Attr {
		err := decodeVoteAttr(attr, vote)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>add constants for Vote.VoteTypeID<commit_after>package stackoverflow

import (
	"encoding/xml"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// http://blog.stackoverflow.com/2009/06/stack-overflow-creative-commons-data-dump/#comment-24147
const (
	VoteAcceptedByOriginator = 1
	VoteUpMod                = 2
	VoteDownMod              = 3
	VoteOffensive            = 4
	VoteFavorite             = 5
	VoteClose                = 6
	VoteReopen               = 7
	VoteBountyStart          = 8
	VoteBountyClose          = 9
	VoteDeletion             = 10
	VoteUndeletion           = 11
	VoteSpam                 = 12
	VoteInformModerator      = 13
)

// Vote describes a vote
type Vote struct {
	ID           int
	PostID       int
	VoteTypeID   int
	UserID       int
	BountyAmount int
	CreationDate time.Time
}

func decodeVoteAttr(attr xml.Attr, vote *Vote) error {
	var err error
	name := strings.ToLower(attr.Name.Local)
	v := attr.Value
	switch name {
	case "id":
		vote.ID, err = strconv.Atoi(v)
	case "postid":
		vote.PostID, err = strconv.Atoi(v)
	case "userid":
		vote.UserID, err = strconv.Atoi(v)
	case "votetypeid":
		vote.VoteTypeID, err = strconv.Atoi(v)
	case "bountyamount":
		vote.BountyAmount, err = strconv.Atoi(v)
	case "creationdate":
		vote.CreationDate, err = decodeTime(v)
	default:
		err = fmt.Errorf("unknown vote field: '%s'", name)
	}
	return err
}

func decodeVoteRow(t xml.Token, vote *Vote) error {
	// have been checked before that this is "row" element
	e, _ := t.(xml.StartElement)
	for _, attr := range e.Attr {
		err := decodeVoteAttr(attr, vote)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package util

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"v.io/jiri/lib/project"
	"v.io/jiri/lib/tool"
	"v.io/x/lib/envvar"
)

const (
	// TODO(nlacasse): Rename to JIRI_PROFILE.
	jiriProfileEnv = "V23_PROFILE"
	javaEnv        = "JAVA_HOME"
)

// TODO(nlacasse): Rename this.
// VanadiumEnvironment returns the environment variables setting for the
// project. The util package captures the original state of the relevant
// environment variables when the tool is initialized and every invocation of
// this function updates this original state according to the jiri tool
// configuration.
//
// By default, the Go and VDL workspaces are added to the GOPATH and VDLPATH
// environment variables respectively. In addition, the V23_PROFILE environment
// variable can be used to activate an environment variable setting for various
// development profiles of the project (e.g. arm, android, java, or nacl).
// Unlike the default setting, the setting enabled by the V23_PROFILE
// environment variable can override existing environment.
func VanadiumEnvironment(ctx *tool.Context) (*envvar.Vars, error) {
	env := envvar.VarsFromOS()
	root, err := project.V23Root()
	if err != nil {
		return nil, err
	}
	config, err := LoadConfig(ctx)
	if err != nil {
		return nil, err
	}
	env.Set("CGO_ENABLED", "1")
	if err := setGoPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if err := setVdlPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if err := setSyncbaseEnv(ctx, env, root); err != nil {
		return nil, err
	}
	if profile := os.Getenv(jiriProfileEnv); profile != "" {
		fmt.Fprintf(ctx.Stdout(), `NOTE: Enabling environment variable setting for %q.
This can override values of existing environment variables.
`, profile)
		switch profile {
		case "android":
			// Cross-compilation for android on linux.
			if err := setAndroidEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "arm":
			// Cross-compilation for arm on linux.
			if err := setArmEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "java":
			// Building of a Go shared library for Java.
			if err := setJavaEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "nacl":
			// Cross-compilation for nacl.
			if err := setNaclEnv(ctx, env, root); err != nil {
				return nil, err
			}
		default:
			fmt.Fprintf(ctx.Stderr(), "Unknown environment profile %q", profile)
		}
	}
	return env, nil
}

// setJavaEnv sets the environment variables used for building a Go
// shared library that is invoked from Java code. If Java is not
// installed on the host, this function is a no-op.
func setJavaEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	jdkHome := os.Getenv(javaEnv)
	if jdkHome == "" {
		return nil
	}

	// Compile using Java Go 1.5 (installed via 'jiri profile install java').
	javaGoDir := filepath.Join(root, "third_party", "java", "go")

	// Set Go-specific environment variables.
	cflags := env.GetTokens("CGO_CFLAGS", " ")
	cflags = append(cflags, filepath.Join("-I"+jdkHome, "include"), filepath.Join("-I"+jdkHome, "include", runtime.GOOS))
	env.SetTokens("CGO_CFLAGS", cflags, " ")
	env.Set("GOROOT", javaGoDir)

	// Update PATH.
	if _, err := ctx.Run().Stat(javaGoDir); err != nil {
		return fmt.Errorf("Couldn't find java go installation directory %s: did you run \"jiri profile install java\"?", javaGoDir)
	}
	path := env.GetTokens("PATH", ":")
	path = append([]string{filepath.Join(javaGoDir, "bin")}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setAndroidEnv sets the environment variables used for android
// cross-compilation.
func setAndroidEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	// Compile using Android Go 1.5 (installed via
	// 'jiri profile install android').
	androidGoDir := filepath.Join(root, "third_party", "android", "go")

	// Set Go-specific environment variables.
	env.Set("GOARCH", "arm")
	env.Set("GOARM", "7")
	env.Set("GOOS", "android")
	env.Set("GOROOT", androidGoDir)

	// Update PATH.
	if _, err := ctx.Run().Stat(androidGoDir); err != nil {
		return fmt.Errorf("Couldn't find android Go installation directory %s: did you run \"jiri profile install android\"?", androidGoDir)
	}
	path := env.GetTokens("PATH", ":")
	path = append([]string{filepath.Join(androidGoDir, "bin")}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setArmEnv sets the environment variables used for arm cross-compilation.
func setArmEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	// Set Go specific environment variables.
	env.Set("GOARCH", "arm")
	env.Set("GOARM", "6")
	env.Set("GOOS", "linux")

	// Add the paths to arm cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "cout", "xgcc", "cross_arm"),
		filepath.Join(root, "third_party", "repos", "go_arm", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")

	return nil
}

// setGoPath adds the paths of Go workspaces to the GOPATH variable.
func setGoPath(ctx *tool.Context, env *envvar.Vars, root string, config *Config) error {
	return setPathHelper(ctx, env, "GOPATH", root, config.GoWorkspaces(), "")
}

// setVdlPath adds the paths of VDL workspaces to the VDLPATH variable.
func setVdlPath(ctx *tool.Context, env *envvar.Vars, root string, config *Config) error {
	return setPathHelper(ctx, env, "VDLPATH", root, config.VDLWorkspaces(), "src")
}

// setPathHelper is a utility function for setting path environment
// variables for different types of workspaces.
func setPathHelper(ctx *tool.Context, env *envvar.Vars, name, root string, workspaces []string, suffix string) error {
	path := env.GetTokens(name, ":")
	projects, _, err := project.ReadManifest(ctx)
	if err != nil {
		return err
	}
	for _, workspace := range workspaces {
		absWorkspace := filepath.Join(root, workspace, suffix)
		// Only append an entry to the path if the workspace is rooted
		// under a jiri project that exists locally or vice versa.
		for _, project := range projects {
			// We check if <project.Path> is a prefix of <absWorkspace> to
			// account for Go workspaces nested under a single jiri project,
			// such as: $V23_ROOT/release/projects/chat/go.
			//
			// We check if <absWorkspace> is a prefix of <project.Path> to
			// account for Go workspaces that span multiple jiri projects,
			// such as: $V23_ROOT/release/go.
			if strings.HasPrefix(absWorkspace, project.Path) || strings.HasPrefix(project.Path, absWorkspace) {
				if _, err := ctx.Run().Stat(filepath.Join(absWorkspace)); err == nil {
					path = append(path, absWorkspace)
					break
				}
			}
		}
	}
	env.SetTokens(name, path, ":")
	return nil
}

// TODO(nlacasse): Move setNaclEnv and setSyncbaseEnv out of jiri since they
// are not general-purpose.

// setNaclEnv sets the environment variables used for nacl
// cross-compilation.
func setNaclEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	env.Set("GOARCH", "amd64p32")
	env.Set("GOOS", "nacl")

	// Add the path to nacl cross-compiler to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "repos", "go_ppapi", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")

	return nil
}

// setSyncbaseEnv adds the LevelDB third-party C++ libraries Vanadium
// Go code depends on to the CGO_CFLAGS and CGO_LDFLAGS variables.
func setSyncbaseEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	libs := []string{
		"leveldb",
		"snappy",
	}
	// TODO(rogulenko): get these vars from a config file.
	goos, goarch := os.Getenv("GOOS"), os.Getenv("GOARCH")
	if goos == "" {
		goos = runtime.GOOS
	}
	if goarch == "" {
		goarch = runtime.GOARCH
	}
	for _, lib := range libs {
		cflags := env.GetTokens("CGO_CFLAGS", " ")
		cxxflags := env.GetTokens("CGO_CXXFLAGS", " ")
		ldflags := env.GetTokens("CGO_LDFLAGS", " ")
		dir, err := ThirdPartyCCodePath(goos, goarch)
		if err != nil {
			return err
		}
		dir = filepath.Join(dir, lib)
		if _, err := ctx.Run().Stat(dir); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			continue
		}
		cflags = append(cflags, filepath.Join("-I"+dir, "include"))
		cxxflags = append(cxxflags, filepath.Join("-I"+dir, "include"))
		ldflags = append(ldflags, filepath.Join("-L"+dir, "lib"))
		if runtime.GOARCH == "linux" {
			ldflags = append(ldflags, "-Wl,-rpath", filepath.Join(dir, "lib"))
		}
		env.SetTokens("CGO_CFLAGS", cflags, " ")
		env.SetTokens("CGO_CXXFLAGS", cxxflags, " ")
		env.SetTokens("CGO_LDFLAGS", ldflags, " ")
	}
	return nil
}
<commit_msg>use env.Get rather than os.Getenv to determine GOOS and GOARCH for syncbase env<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package util

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"v.io/jiri/lib/project"
	"v.io/jiri/lib/tool"
	"v.io/x/lib/envvar"
)

const (
	// TODO(nlacasse): Rename to JIRI_PROFILE.
	jiriProfileEnv = "V23_PROFILE"
	javaEnv        = "JAVA_HOME"
)

// TODO(nlacasse): Rename this.
// VanadiumEnvironment returns the environment variables setting for the
// project. The util package captures the original state of the relevant
// environment variables when the tool is initialized and every invocation of
// this function updates this original state according to the jiri tool
// configuration.
//
// By default, the Go and VDL workspaces are added to the GOPATH and VDLPATH
// environment variables respectively. In addition, the V23_PROFILE environment
// variable can be used to activate an environment variable setting for various
// development profiles of the project (e.g. arm, android, java, or nacl).
// Unlike the default setting, the setting enabled by the V23_PROFILE
// environment variable can override existing environment.
func VanadiumEnvironment(ctx *tool.Context) (*envvar.Vars, error) {
	env := envvar.VarsFromOS()
	root, err := project.V23Root()
	if err != nil {
		return nil, err
	}
	config, err := LoadConfig(ctx)
	if err != nil {
		return nil, err
	}
	env.Set("CGO_ENABLED", "1")
	if err := setGoPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if err := setVdlPath(ctx, env, root, config); err != nil {
		return nil, err
	}
	if profile := os.Getenv(jiriProfileEnv); profile != "" {
		fmt.Fprintf(ctx.Stdout(), `NOTE: Enabling environment variable setting for %q.
This can override values of existing environment variables.
`, profile)
		switch profile {
		case "android":
			// Cross-compilation for android on linux.
			if err := setAndroidEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "arm":
			// Cross-compilation for arm on linux.
			if err := setArmEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "java":
			// Building of a Go shared library for Java.
			if err := setJavaEnv(ctx, env, root); err != nil {
				return nil, err
			}
		case "nacl":
			// Cross-compilation for nacl.
			if err := setNaclEnv(ctx, env, root); err != nil {
				return nil, err
			}
		default:
			fmt.Fprintf(ctx.Stderr(), "Unknown environment profile %q", profile)
		}
	}
	if err := setSyncbaseEnv(ctx, env, root); err != nil {
		return nil, err
	}
	return env, nil
}

// setJavaEnv sets the environment variables used for building a Go
// shared library that is invoked from Java code. If Java is not
// installed on the host, this function is a no-op.
func setJavaEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	jdkHome := os.Getenv(javaEnv)
	if jdkHome == "" {
		return nil
	}

	// Compile using Java Go 1.5 (installed via 'jiri profile install java').
	javaGoDir := filepath.Join(root, "third_party", "java", "go")

	// Set Go-specific environment variables.
	cflags := env.GetTokens("CGO_CFLAGS", " ")
	cflags = append(cflags, filepath.Join("-I"+jdkHome, "include"), filepath.Join("-I"+jdkHome, "include", runtime.GOOS))
	env.SetTokens("CGO_CFLAGS", cflags, " ")
	env.Set("GOROOT", javaGoDir)

	// Update PATH.
	if _, err := ctx.Run().Stat(javaGoDir); err != nil {
		return fmt.Errorf("Couldn't find java go installation directory %s: did you run \"jiri profile install java\"?", javaGoDir)
	}
	path := env.GetTokens("PATH", ":")
	path = append([]string{filepath.Join(javaGoDir, "bin")}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setAndroidEnv sets the environment variables used for android
// cross-compilation.
func setAndroidEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	// Compile using Android Go 1.5 (installed via
	// 'jiri profile install android').
	androidGoDir := filepath.Join(root, "third_party", "android", "go")

	// Set Go-specific environment variables.
	env.Set("GOARCH", "arm")
	env.Set("GOARM", "7")
	env.Set("GOOS", "android")
	env.Set("GOROOT", androidGoDir)

	// Update PATH.
	if _, err := ctx.Run().Stat(androidGoDir); err != nil {
		return fmt.Errorf("Couldn't find android Go installation directory %s: did you run \"jiri profile install android\"?", androidGoDir)
	}
	path := env.GetTokens("PATH", ":")
	path = append([]string{filepath.Join(androidGoDir, "bin")}, path...)
	env.SetTokens("PATH", path, ":")
	return nil
}

// setArmEnv sets the environment variables used for arm cross-compilation.
func setArmEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	// Set Go specific environment variables.
	env.Set("GOARCH", "arm")
	env.Set("GOARM", "6")
	env.Set("GOOS", "linux")

	// Add the paths to arm cross-compilation tools to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "cout", "xgcc", "cross_arm"),
		filepath.Join(root, "third_party", "repos", "go_arm", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")

	return nil
}

// setGoPath adds the paths of Go workspaces to the GOPATH variable.
func setGoPath(ctx *tool.Context, env *envvar.Vars, root string, config *Config) error {
	return setPathHelper(ctx, env, "GOPATH", root, config.GoWorkspaces(), "")
}

// setVdlPath adds the paths of VDL workspaces to the VDLPATH variable.
func setVdlPath(ctx *tool.Context, env *envvar.Vars, root string, config *Config) error {
	return setPathHelper(ctx, env, "VDLPATH", root, config.VDLWorkspaces(), "src")
}

// setPathHelper is a utility function for setting path environment
// variables for different types of workspaces.
func setPathHelper(ctx *tool.Context, env *envvar.Vars, name, root string, workspaces []string, suffix string) error {
	path := env.GetTokens(name, ":")
	projects, _, err := project.ReadManifest(ctx)
	if err != nil {
		return err
	}
	for _, workspace := range workspaces {
		absWorkspace := filepath.Join(root, workspace, suffix)
		// Only append an entry to the path if the workspace is rooted
		// under a jiri project that exists locally or vice versa.
		for _, project := range projects {
			// We check if <project.Path> is a prefix of <absWorkspace> to
			// account for Go workspaces nested under a single jiri project,
			// such as: $V23_ROOT/release/projects/chat/go.
			//
			// We check if <absWorkspace> is a prefix of <project.Path> to
			// account for Go workspaces that span multiple jiri projects,
			// such as: $V23_ROOT/release/go.
			if strings.HasPrefix(absWorkspace, project.Path) || strings.HasPrefix(project.Path, absWorkspace) {
				if _, err := ctx.Run().Stat(filepath.Join(absWorkspace)); err == nil {
					path = append(path, absWorkspace)
					break
				}
			}
		}
	}
	env.SetTokens(name, path, ":")
	return nil
}

// TODO(nlacasse): Move setNaclEnv and setSyncbaseEnv out of jiri since they
// are not general-purpose.

// setNaclEnv sets the environment variables used for nacl
// cross-compilation.
func setNaclEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	env.Set("GOARCH", "amd64p32")
	env.Set("GOOS", "nacl")

	// Add the path to nacl cross-compiler to the PATH.
	path := env.GetTokens("PATH", ":")
	path = append([]string{
		filepath.Join(root, "third_party", "repos", "go_ppapi", "bin"),
	}, path...)
	env.SetTokens("PATH", path, ":")

	return nil
}

// setSyncbaseEnv adds the LevelDB third-party C++ libraries Vanadium
// Go code depends on to the CGO_CFLAGS and CGO_LDFLAGS variables.
func setSyncbaseEnv(ctx *tool.Context, env *envvar.Vars, root string) error {
	libs := []string{
		"leveldb",
		"snappy",
	}
	// TODO(rogulenko): get these vars from a config file.
	goos, goarch := env.Get("GOOS"), env.Get("GOARCH")
	if goos == "" {
		goos = runtime.GOOS
	}
	if goarch == "" {
		goarch = runtime.GOARCH
	}
	for _, lib := range libs {
		cflags := env.GetTokens("CGO_CFLAGS", " ")
		cxxflags := env.GetTokens("CGO_CXXFLAGS", " ")
		ldflags := env.GetTokens("CGO_LDFLAGS", " ")
		dir, err := ThirdPartyCCodePath(goos, goarch)
		if err != nil {
			return err
		}
		dir = filepath.Join(dir, lib)
		if _, err := ctx.Run().Stat(dir); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			continue
		}
		cflags = append(cflags, filepath.Join("-I"+dir, "include"))
		cxxflags = append(cxxflags, filepath.Join("-I"+dir, "include"))
		ldflags = append(ldflags, filepath.Join("-L"+dir, "lib"))
		if runtime.GOARCH == "linux" {
			ldflags = append(ldflags, "-Wl,-rpath", filepath.Join(dir, "lib"))
		}
		env.SetTokens("CGO_CFLAGS", cflags, " ")
		env.SetTokens("CGO_CXXFLAGS", cxxflags, " ")
		env.SetTokens("CGO_LDFLAGS", ldflags, " ")
	}
	return nil
}
<|endoftext|>