<commit_before>package atlas

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"runtime"
	"strings"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-rootcerts"
)

const (
	// atlasDefaultEndpoint is the default base URL for connecting to Atlas.
	atlasDefaultEndpoint = "https://atlas.hashicorp.com"

	// atlasEndpointEnvVar is the environment variable that overrrides the
	// default Atlas address.
	atlasEndpointEnvVar = "ATLAS_ADDRESS"

	// atlasCAFileEnvVar is the environment variable that causes the client to
	// load trusted certs from a file
	atlasCAFileEnvVar = "ATLAS_CAFILE"

	// atlasCAPathEnvVar is the environment variable that causes the client to
	// load trusted certs from a directory
	atlasCAPathEnvVar = "ATLAS_CAPATH"

	// atlasTokenHeader is the header key used for authenticating with Atlas
	atlasTokenHeader = "X-Atlas-Token"
)

var projectURL = "https://github.com/hashicorp/atlas-go"
var userAgent = fmt.Sprintf("AtlasGo/1.0 (+%s; %s)",
	projectURL, runtime.Version())

// ErrAuth is the error returned if a 401 is returned by an API request.
var ErrAuth = fmt.Errorf("authentication failed")

// ErrNotFound is the error returned if a 404 is returned by an API request.
var ErrNotFound = fmt.Errorf("resource not found")

// RailsError represents an error that was returned from the Rails server.
type RailsError struct {
	Errors []string `json:"errors"`
}

// Error collects all of the errors in the RailsError and returns a comma-
// separated list of the errors that were returned from the server.
func (re *RailsError) Error() string {
	return strings.Join(re.Errors, ", ")
}

// Client represents a single connection to a Atlas API endpoint.
type Client struct {
	// URL is the full endpoint address to the Atlas server including the
	// protocol, port, and path.
	URL *url.URL

	// Token is the Atlas authentication token
	Token string

	// HTTPClient is the underlying http client with which to make requests.
	HTTPClient *http.Client

	// DefaultHeaders is a set of headers that will be added to every request.
	// This minimally includes the atlas user-agent string.
	DefaultHeader http.Header
}

// DefaultClient returns a client that connects to the Atlas API.
func DefaultClient() *Client {
	atlasEndpoint := os.Getenv(atlasEndpointEnvVar)
	if atlasEndpoint == "" {
		atlasEndpoint = atlasDefaultEndpoint
	}

	client, err := NewClient(atlasEndpoint)
	if err != nil {
		panic(err)
	}

	return client
}

// NewClient creates a new Atlas Client from the given URL (as a string). If
// the URL cannot be parsed, an error is returned. The HTTPClient is set to
// an empty http.Client, but this can be changed programmatically by setting
// client.HTTPClient. The user can also programmatically set the URL as a
// *url.URL.
func NewClient(urlString string) (*Client, error) {
	if len(urlString) == 0 {
		return nil, fmt.Errorf("client: missing url")
	}

	parsedURL, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	token := os.Getenv("ATLAS_TOKEN")
	if token != "" {
		log.Printf("[DEBUG] using ATLAS_TOKEN (%s)", maskString(token))
	}

	client := &Client{
		URL:           parsedURL,
		Token:         token,
		DefaultHeader: make(http.Header),
	}

	client.DefaultHeader.Set("User-Agent", userAgent)

	if err := client.init(); err != nil {
		return nil, err
	}

	return client, nil
}

// init() sets defaults on the client.
func (c *Client) init() error {
	c.HTTPClient = cleanhttp.DefaultClient()
	tlsConfig := &tls.Config{}
	err := rootcerts.ConfigureTLS(tlsConfig, &rootcerts.Config{
		CAFile: os.Getenv(atlasCAFileEnvVar),
		CAPath: os.Getenv(atlasCAPathEnvVar),
	})
	if err != nil {
		return err
	}
	t := cleanhttp.DefaultTransport()
	t.TLSClientConfig = tlsConfig
	c.HTTPClient.Transport = t
	return nil
}

// RequestOptions is the list of options to pass to the request.
type RequestOptions struct {
	// Params is a map of key-value pairs that will be added to the Request.
	Params map[string]string

	// Headers is a map of key-value pairs that will be added to the Request.
	Headers map[string]string

	// Body is an io.Reader object that will be streamed or uploaded with the
	// Request. BodyLength is the final size of the Body.
	Body       io.Reader
	BodyLength int64
}

// Request creates a new HTTP request using the given verb and sub path.
func (c *Client) Request(verb, spath string, ro *RequestOptions) (*http.Request, error) {
	log.Printf("[INFO] request: %s %s", verb, spath)

	// Ensure we have a RequestOptions struct (passing nil is an acceptable)
	if ro == nil {
		ro = new(RequestOptions)
	}

	// Create a new URL with the appended path
	u := *c.URL
	u.Path = path.Join(c.URL.Path, spath)

	// Add the token and other params
	if c.Token != "" {
		log.Printf("[DEBUG] request: appending token (%s)", maskString(c.Token))
		if ro.Headers == nil {
			ro.Headers = make(map[string]string)
		}

		ro.Headers[atlasTokenHeader] = c.Token
	}

	return c.rawRequest(verb, &u, ro)
}

func (c *Client) putFile(rawURL string, r io.Reader, size int64) error {
	log.Printf("[INFO] putting file: %s", rawURL)

	url, err := url.Parse(rawURL)
	if err != nil {
		return err
	}

	request, err := c.rawRequest("PUT", url, &RequestOptions{
		Body:       r,
		BodyLength: size,
	})
	if err != nil {
		return err
	}

	if _, err := checkResp(c.HTTPClient.Do(request)); err != nil {
		return err
	}

	return nil
}

// rawRequest accepts a verb, URL, and RequestOptions struct and returns the
// constructed http.Request and any errors that occurred
func (c *Client) rawRequest(verb string, u *url.URL, ro *RequestOptions) (*http.Request, error) {
	if verb == "" {
		return nil, fmt.Errorf("client: missing verb")
	}

	if u == nil {
		return nil, fmt.Errorf("client: missing URL.url")
	}

	if ro == nil {
		return nil, fmt.Errorf("client: missing RequestOptions")
	}

	// Add the token and other params
	var params = make(url.Values)
	for k, v := range ro.Params {
		params.Add(k, v)
	}
	u.RawQuery = params.Encode()

	// Create the request object
	request, err := http.NewRequest(verb, u.String(), ro.Body)
	if err != nil {
		return nil, err
	}

	// set our default headers first
	for k, v := range c.DefaultHeader {
		request.Header[k] = v
	}

	// Add any request headers (auth will be here if set)
	for k, v := range ro.Headers {
		request.Header.Add(k, v)
	}

	// Add content-length if we have it
	if ro.BodyLength > 0 {
		request.ContentLength = ro.BodyLength
	}

	log.Printf("[DEBUG] raw request: %#v", request)

	return request, nil
}

// checkResp wraps http.Client.Do() and verifies that the request was
// successful. A non-200 request returns an error formatted to included any
// validation problems or otherwise.
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher up the chain, so
	// just return that
	if err != nil {
		return resp, err
	}

	log.Printf("[INFO] response: %d (%s)", resp.StatusCode, resp.Status)
	var buf bytes.Buffer
	if _, err := io.Copy(&buf, resp.Body); err != nil {
		log.Printf("[ERR] response: error copying response body")
	} else {
		log.Printf("[DEBUG] response: %s", buf.String())

		// We are going to reset the response body, so we need to close the old
		// one or else it will leak.
		resp.Body.Close()
		resp.Body = &bytesReadCloser{&buf}
	}

	switch resp.StatusCode {
	case 200:
		return resp, nil
	case 201:
		return resp, nil
	case 202:
		return resp, nil
	case 204:
		return resp, nil
	case 400:
		return nil, parseErr(resp)
	case 401:
		return nil, ErrAuth
	case 404:
		return nil, ErrNotFound
	case 422:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("client: %s", resp.Status)
	}
}

// parseErr is used to take an error JSON response and return a single string
// for use in error messages.
func parseErr(r *http.Response) error {
	re := &RailsError{}

	if err := decodeJSON(r, &re); err != nil {
		return fmt.Errorf("error decoding JSON body: %s", err)
	}

	return re
}

// decodeJSON is used to JSON decode a body into an interface.
func decodeJSON(resp *http.Response, out interface{}) error {
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	return dec.Decode(out)
}

// bytesReadCloser is a simple wrapper around a bytes buffer that implements
// Close as a noop.
type bytesReadCloser struct {
	*bytes.Buffer
}

func (nrc *bytesReadCloser) Close() error {
	// we don't actually have to do anything here, since the buffer is just some
	// data in memory  and the error is initialized to no-error
	return nil
}
<commit_msg>Added ATLAS_TLS_NOVERIFY env option<commit_after>package atlas

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"runtime"
	"strings"

	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-rootcerts"
)

const (
	// atlasDefaultEndpoint is the default base URL for connecting to Atlas.
	atlasDefaultEndpoint = "https://atlas.hashicorp.com"

	// atlasEndpointEnvVar is the environment variable that overrrides the
	// default Atlas address.
	atlasEndpointEnvVar = "ATLAS_ADDRESS"

	// atlasCAFileEnvVar is the environment variable that causes the client to
	// load trusted certs from a file
	atlasCAFileEnvVar = "ATLAS_CAFILE"

	// atlasCAPathEnvVar is the environment variable that causes the client to
	// load trusted certs from a directory
	atlasCAPathEnvVar = "ATLAS_CAPATH"

	// atlasTLSNoVerifyEnvVar disables TLS verification, similar to curl -k
	// This defaults to false (verify) and will change to true (skip
	// verification) with any non-empty value
	atlasTLSNoVerifyEnvVar = "ATLAS_TLS_NOVERIFY"

	// atlasTokenHeader is the header key used for authenticating with Atlas
	atlasTokenHeader = "X-Atlas-Token"
)

var projectURL = "https://github.com/hashicorp/atlas-go"
var userAgent = fmt.Sprintf("AtlasGo/1.0 (+%s; %s)",
	projectURL, runtime.Version())

// ErrAuth is the error returned if a 401 is returned by an API request.
var ErrAuth = fmt.Errorf("authentication failed")

// ErrNotFound is the error returned if a 404 is returned by an API request.
var ErrNotFound = fmt.Errorf("resource not found")

// RailsError represents an error that was returned from the Rails server.
type RailsError struct {
	Errors []string `json:"errors"`
}

// Error collects all of the errors in the RailsError and returns a comma-
// separated list of the errors that were returned from the server.
func (re *RailsError) Error() string {
	return strings.Join(re.Errors, ", ")
}

// Client represents a single connection to a Atlas API endpoint.
type Client struct {
	// URL is the full endpoint address to the Atlas server including the
	// protocol, port, and path.
	URL *url.URL

	// Token is the Atlas authentication token
	Token string

	// HTTPClient is the underlying http client with which to make requests.
	HTTPClient *http.Client

	// DefaultHeaders is a set of headers that will be added to every request.
	// This minimally includes the atlas user-agent string.
	DefaultHeader http.Header
}

// DefaultClient returns a client that connects to the Atlas API.
func DefaultClient() *Client {
	atlasEndpoint := os.Getenv(atlasEndpointEnvVar)
	if atlasEndpoint == "" {
		atlasEndpoint = atlasDefaultEndpoint
	}

	client, err := NewClient(atlasEndpoint)
	if err != nil {
		panic(err)
	}

	return client
}

// NewClient creates a new Atlas Client from the given URL (as a string). If
// the URL cannot be parsed, an error is returned. The HTTPClient is set to
// an empty http.Client, but this can be changed programmatically by setting
// client.HTTPClient. The user can also programmatically set the URL as a
// *url.URL.
func NewClient(urlString string) (*Client, error) {
	if len(urlString) == 0 {
		return nil, fmt.Errorf("client: missing url")
	}

	parsedURL, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	token := os.Getenv("ATLAS_TOKEN")
	if token != "" {
		log.Printf("[DEBUG] using ATLAS_TOKEN (%s)", maskString(token))
	}

	client := &Client{
		URL:           parsedURL,
		Token:         token,
		DefaultHeader: make(http.Header),
	}

	client.DefaultHeader.Set("User-Agent", userAgent)

	if err := client.init(); err != nil {
		return nil, err
	}

	return client, nil
}

// init() sets defaults on the client.
func (c *Client) init() error {
	c.HTTPClient = cleanhttp.DefaultClient()
	tlsConfig := &tls.Config{}
	if os.Getenv(atlasTLSNoVerifyEnvVar) != "" {
		tlsConfig.InsecureSkipVerify = true
	}
	err := rootcerts.ConfigureTLS(tlsConfig, &rootcerts.Config{
		CAFile: os.Getenv(atlasCAFileEnvVar),
		CAPath: os.Getenv(atlasCAPathEnvVar),
	})
	if err != nil {
		return err
	}
	t := cleanhttp.DefaultTransport()
	t.TLSClientConfig = tlsConfig
	c.HTTPClient.Transport = t
	return nil
}

// RequestOptions is the list of options to pass to the request.
type RequestOptions struct {
	// Params is a map of key-value pairs that will be added to the Request.
	Params map[string]string

	// Headers is a map of key-value pairs that will be added to the Request.
	Headers map[string]string

	// Body is an io.Reader object that will be streamed or uploaded with the
	// Request. BodyLength is the final size of the Body.
	Body       io.Reader
	BodyLength int64
}

// Request creates a new HTTP request using the given verb and sub path.
func (c *Client) Request(verb, spath string, ro *RequestOptions) (*http.Request, error) {
	log.Printf("[INFO] request: %s %s", verb, spath)

	// Ensure we have a RequestOptions struct (passing nil is an acceptable)
	if ro == nil {
		ro = new(RequestOptions)
	}

	// Create a new URL with the appended path
	u := *c.URL
	u.Path = path.Join(c.URL.Path, spath)

	// Add the token and other params
	if c.Token != "" {
		log.Printf("[DEBUG] request: appending token (%s)", maskString(c.Token))
		if ro.Headers == nil {
			ro.Headers = make(map[string]string)
		}

		ro.Headers[atlasTokenHeader] = c.Token
	}

	return c.rawRequest(verb, &u, ro)
}

func (c *Client) putFile(rawURL string, r io.Reader, size int64) error {
	log.Printf("[INFO] putting file: %s", rawURL)

	url, err := url.Parse(rawURL)
	if err != nil {
		return err
	}

	request, err := c.rawRequest("PUT", url, &RequestOptions{
		Body:       r,
		BodyLength: size,
	})
	if err != nil {
		return err
	}

	if _, err := checkResp(c.HTTPClient.Do(request)); err != nil {
		return err
	}

	return nil
}

// rawRequest accepts a verb, URL, and RequestOptions struct and returns the
// constructed http.Request and any errors that occurred
func (c *Client) rawRequest(verb string, u *url.URL, ro *RequestOptions) (*http.Request, error) {
	if verb == "" {
		return nil, fmt.Errorf("client: missing verb")
	}

	if u == nil {
		return nil, fmt.Errorf("client: missing URL.url")
	}

	if ro == nil {
		return nil, fmt.Errorf("client: missing RequestOptions")
	}

	// Add the token and other params
	var params = make(url.Values)
	for k, v := range ro.Params {
		params.Add(k, v)
	}
	u.RawQuery = params.Encode()

	// Create the request object
	request, err := http.NewRequest(verb, u.String(), ro.Body)
	if err != nil {
		return nil, err
	}

	// set our default headers first
	for k, v := range c.DefaultHeader {
		request.Header[k] = v
	}

	// Add any request headers (auth will be here if set)
	for k, v := range ro.Headers {
		request.Header.Add(k, v)
	}

	// Add content-length if we have it
	if ro.BodyLength > 0 {
		request.ContentLength = ro.BodyLength
	}

	log.Printf("[DEBUG] raw request: %#v", request)

	return request, nil
}

// checkResp wraps http.Client.Do() and verifies that the request was
// successful. A non-200 request returns an error formatted to included any
// validation problems or otherwise.
func checkResp(resp *http.Response, err error) (*http.Response, error) {
	// If the err is already there, there was an error higher up the chain, so
	// just return that
	if err != nil {
		return resp, err
	}

	log.Printf("[INFO] response: %d (%s)", resp.StatusCode, resp.Status)
	var buf bytes.Buffer
	if _, err := io.Copy(&buf, resp.Body); err != nil {
		log.Printf("[ERR] response: error copying response body")
	} else {
		log.Printf("[DEBUG] response: %s", buf.String())

		// We are going to reset the response body, so we need to close the old
		// one or else it will leak.
		resp.Body.Close()
		resp.Body = &bytesReadCloser{&buf}
	}

	switch resp.StatusCode {
	case 200:
		return resp, nil
	case 201:
		return resp, nil
	case 202:
		return resp, nil
	case 204:
		return resp, nil
	case 400:
		return nil, parseErr(resp)
	case 401:
		return nil, ErrAuth
	case 404:
		return nil, ErrNotFound
	case 422:
		return nil, parseErr(resp)
	default:
		return nil, fmt.Errorf("client: %s", resp.Status)
	}
}

// parseErr is used to take an error JSON response and return a single string
// for use in error messages.
func parseErr(r *http.Response) error {
	re := &RailsError{}

	if err := decodeJSON(r, &re); err != nil {
		return fmt.Errorf("error decoding JSON body: %s", err)
	}

	return re
}

// decodeJSON is used to JSON decode a body into an interface.
func decodeJSON(resp *http.Response, out interface{}) error {
	defer resp.Body.Close()
	dec := json.NewDecoder(resp.Body)
	return dec.Decode(out)
}

// bytesReadCloser is a simple wrapper around a bytes buffer that implements
// Close as a noop.
type bytesReadCloser struct {
	*bytes.Buffer
}

func (nrc *bytesReadCloser) Close() error {
	// we don't actually have to do anything here, since the buffer is just some
	// data in memory  and the error is initialized to no-error
	return nil
}
<|endoftext|>


<commit_before>package interact

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

var (
	// ErrNoOptionSelected is returned when the user has not selected either yes or no
	ErrNoOptionSelected = errors.New("Please select y/n!")
)

// ConfirmDefault specifies what an empty user input defaults to
type ConfirmDefault int

// Possibel options for what an empty user input defaults to
const (
	ConfirmDefaultToYes ConfirmDefault = iota
	ConfirmDefaultToNo
	ConfirmNoDefault
)

// Confirm provides the message to the user and asks yes or no. If the user
// doesn't select either of the possible answers ErrNoOptionSelected will be
// returned
func Confirm(message string, def ConfirmDefault) (bool, error) {
	var options string
	switch def {
	case ConfirmDefaultToYes:
		options = "[Y/n]"
	case ConfirmDefaultToNo:
		options = "[y/N]"
	case ConfirmNoDefault:
		options = "[y/n]"
	}
	fmt.Printf("%s %s: ", message, options)

	rd := bufio.NewReader(os.Stdin)
	line, err := rd.ReadString('\n')
	input := strings.TrimSpace(line)
	if err != nil {
		return false, err
	} else if input == "" {
		switch def {
		case ConfirmDefaultToYes:
			return true, nil
		case ConfirmDefaultToNo:
			return false, nil
		case ConfirmNoDefault:
			return false, ErrNoOptionSelected
		}
	}
	switch input {
	case "y":
		return true, nil
	case "n":
		return false, nil
	}
	return false, ErrNoOptionSelected
}
<commit_msg>fix typo in docs<commit_after>package interact

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

var (
	// ErrNoOptionSelected is returned when the user has not selected either yes or no
	ErrNoOptionSelected = errors.New("Please select y/n!")
)

// ConfirmDefault specifies what an empty user input defaults to
type ConfirmDefault int

// Possible options for what an empty user input defaults to
const (
	ConfirmDefaultToYes ConfirmDefault = iota
	ConfirmDefaultToNo
	ConfirmNoDefault
)

// Confirm provides the message to the user and asks yes or no. If the user
// doesn't select either of the possible answers ErrNoOptionSelected will be
// returned
func Confirm(message string, def ConfirmDefault) (bool, error) {
	var options string
	switch def {
	case ConfirmDefaultToYes:
		options = "[Y/n]"
	case ConfirmDefaultToNo:
		options = "[y/N]"
	case ConfirmNoDefault:
		options = "[y/n]"
	}
	fmt.Printf("%s %s: ", message, options)

	rd := bufio.NewReader(os.Stdin)
	line, err := rd.ReadString('\n')
	input := strings.TrimSpace(line)
	if err != nil {
		return false, err
	} else if input == "" {
		switch def {
		case ConfirmDefaultToYes:
			return true, nil
		case ConfirmDefaultToNo:
			return false, nil
		case ConfirmNoDefault:
			return false, ErrNoOptionSelected
		}
	}
	switch input {
	case "y":
		return true, nil
	case "n":
		return false, nil
	}
	return false, ErrNoOptionSelected
}
<|endoftext|>


<commit_before>package siacore

import (
	"errors"
	"net"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/network"
)

const (
	MaxCatchUpBlocks = 100
)

var (
	GenesisAddress   = CoinAddress{}         // TODO: NEED TO CREATE A HARDCODED ADDRESS.
	GenesisTimestamp = Timestamp(1415904418) // Approx. 1:47pm EST Nov. 13th, 2014
)

// CreateGenesisState will create the state that contains the genesis block and
// nothing else.
func CreateGenesisState() (s *State) {
	// Create a new state and initialize the maps.
	s = new(State)
	s.BlockRoot = new(BlockNode)
	s.BadBlocks = make(map[BlockID]struct{})
	s.BlockMap = make(map[BlockID]*BlockNode)
	s.OrphanMap = make(map[BlockID]map[BlockID]*Block)
	s.CurrentPath = make(map[BlockHeight]BlockID)
	s.OpenContracts = make(map[ContractID]*OpenContract)
	s.UnspentOutputs = make(map[OutputID]Output)
	s.SpentOutputs = make(map[OutputID]Output)
	s.TransactionPool = make(map[OutputID]*Transaction)
	s.TransactionList = make(map[OutputID]*Transaction)

	// Create the genesis block and add it as the BlockRoot.
	genesisBlock := &Block{
		Timestamp:    GenesisTimestamp,
		MinerAddress: GenesisAddress,
	}
	s.BlockRoot.Block = genesisBlock
	s.BlockRoot.Height = 0
	for i := range s.BlockRoot.RecentTimestamps {
		s.BlockRoot.RecentTimestamps[i] = GenesisTimestamp
	}
	s.BlockRoot.Target[1] = 16 // Easy enough for a home computer to be able to mine on.
	s.BlockRoot.Depth[0] = 255 // depth of genesis block is set to 111111110000000000000000...
	s.BlockMap[genesisBlock.ID()] = s.BlockRoot

	// Fill out the consensus informaiton for the genesis block.
	s.CurrentBlockID = genesisBlock.ID()
	s.CurrentPath[BlockHeight(0)] = genesisBlock.ID()

	// Create the genesis subsidy output.
	genesisSubsidyOutput := Output{
		Value:     CalculateCoinbase(0),
		SpendHash: GenesisAddress,
	}
	s.UnspentOutputs[genesisBlock.SubsidyID()] = genesisSubsidyOutput

	return
}

// SendBlocks sends all known blocks from the given height forward from the
// longest known fork.
func (s *State) SendBlocks(conn net.Conn, data []byte) (err error) {
	// Get the starting point.
	start := BlockHeight(encoding.DecUint64(data))
	if start > s.Height() {
		err = errors.New("start is greater than the height of the longest known fork.")
		return
	}

	// Build an array of blocks.
	var blocks []Block
	for i := start; i < start+MaxCatchUpBlocks; i++ {
		b := s.BlockAtHeight(i)
		if b == nil {
			break
		}
		blocks = append(blocks, *b)
	}

	// Encode and send the blocks.
	encBlocks := encoding.Marshal(blocks)
	encLen := encoding.EncUint64(uint64(len(encBlocks)))
	_, err = conn.Write(append(encLen[:4], encBlocks...))
	if err != nil {
		return
	}

	return
}

// CatchUp requests a maximum of 100 blocks from a peer, starting from the
// current height. It can be called repeatedly to download the full chain.
func (s *State) CatchUp(start BlockHeight) func(net.Conn) error {
	encbh := encoding.EncUint64(uint64(start))

	// TODO: WHY ARE WE RETURNING A FUNCTION? THIS CODE IS REALLY SLOPPY ALSO.
	return func(conn net.Conn) error {
		conn.Write(append([]byte{'R', 4, 0, 0, 0}, encbh[:4]...))
		var blocks []Block
		encBlocks, err := network.ReadPrefix(conn)
		if err != nil {
			return err
		}
		if err = encoding.Unmarshal(encBlocks, &blocks); err != nil {
			return err
		}
		for i := range blocks {
			if err = s.AcceptBlock(blocks[i]); err != nil {
				if err == UnknownOrphanErr && i == 0 {
					if start > 6 && start > s.Height()-20 {
						// TODO: HAVE LUKE FIX THIS LINE OF CODE. IDEALLY, THIS
						// CODE ALSO ONLY ASKS FOR BLOCKS FROM START-6 TO
						// START, BECAUSE WE'RE ALREADY CAUGHT UP FROM START
						// FORWARD.
						s.CatchUp(start - 6)
					}
				}
			} else if err != BlockKnownErr && err != FutureBlockErr {
				// Return if there's an error, but don't return for benign
				// errors: BlockKnownErr and FutureBlockErr are both benign.
				return
			}
		}
		if len(blocks) < MaxCatchUpBlocks {
			return errors.New("finished catching up")
		}
		return nil
	}
}
<commit_msg>fix CatchUp logic to be byzantine-safe<commit_after>package siacore

import (
	"errors"
	"net"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/network"
)

const (
	MaxCatchUpBlocks = 100
)

var (
	GenesisAddress   = CoinAddress{}         // TODO: NEED TO CREATE A HARDCODED ADDRESS.
	GenesisTimestamp = Timestamp(1415904418) // Approx. 1:47pm EST Nov. 13th, 2014
)

// CreateGenesisState will create the state that contains the genesis block and
// nothing else.
func CreateGenesisState() (s *State) {
	// Create a new state and initialize the maps.
	s = new(State)
	s.BlockRoot = new(BlockNode)
	s.BadBlocks = make(map[BlockID]struct{})
	s.BlockMap = make(map[BlockID]*BlockNode)
	s.OrphanMap = make(map[BlockID]map[BlockID]*Block)
	s.CurrentPath = make(map[BlockHeight]BlockID)
	s.OpenContracts = make(map[ContractID]*OpenContract)
	s.UnspentOutputs = make(map[OutputID]Output)
	s.SpentOutputs = make(map[OutputID]Output)
	s.TransactionPool = make(map[OutputID]*Transaction)
	s.TransactionList = make(map[OutputID]*Transaction)

	// Create the genesis block and add it as the BlockRoot.
	genesisBlock := &Block{
		Timestamp:    GenesisTimestamp,
		MinerAddress: GenesisAddress,
	}
	s.BlockRoot.Block = genesisBlock
	s.BlockRoot.Height = 0
	for i := range s.BlockRoot.RecentTimestamps {
		s.BlockRoot.RecentTimestamps[i] = GenesisTimestamp
	}
	s.BlockRoot.Target[1] = 16 // Easy enough for a home computer to be able to mine on.
	s.BlockRoot.Depth[0] = 255 // depth of genesis block is set to 111111110000000000000000...
	s.BlockMap[genesisBlock.ID()] = s.BlockRoot

	// Fill out the consensus informaiton for the genesis block.
	s.CurrentBlockID = genesisBlock.ID()
	s.CurrentPath[BlockHeight(0)] = genesisBlock.ID()

	// Create the genesis subsidy output.
	genesisSubsidyOutput := Output{
		Value:     CalculateCoinbase(0),
		SpendHash: GenesisAddress,
	}
	s.UnspentOutputs[genesisBlock.SubsidyID()] = genesisSubsidyOutput

	return
}

// SendBlocks sends all known blocks from the given height forward from the
// longest known fork.
func (s *State) SendBlocks(conn net.Conn, data []byte) (err error) {
	// Get the starting point.
	start := BlockHeight(encoding.DecUint64(data))
	if start > s.Height() {
		err = errors.New("start is greater than the height of the longest known fork.")
		return
	}

	// Build an array of blocks.
	var blocks []Block
	for i := start; i < start+MaxCatchUpBlocks; i++ {
		b := s.BlockAtHeight(i)
		if b == nil {
			break
		}
		blocks = append(blocks, *b)
	}

	// Encode and send the blocks.
	encBlocks := encoding.Marshal(blocks)
	encLen := encoding.EncUint64(uint64(len(encBlocks)))
	_, err = conn.Write(append(encLen[:4], encBlocks...))
	if err != nil {
		return
	}

	return
}

// CatchUp requests a maximum of 100 blocks from a peer, starting from the
// current height. It can be called repeatedly to download the full chain.
func (s *State) CatchUp(start BlockHeight) func(net.Conn) error {
	encbh := encoding.EncUint64(uint64(start))

	// TODO: WHY ARE WE RETURNING A FUNCTION? THIS CODE IS REALLY SLOPPY ALSO.
	return func(conn net.Conn) error {
		conn.Write(append([]byte{'R', 4, 0, 0, 0}, encbh[:4]...))
		var blocks []Block
		encBlocks, err := network.ReadPrefix(conn)
		if err != nil {
			return err
		}
		if err = encoding.Unmarshal(encBlocks, &blocks); err != nil {
			return err
		}
		for i := range blocks {
			if err = s.AcceptBlock(blocks[i]); err != nil {
				if err == UnknownOrphanErr && i == 0 {
					if s.Height() <= 20 && start > 1 {
						s.CatchUp(1)
					} else if start > s.Height()-20 {
						// TODO: HAVE LUKE FIX THIS LINE OF CODE. IDEALLY, THIS
						// CODE ALSO ONLY ASKS FOR BLOCKS FROM START-6 TO
						// START, BECAUSE WE'RE ALREADY CAUGHT UP FROM START
						// FORWARD.
						s.CatchUp(start - 6)
					}
				}
			} else if err != BlockKnownErr && err != FutureBlockErr {
				// Return if there's an error, but don't return for benign
				// errors: BlockKnownErr and FutureBlockErr are both benign.
				return err
			}
		}
		if len(blocks) < MaxCatchUpBlocks {
			return errors.New("finished catching up")
		}
		return nil
	}
}
<|endoftext|>


<commit_before>package apk

import (
	"archive/zip"
	"bytes"
	"io/ioutil"
	"testing"
)

func TestFormatFilenameShort(t *testing.T) {
	t.Parallel()

	fn := []byte("LocalizedFormats_fr.properties")
	expected := []byte("LocalizedFormats_fr.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameLong(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties")
	expected := []byte("assets/org/apache/commons/math3/exception/util/LocalizedFormats_f\n r.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameInvalidUTF8(t *testing.T) {
	t.Parallel()

	_, err := formatFilename([]byte{0xff, 0xfe, 0xfd})
	if err == nil {
		t.Fatal("format filename did not error for invalid UTF8")
	}
}

func TestFormatFilenameLonger(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commons/math3/exception/util/assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties")
	expected := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commo\n ns/math3/exception/util/assets/org/apache/commons/math3/exception/util\n /LocalizedFormats_fr.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameExact(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commons/math3/exception/util/assets/org/apache/commons/math3/exception/util")
	expected := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commo\n ns/math3/exception/util/assets/org/apache/commons/math3/exception/util")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%+v\nGot:\n%+v", expected, formatted)
	}
}

func TestMakingJarManifest(t *testing.T) {
	manifest, sigfile, err := makeJARManifests(smallZip)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(manifest, smallZipManifest) {
		t.Fatalf("manifest mismatch. Expect:\n%s\nGot:\n%s", smallZipManifest, manifest)
	}
	if !bytes.Equal(sigfile, smallZipSignatureFile) {
		t.Fatalf("signature file mismatch. Expect:\n%s\nGot:\n%s", smallZipSignatureFile, sigfile)
	}
}

func TestRepackAndAlign(t *testing.T) {
	repackedZip, err := repackAndAlignJAR(smallZip, smallZipManifest, smallZipSignatureFile, smallZipSignature)
	if err != nil {
		t.Fatal(err)
	}

	zipReader := bytes.NewReader(repackedZip)
	r, err := zip.NewReader(zipReader, int64(len(repackedZip)))
	if err != nil {
		t.Fatal(err)
	}
	var hasData, hasManifest, hasSignatureFile, hasSignature bool
	for _, f := range r.File {
		rc, err := f.Open()
		defer rc.Close()
		if err != nil {
			t.Fatal(err)
		}
		data, err := ioutil.ReadAll(rc)
		if err != nil {
			t.Fatal(err)
		}
		switch f.Name {
		case "foo.txt":
			if !bytes.Equal(data, []byte("bar \n")) {
				t.Fatalf("data mismatch. Expect: 'bar \\n' Got:%q", data)
			}
			hasData = true
		case "META-INF/MANIFEST.MF":
			if !bytes.Equal(data, smallZipManifest) {
				t.Fatalf("manifest mismatch. Expect:\n%s\nGot:\n%s", smallZipManifest, data)
			}
			hasManifest = true
		case "META-INF/SIGNATURE.SF":
			if !bytes.Equal(data, smallZipSignatureFile) {
				t.Fatalf("signature file mismatch. Expect:\n%s\nGot:\n%s", smallZipSignatureFile, data)
			}
			hasSignatureFile = true
		case "META-INF/SIGNATURE.RSA":
			if !bytes.Equal(data, smallZipSignature) {
				t.Fatalf("signature mismatch. Expect:\n%x\nGot:\n%x", smallZipSignature, data)
			}
			hasSignature = true
		default:
			t.Fatalf("found unknown file in zip archive: %s", f.Name)
		}
	}
	if !hasData {
		t.Fatal("data file not found in zip archive")
	}
	if !hasManifest {
		t.Fatal("manifest file not found in zip archive")
	}
	if !hasSignatureFile {
		t.Fatal("signature file not found in zip archive")
	}
	if !hasSignature {
		t.Fatal("signature not found in zip archive")
	}
}

func TestIsSignatureFile(t *testing.T) {
	var testcases = []struct {
		expect   bool
		filename string
	}{
		{true, "META-INF/SIGNATURE.RSA"},
		{true, "META-INF/signature.rsa"},
		{true, "META-INF/SiGnAtUre.RSA"},
		{true, "META-INF/SIGNATURE.DSA"},
		{true, "META-INF/signature.dsa"},
		{true, "META-INF/SiGnAtUre.DSA"},
		{true, "META-INF/MANIFEST.MF"},
		{true, "META-INF/manifest.mf"},
		{true, "META-INF/signature.sf"},
		{true, "META-INF/signature.SF"},
		{true, "META-INF/SIG-foo"},
		{true, "META-INF/sig-bar"},
		{false, "META-INF/foo.bar"},
		{false, "META-INF/foo.rsa.bar"},
		{false, "META-INF/foo.RSA.bar"},
		{false, "META-INF/.mf.foo"},
	}
	for i, testcase := range testcases {
		if isSignatureFile(testcase.filename) != testcase.expect {
			t.Fatalf("testcase %d failed. %q returned %t, expected %t",
				i, testcase.filename, isSignatureFile(testcase.filename), testcase.expect)
		}
	}
}

// Fixtures can be added by converting APKs to string literals using hexdump, eg:
// hexdump -v -e '16/1 "_x%02X" "\n"' /tmp/fakeapk/fakeapk.zip | sed 's/_/\\/g; s/\\x  //g; s/.*/    "&"/'

// a dummy zip file with a single file in it
var smallZip = []byte("\x50\x4B\x03\x04\x0A\x00\x00\x00\x00\x00\x0D\x8E\x70\x4C\xD4\x32" +
	"\x6E\x84\x05\x00\x00\x00\x05\x00\x00\x00\x07\x00\x1C\x00\x66\x6F" +
	"\x6F\x2E\x74\x78\x74\x55\x54\x09\x00\x03\xAA\x3B\xAC\x5A\xAA\x3B" +
	"\xAC\x5A\x75\x78\x0B\x00\x01\x04\xE8\x03\x00\x00\x04\xE8\x03\x00" +
	"\x00\x62\x61\x72\x20\x0A\x50\x4B\x01\x02\x1E\x03\x0A\x00\x00\x00" +
	"\x00\x00\x0D\x8E\x70\x4C\xD4\x32\x6E\x84\x05\x00\x00\x00\x05\x00" +
	"\x00\x00\x07\x00\x18\x00\x00\x00\x00\x00\x01\x00\x00\x00\xA4\x81" +
	"\x00\x00\x00\x00\x66\x6F\x6F\x2E\x74\x78\x74\x55\x54\x05\x00\x03" +
	"\xAA\x3B\xAC\x5A\x75\x78\x0B\x00\x01\x04\xE8\x03\x00\x00\x04\xE8" +
	"\x03\x00\x00\x50\x4B\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00\x4D" +
	"\x00\x00\x00\x46\x00\x00\x00\x00\x00")

var smallZipManifest = []byte(`Manifest-Version: 1.0
Built-By: Generated-by-Autograph
Created-By: go.mozilla.org/autograph

Name: foo.txt
SHA-256-Digest: aE/i11OBml60LVIdT0GjCqhQdvwtRRY+sL8ySX8+1EY=

`)

var smallZipSignatureFile = []byte(`Signature-Version: 1.0
Created-By: 1.0.0 autograph-client (go.mozilla.org/autograph)
SHA-256-Digest-Manifest: LpKUd4ScoPSLipDW2FHd6XJqnxx3pgCBEXygZPGQFCw=

Name: foo.txt
SHA-256-Digest: aE/i11OBml60LVIdT0GjCqhQdvwtRRY+sL8ySX8+1EY=

`)

var smallZipSignature = []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
<commit_msg>signer: parallelize APK JAR tests<commit_after>package apk

import (
	"archive/zip"
	"bytes"
	"io/ioutil"
	"testing"
)

func TestFormatFilenameShort(t *testing.T) {
	t.Parallel()

	fn := []byte("LocalizedFormats_fr.properties")
	expected := []byte("LocalizedFormats_fr.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameLong(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties")
	expected := []byte("assets/org/apache/commons/math3/exception/util/LocalizedFormats_f\n r.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameInvalidUTF8(t *testing.T) {
	t.Parallel()

	_, err := formatFilename([]byte{0xff, 0xfe, 0xfd})
	if err == nil {
		t.Fatal("format filename did not error for invalid UTF8")
	}
}

func TestFormatFilenameLonger(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commons/math3/exception/util/assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties")
	expected := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commo\n ns/math3/exception/util/assets/org/apache/commons/math3/exception/util\n /LocalizedFormats_fr.properties")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%s\nGot:\n%s", expected, formatted)
	}
}

func TestFormatFilenameExact(t *testing.T) {
	t.Parallel()

	fn := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commons/math3/exception/util/assets/org/apache/commons/math3/exception/util")
	expected := []byte("assets/org/apache/commons/math3/exception/assets/org/apache/commo\n ns/math3/exception/util/assets/org/apache/commons/math3/exception/util")

	formatted, err := formatFilename(fn)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(formatted, expected) {
		t.Fatalf("manifest filename mismatch Expected:\n%+v\nGot:\n%+v", expected, formatted)
	}
}

func TestMakingJarManifest(t *testing.T) {
	t.Parallel()

	manifest, sigfile, err := makeJARManifests(smallZip)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(manifest, smallZipManifest) {
		t.Fatalf("manifest mismatch. Expect:\n%s\nGot:\n%s", smallZipManifest, manifest)
	}
	if !bytes.Equal(sigfile, smallZipSignatureFile) {
		t.Fatalf("signature file mismatch. Expect:\n%s\nGot:\n%s", smallZipSignatureFile, sigfile)
	}
}

func TestRepackAndAlign(t *testing.T) {
	t.Parallel()

	repackedZip, err := repackAndAlignJAR(smallZip, smallZipManifest, smallZipSignatureFile, smallZipSignature)
	if err != nil {
		t.Fatal(err)
	}

	zipReader := bytes.NewReader(repackedZip)
	r, err := zip.NewReader(zipReader, int64(len(repackedZip)))
	if err != nil {
		t.Fatal(err)
	}
	var hasData, hasManifest, hasSignatureFile, hasSignature bool
	for _, f := range r.File {
		rc, err := f.Open()
		defer rc.Close()
		if err != nil {
			t.Fatal(err)
		}
		data, err := ioutil.ReadAll(rc)
		if err != nil {
			t.Fatal(err)
		}
		switch f.Name {
		case "foo.txt":
			if !bytes.Equal(data, []byte("bar \n")) {
				t.Fatalf("data mismatch. Expect: 'bar \\n' Got:%q", data)
			}
			hasData = true
		case "META-INF/MANIFEST.MF":
			if !bytes.Equal(data, smallZipManifest) {
				t.Fatalf("manifest mismatch. Expect:\n%s\nGot:\n%s", smallZipManifest, data)
			}
			hasManifest = true
		case "META-INF/SIGNATURE.SF":
			if !bytes.Equal(data, smallZipSignatureFile) {
				t.Fatalf("signature file mismatch. Expect:\n%s\nGot:\n%s", smallZipSignatureFile, data)
			}
			hasSignatureFile = true
		case "META-INF/SIGNATURE.RSA":
			if !bytes.Equal(data, smallZipSignature) {
				t.Fatalf("signature mismatch. Expect:\n%x\nGot:\n%x", smallZipSignature, data)
			}
			hasSignature = true
		default:
			t.Fatalf("found unknown file in zip archive: %s", f.Name)
		}
	}
	if !hasData {
		t.Fatal("data file not found in zip archive")
	}
	if !hasManifest {
		t.Fatal("manifest file not found in zip archive")
	}
	if !hasSignatureFile {
		t.Fatal("signature file not found in zip archive")
	}
	if !hasSignature {
		t.Fatal("signature not found in zip archive")
	}
}

func TestIsSignatureFile(t *testing.T) {
	t.Parallel()

	var testcases = []struct {
		expect   bool
		filename string
	}{
		{true, "META-INF/SIGNATURE.RSA"},
		{true, "META-INF/signature.rsa"},
		{true, "META-INF/SiGnAtUre.RSA"},
		{true, "META-INF/SIGNATURE.DSA"},
		{true, "META-INF/signature.dsa"},
		{true, "META-INF/SiGnAtUre.DSA"},
		{true, "META-INF/MANIFEST.MF"},
		{true, "META-INF/manifest.mf"},
		{true, "META-INF/signature.sf"},
		{true, "META-INF/signature.SF"},
		{true, "META-INF/SIG-foo"},
		{true, "META-INF/sig-bar"},
		{false, "META-INF/foo.bar"},
		{false, "META-INF/foo.rsa.bar"},
		{false, "META-INF/foo.RSA.bar"},
		{false, "META-INF/.mf.foo"},
	}
	for i, testcase := range testcases {
		if isSignatureFile(testcase.filename) != testcase.expect {
			t.Fatalf("testcase %d failed. %q returned %t, expected %t",
				i, testcase.filename, isSignatureFile(testcase.filename), testcase.expect)
		}
	}
}

// Fixtures can be added by converting APKs to string literals using hexdump, eg:
// hexdump -v -e '16/1 "_x%02X" "\n"' /tmp/fakeapk/fakeapk.zip | sed 's/_/\\/g; s/\\x  //g; s/.*/    "&"/'

// a dummy zip file with a single file in it
var smallZip = []byte("\x50\x4B\x03\x04\x0A\x00\x00\x00\x00\x00\x0D\x8E\x70\x4C\xD4\x32" +
	"\x6E\x84\x05\x00\x00\x00\x05\x00\x00\x00\x07\x00\x1C\x00\x66\x6F" +
	"\x6F\x2E\x74\x78\x74\x55\x54\x09\x00\x03\xAA\x3B\xAC\x5A\xAA\x3B" +
	"\xAC\x5A\x75\x78\x0B\x00\x01\x04\xE8\x03\x00\x00\x04\xE8\x03\x00" +
	"\x00\x62\x61\x72\x20\x0A\x50\x4B\x01\x02\x1E\x03\x0A\x00\x00\x00" +
	"\x00\x00\x0D\x8E\x70\x4C\xD4\x32\x6E\x84\x05\x00\x00\x00\x05\x00" +
	"\x00\x00\x07\x00\x18\x00\x00\x00\x00\x00\x01\x00\x00\x00\xA4\x81" +
	"\x00\x00\x00\x00\x66\x6F\x6F\x2E\x74\x78\x74\x55\x54\x05\x00\x03" +
	"\xAA\x3B\xAC\x5A\x75\x78\x0B\x00\x01\x04\xE8\x03\x00\x00\x04\xE8" +
	"\x03\x00\x00\x50\x4B\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00\x4D" +
	"\x00\x00\x00\x46\x00\x00\x00\x00\x00")

var smallZipManifest = []byte(`Manifest-Version: 1.0
Built-By: Generated-by-Autograph
Created-By: go.mozilla.org/autograph

Name: foo.txt
SHA-256-Digest: aE/i11OBml60LVIdT0GjCqhQdvwtRRY+sL8ySX8+1EY=

`)

var smallZipSignatureFile = []byte(`Signature-Version: 1.0
Created-By: 1.0.0 autograph-client (go.mozilla.org/autograph)
SHA-256-Digest-Manifest: LpKUd4ScoPSLipDW2FHd6XJqnxx3pgCBEXygZPGQFCw=

Name: foo.txt
SHA-256-Digest: aE/i11OBml60LVIdT0GjCqhQdvwtRRY+sL8ySX8+1EY=

`)

var smallZipSignature = []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00")
<|endoftext|>


<commit_before>package review

import (
	"context"
	"fmt"
	"os"
	"strings"

	flowutil "github.com/codelingo/codelingo/sdk/flow"
	"github.com/codelingo/lingo/app/commands/verify"
	"github.com/codelingo/lingo/app/util"
	"github.com/codelingo/lingo/app/util/common/config"
	"github.com/codelingo/lingo/vcs"
	"github.com/codelingo/rpc/flow"
	"github.com/golang/protobuf/proto"
	"github.com/juju/errors"
	"github.com/urfave/cli"
)

const (
	vcsGit string = "git"
	vcsP4  string = "perforce"
)

var CLIApp = &flowutil.CLIApp{
	App: cli.App{

		Name:    "review",
		Usage:   "Review code following tenets in codelingo.yaml.",
		Version: "0.0.0",

		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  util.DiffFlg.String(),
				Usage: "Review only unstaged changes in the working tree.",
			},
			cli.StringFlag{
				Name:  util.OutputFlg.String(),
				Usage: "File to save found issues to.",
			},
			cli.StringFlag{
				Name:  util.FormatFlg.String(),
				Value: "json-pretty",
				Usage: "How to format the found issues. Possible values are: json, json-pretty.",
			},
			cli.BoolFlag{
				Name:  util.KeepAllFlg.String(),
				Usage: "Keep all issues and don't be prompted to confirm each issue.",
			},
			cli.StringFlag{
				Name:  util.DirectoryFlg.String(),
				Usage: "Review a given directory.",
			},
			cli.BoolFlag{
				Name:  "debug",
				Usage: "Display debug messages",
			},
			cli.BoolFlag{
				Name:   "insecure",
				Hidden: true,
				Usage:  "Review without TLS",
			},
			// cli.BoolFlag{
			// 	Name:  "all",
			// 	Usage: "review all files under all directories from pwd down",
			// },
		},
		// "$ lingo review" will review any unstaged changes from pwd down.
		// "$ lingo review [<filename>]" will review any unstaged changes in the named files.
		// "$ lingo review --all [<filename>]" will review all code in the named files.
	},
	Tagline: "Automated Code Reviews",
	Request: reviewAction,
}

func reviewRequire() error {
	reqs := []verify.Require{verify.VCSRq, verify.HomeRq, verify.AuthRq, verify.ConfigRq, verify.VersionRq}
	for _, req := range reqs {
		err := req.Verify()
		if err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}

func reviewAction(cliCtx *cli.Context) (chan proto.Message, <-chan *flowutil.UserVar, chan error, func(), error) {
	err := reviewRequire()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)
	}

	defer util.Logger.Sync()
	if cliCtx.IsSet("debug") {
		util.Logger.Debugw("reviewCMD called")
	}
	dir := cliCtx.String("directory")
	if dir != "" {
		if err := os.Chdir(dir); err != nil {
			return nil, nil, nil, nil, errors.Trace(err)
		}
	}

	insecure := cliCtx.IsSet("insecure")
	util.Logger.Debugf("insecure %t", insecure)

	dotlingo, err := ReadDotLingo(cliCtx)
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	vcsType, repo, err := vcs.New()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	// TODO: replace this system with nfs-like communication.
	fmt.Println("Syncing your repo...")
	if err = vcs.SyncRepo(vcsType, repo); err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	owner, name, err := repo.OwnerAndNameFromRemote()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	sha, err := repo.CurrentCommitId()
	if err != nil {
		if flowutil.NoCommitErr(err) {
			return nil, nil, nil, nil, errors.New(flowutil.NoCommitErrMsg)
		}

		return nil, nil, nil, nil, errors.Trace(err)

	}

	patches, err := repo.Patches()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	var patchesSize int64
	for _, patch := range patches {
		patchesSize += int64(len([]byte(patch)))
	}
	if patchesSize >= 1024*1024 { // >= 1MB; default max GRPC msg size accepted by the servers is 4MB.
		util.UserFacingWarning("Warning: large diffs can be error prone. You may need to commit your changes.")
	}

	workingDir, err := repo.WorkingDir()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	cfg, err := config.Platform()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	vcsTypeStr, err := vcs.TypeToString(vcsType)
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	parts := strings.Split(wd, "/")
	path := parts[2] + "/" + parts[len(parts)-1]

	ctx, cancel := util.UserCancelContext(context.Background())
	req := &flow.ReviewRequest{
		Repo:     name,
		Sha:      sha,
		Patches:  patches,
		Vcs:      vcsTypeStr,
		Dir:      workingDir,
		Dotlingo: dotlingo,
		Path:     path,
	}
	switch vcsTypeStr {
	case vcsGit:
		addr, err := cfg.GitServerAddr()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		hostname, err := cfg.GitRemoteName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}

		req.Host = addr
		req.Hostname = hostname
		req.OwnerOrDepot = &flow.ReviewRequest_Owner{owner}
	case vcsP4:
		addr, err := cfg.P4ServerAddr()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		hostname, err := cfg.P4RemoteName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		depot, err := cfg.P4RemoteDepotName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		name = owner + "/" + name

		req.Host = addr
		req.Hostname = hostname
		req.OwnerOrDepot = &flow.ReviewRequest_Depot{depot}
		req.Repo = name
	default:
		return nil, nil, nil, nil, errors.Errorf("Invalid VCS '%s'", vcsTypeStr)
	}

	fmt.Println("Running review flow...")
	resultc, userVarc, errc, err := RequestReview(ctx, req, insecure)
	return resultc, userVarc, errc, cancel, errors.Trace(err)
}
<commit_msg>Find path for go projects<commit_after>package review

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	flowutil "github.com/codelingo/codelingo/sdk/flow"
	"github.com/codelingo/lingo/app/commands/verify"
	"github.com/codelingo/lingo/app/util"
	"github.com/codelingo/lingo/app/util/common/config"
	"github.com/codelingo/lingo/vcs"
	"github.com/codelingo/rpc/flow"
	"github.com/golang/protobuf/proto"
	"github.com/juju/errors"
	"github.com/urfave/cli"
)

const (
	vcsGit string = "git"
	vcsP4  string = "perforce"
)

var CLIApp = &flowutil.CLIApp{
	App: cli.App{

		Name:    "review",
		Usage:   "Review code following tenets in codelingo.yaml.",
		Version: "0.0.0",

		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  util.DiffFlg.String(),
				Usage: "Review only unstaged changes in the working tree.",
			},
			cli.StringFlag{
				Name:  util.OutputFlg.String(),
				Usage: "File to save found issues to.",
			},
			cli.StringFlag{
				Name:  util.FormatFlg.String(),
				Value: "json-pretty",
				Usage: "How to format the found issues. Possible values are: json, json-pretty.",
			},
			cli.BoolFlag{
				Name:  util.KeepAllFlg.String(),
				Usage: "Keep all issues and don't be prompted to confirm each issue.",
			},
			cli.StringFlag{
				Name:  util.DirectoryFlg.String(),
				Usage: "Review a given directory.",
			},
			cli.BoolFlag{
				Name:  "debug",
				Usage: "Display debug messages",
			},
			cli.BoolFlag{
				Name:   "insecure",
				Hidden: true,
				Usage:  "Review without TLS",
			},
			// cli.BoolFlag{
			// 	Name:  "all",
			// 	Usage: "review all files under all directories from pwd down",
			// },
		},
		// "$ lingo review" will review any unstaged changes from pwd down.
		// "$ lingo review [<filename>]" will review any unstaged changes in the named files.
		// "$ lingo review --all [<filename>]" will review all code in the named files.
	},
	Tagline: "Automated Code Reviews",
	Request: reviewAction,
}

func reviewRequire() error {
	reqs := []verify.Require{verify.VCSRq, verify.HomeRq, verify.AuthRq, verify.ConfigRq, verify.VersionRq}
	for _, req := range reqs {
		err := req.Verify()
		if err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}

func reviewAction(cliCtx *cli.Context) (chan proto.Message, <-chan *flowutil.UserVar, chan error, func(), error) {
	err := reviewRequire()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)
	}

	defer util.Logger.Sync()
	if cliCtx.IsSet("debug") {
		util.Logger.Debugw("reviewCMD called")
	}
	dir := cliCtx.String("directory")
	if dir != "" {
		if err := os.Chdir(dir); err != nil {
			return nil, nil, nil, nil, errors.Trace(err)
		}
	}

	insecure := cliCtx.IsSet("insecure")
	util.Logger.Debugf("insecure %t", insecure)

	dotlingo, err := ReadDotLingo(cliCtx)
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	vcsType, repo, err := vcs.New()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	// TODO: replace this system with nfs-like communication.
	fmt.Println("Syncing your repo...")
	if err = vcs.SyncRepo(vcsType, repo); err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	owner, name, err := repo.OwnerAndNameFromRemote()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	sha, err := repo.CurrentCommitId()
	if err != nil {
		if flowutil.NoCommitErr(err) {
			return nil, nil, nil, nil, errors.New(flowutil.NoCommitErrMsg)
		}

		return nil, nil, nil, nil, errors.Trace(err)

	}

	patches, err := repo.Patches()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	var patchesSize int64
	for _, patch := range patches {
		patchesSize += int64(len([]byte(patch)))
	}
	if patchesSize >= 1024*1024 { // >= 1MB; default max GRPC msg size accepted by the servers is 4MB.
		util.UserFacingWarning("Warning: large diffs can be error prone. You may need to commit your changes.")
	}

	workingDir, err := repo.WorkingDir()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}

	cfg, err := config.Platform()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	vcsTypeStr, err := vcs.TypeToString(vcsType)
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	// Build `path` for go projects as "reponame" for a local repo or "github.com/username/reponame" for a github repo
	// path variable is initially empty and will remain empty for none go repos
	path := ""

	// Find the current working directory, i.e. where user executed `lingo run review` command on the repo
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)

	}
	// Check if the repo is a go project
	goProject, err := isGoProject(wd)
	if err != nil {
		return nil, nil, nil, nil, errors.Trace(err)
	}

	// Check if GOPATH is set
	goPath := os.Getenv("GOPATH")

	if goPath != "" && goProject {
		if strings.Contains(wd, "github.com") { // Check if the working directory includes "github.com/username/reponame"
			swd := wd[strings.Index(wd, "github.com"):]
			parts := strings.Split(swd, "/")
			if len(parts) == 3 {
				path = swd
			}
		} else if strings.HasPrefix(wd, goPath+"/src/") { //Check if repo is local and lives in GOPATH/src/reponame
			swd := wd[strings.Index(wd, "src/"):]
			parts := strings.Split(swd, "/")
			if len(parts) == 2 {
				path = parts[1]
			}
		}
		// If none of the above, path remains empty
	}

	ctx, cancel := util.UserCancelContext(context.Background())

	var req *flow.ReviewRequest
	// If path is not empty, it is sent in ReviewRequest and used in go lexicon
	if path != "" {
		req = &flow.ReviewRequest{
			Repo:     name,
			Sha:      sha,
			Patches:  patches,
			Vcs:      vcsTypeStr,
			Dir:      workingDir,
			Dotlingo: dotlingo,
			Path:     path,
		}
	} else {
		req = &flow.ReviewRequest{
			Repo:     name,
			Sha:      sha,
			Patches:  patches,
			Vcs:      vcsTypeStr,
			Dir:      workingDir,
			Dotlingo: dotlingo,
		}
	}

	switch vcsTypeStr {
	case vcsGit:
		addr, err := cfg.GitServerAddr()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		hostname, err := cfg.GitRemoteName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}

		req.Host = addr
		req.Hostname = hostname
		req.OwnerOrDepot = &flow.ReviewRequest_Owner{owner}
	case vcsP4:
		addr, err := cfg.P4ServerAddr()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		hostname, err := cfg.P4RemoteName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		depot, err := cfg.P4RemoteDepotName()
		if err != nil {
			return nil, nil, nil, nil, errors.Trace(err)

		}
		name = owner + "/" + name

		req.Host = addr
		req.Hostname = hostname
		req.OwnerOrDepot = &flow.ReviewRequest_Depot{depot}
		req.Repo = name
	default:
		return nil, nil, nil, nil, errors.Errorf("Invalid VCS '%s'", vcsTypeStr)
	}

	fmt.Println("Running review flow...")
	resultc, userVarc, errc, err := RequestReview(ctx, req, insecure)
	return resultc, userVarc, errc, cancel, errors.Trace(err)
}

func isGoProject(currentDir string) (bool, error) {
	isGo := false
	items, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return false, err
	}
	for _, item := range items {
		if item.IsDir() {
			isGo, err = isGoProject(filepath.Join(currentDir, item.Name()))
			if err != nil {
				return false, err
			}
		} else if filepath.Ext(item.Name()) == ".go" {
			isGo = true
			break
		}
	}
	return isGo, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"time"

	"github.com/antage/eventsource"
)

var domains = [...]string{
	"facebook.com",
	"disney.com",
	"google.com",
	"microsoft.com",
	"foobar.io",
	"fast.com",
	"netflix.com",
	"anandtech.com",
	"arstechnica.com",
	"www.bbc.co.uk",
	"tumblr.com",
	"instagram.com",
	"twitter.com",
	"phoronix.com",
	"channel4.com",
	"theregister.co.uk",
	"gov.uk",
}

var bricksong = [...]string{
	"Dear Sir I write this note to inform you of my plight",
	"And at the time of writing I am not a pretty sight",
	"My body is all black and blue, my face a deathly gray",
	"I write this note to tell why Paddy's not at work today",

	"While working on the fourteenth floor, some bricks I had to clear",
	"And to throw them down from off the top seemed quite a good idea",
	"But the gaffer wasn't very pleased, he was an awful sod",
	"He said I had to cart them down the ladder in me hod.",

	"Well clearing all those bricks by hand, it seemed so very slow",
	"So I hoisted up a barrel and secured the rope below",
	"But in my haste to do the job, I was too blind to see",
	"That a barrel full of building bricks is heavier than me.",

	"So when I had untied the rope, the barrel fell like lead",
	"And clinging tightly to the rope I started up instead",
	"I took off like a rocket and to my dismay I found",
	"That half way up I met the bloody barrel coming down.",

	"Well the barrel broke my shoulder as on to the ground it sped",
	"And when I reached the top I banged the pulley with me head",
	"I held on tight, though numb with shock from this almighty blow",
	"And the barrel spilled out half its load fourteen floors below",

	"Now when those building bricks fell from the barrel to the floor",
	"I then outweighed the barrel so I started down once more",
	"I held on tightly to the rope as I flew to the ground",
	"And I landed on those building bricks that were scattered all around.",

	"Now as I lay there on the deck I thought I'd passed the worst",
	"But when the barrel reached the top, that's when the bottom burst",
	"A shower of bricks came down on me, I knew I had no hope",
	"In all of this confusion, I let go the bloody rope.",

	"The barrel being heavier, it started down once more",
	"And landed right on top of me as I lay on the floor",
	"It broke three ribs and my left arm, and I can only say",
	"That I hope you'll understand why Paddy's not at work today.",
}

type Source struct {
	Name string
	Type string
}

type Fragment struct {
	Id int
	Source
	Text string
}

type Selection struct {
	Fid   int
	Words []int
}

type Domain struct {
	Id int
	Source
	Text string
}

func main() {
	es := eventsource.New(
		&eventsource.Settings{
			Timeout:        5 * time.Second,
			CloseOnTimeout: false,
			IdleTimeout:    30 * time.Minute,
		},
		func(req *http.Request) [][]byte {
			return [][]byte{
				[]byte("X-Accel-Buffering: no"),
				[]byte("Access-Control-Allow-Origin: *"),
				[]byte("Content-Type: text/event-stream"),
				[]byte("Cache-Control: no-cache"),
				[]byte("Connection: keep-alive"),
			}
		},
	)

	// Fixed seed so pattern is repeatable.
	rand.Seed(42)

	defer es.Close()
	http.Handle("/events", es)
	go func() {
		i, id := 0, 0
		bl := len(bricksong)
		dl := len(domains)
		for {
			bi := i % bl
			di := i % dl
			f := Fragment{bi, Source{Name: "SimUser", Type: "User"}, bricksong[bi]}
			d := Domain{di, Source{Name: "SimUser", Type: "User"}, domains[di]}
			jsb, _ := json.Marshal(f)
			jsd, _ := json.Marshal(d)
			es.SendEventMessage(string(jsb), "fragment", strconv.Itoa(id))
			id++
			es.SendEventMessage(string(jsd), "domain", strconv.Itoa(id))
			id++

			if i > bl {
				s := Selection{rand.Intn(bl), []int{1, 3}}
				jss, _ := json.Marshal(s)
				es.SendEventMessage(string(jss), "selection", strconv.Itoa(id))
				id++
			}
			i++
			time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
		}
	}()
	log.Fatal(http.ListenAndServe(":8080", nil))
}
<commit_msg>Use default random seed.<commit_after>package main

import (
	"encoding/json"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"time"

	"github.com/antage/eventsource"
)

var domains = [...]string{
	"facebook.com",
	"disney.com",
	"google.com",
	"microsoft.com",
	"foobar.io",
	"fast.com",
	"netflix.com",
	"anandtech.com",
	"arstechnica.com",
	"www.bbc.co.uk",
	"tumblr.com",
	"instagram.com",
	"twitter.com",
	"phoronix.com",
	"channel4.com",
	"theregister.co.uk",
	"gov.uk",
}

var bricksong = [...]string{
	"Dear Sir I write this note to inform you of my plight",
	"And at the time of writing I am not a pretty sight",
	"My body is all black and blue, my face a deathly gray",
	"I write this note to tell why Paddy's not at work today",

	"While working on the fourteenth floor, some bricks I had to clear",
	"And to throw them down from off the top seemed quite a good idea",
	"But the gaffer wasn't very pleased, he was an awful sod",
	"He said I had to cart them down the ladder in me hod.",

	"Well clearing all those bricks by hand, it seemed so very slow",
	"So I hoisted up a barrel and secured the rope below",
	"But in my haste to do the job, I was too blind to see",
	"That a barrel full of building bricks is heavier than me.",

	"So when I had untied the rope, the barrel fell like lead",
	"And clinging tightly to the rope I started up instead",
	"I took off like a rocket and to my dismay I found",
	"That half way up I met the bloody barrel coming down.",

	"Well the barrel broke my shoulder as on to the ground it sped",
	"And when I reached the top I banged the pulley with me head",
	"I held on tight, though numb with shock from this almighty blow",
	"And the barrel spilled out half its load fourteen floors below",

	"Now when those building bricks fell from the barrel to the floor",
	"I then outweighed the barrel so I started down once more",
	"I held on tightly to the rope as I flew to the ground",
	"And I landed on those building bricks that were scattered all around.",

	"Now as I lay there on the deck I thought I'd passed the worst",
	"But when the barrel reached the top, that's when the bottom burst",
	"A shower of bricks came down on me, I knew I had no hope",
	"In all of this confusion, I let go the bloody rope.",

	"The barrel being heavier, it started down once more",
	"And landed right on top of me as I lay on the floor",
	"It broke three ribs and my left arm, and I can only say",
	"That I hope you'll understand why Paddy's not at work today.",
}

type Source struct {
	Name string
	Type string
}

type Fragment struct {
	Id int
	Source
	Text string
}

type Selection struct {
	Fid   int
	Words []int
}

type Domain struct {
	Id int
	Source
	Text string
}

func main() {
	es := eventsource.New(
		&eventsource.Settings{
			Timeout:        5 * time.Second,
			CloseOnTimeout: false,
			IdleTimeout:    30 * time.Minute,
		},
		func(req *http.Request) [][]byte {
			return [][]byte{
				[]byte("X-Accel-Buffering: no"),
				[]byte("Access-Control-Allow-Origin: *"),
				[]byte("Content-Type: text/event-stream"),
				[]byte("Cache-Control: no-cache"),
				[]byte("Connection: keep-alive"),
			}
		},
	)

	defer es.Close()
	http.Handle("/events", es)
	go func() {
		i, id := 0, 0
		bl := len(bricksong)
		dl := len(domains)
		for {
			bi := i % bl
			di := i % dl
			f := Fragment{bi, Source{Name: "SimUser", Type: "User"}, bricksong[bi]}
			d := Domain{di, Source{Name: "SimUser", Type: "User"}, domains[di]}
			jsb, _ := json.Marshal(f)
			jsd, _ := json.Marshal(d)
			es.SendEventMessage(string(jsb), "fragment", strconv.Itoa(id))
			id++
			es.SendEventMessage(string(jsd), "domain", strconv.Itoa(id))
			id++

			if i > bl {
				s := Selection{rand.Intn(bl), []int{1, 3}}
				jss, _ := json.Marshal(s)
				es.SendEventMessage(string(jss), "selection", strconv.Itoa(id))
				id++
			}
			i++
			time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond)
		}
	}()
	log.Fatal(http.ListenAndServe(":8080", nil))
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

// Package simulator provides a go interface to the Microsoft TPM2 simulator.
package simulator

// // Directories containing .h files in the simulator source
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/include
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/include/prototypes
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/include
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/include/prototypes
// // Allows simulator.c to import files without repeating the source repo path.
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/src
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/src
// // We want to store the NVDATA in memory, not on a file.
// #cgo CFLAGS: -DVTPM=NO -DSIMULATION=NO
// // Flags from ms-tpm-20-ref/TPMCmd/configure.ac
// #cgo CFLAGS: -std=gnu11 -Wall -Wformat-security -fstack-protector-all -fPIC
// // CGO Generates bad code, silence the warnings from it.
// #cgo CFLAGS: -Wall -Wno-unused-variable
// // Link against the system OpenSSL
// #cgo CFLAGS: -DHASH_LIB=Ossl -DSYM_LIB=Ossl -DMATH_LIB=Ossl
// #cgo LDFLAGS: -lcrypto
//
// #include <stdlib.h>
// #include "Tpm.h"
//
// void sync_seeds() {
//     NV_SYNC_PERSISTENT(EPSeed);
//     NV_SYNC_PERSISTENT(SPSeed);
//     NV_SYNC_PERSISTENT(PPSeed);
// }
import "C"
import (
	"bytes"
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"unsafe"

	"github.com/google/go-tpm/tpm2"
)

// Simulator represents a go-tpm compatible interface to the IBM TPM2 simulator.
// Similar to the file-based (for linux) or syscall-based (for Windows) TPM
// handles, no synchronization is provided; the same simulator handle should not
// be used from multiple threads.
type Simulator struct {
	buf bytes.Buffer
}

// ErrSimulatorInUse indicates another open Simulator already exists.
var ErrSimulatorInUse = errors.New("simulator is being used by another caller")

// The simulator is a global resource, so we use the variables below to make
// sure we only ever have one open reference to the Simulator at a time.
var (
	lock  sync.Mutex
	inUse bool
)

// Get the pointer to an initialized, powered on, and started simulator. As only
// one simulator may be running at a time, a second call to Get() will return
// ErrSimulatorInUse until the first Simulator is Closed.
func Get() (*Simulator, error) {
	lock.Lock()
	defer lock.Unlock()
	if inUse {
		return nil, ErrSimulatorInUse
	}

	simulator := &Simulator{}
	if err := simulator.on(true); err != nil {
		return nil, err
	}
	inUse = true
	return simulator, nil
}

// GetWithFixedSeedInsecure behaves like Get() expect that all of the internal
// hierarchy seeds are derived from the input seed. Note that this function
// compromises the security of the keys/seeds and should only be used for tests.
func GetWithFixedSeedInsecure(seed int64) (*Simulator, error) {
	r := rand.New(rand.NewSource(seed))
	s, err := Get()
	if err != nil {
		return nil, err
	}

	// The first two bytes of the seed encode the size (so we don't overwrite)
	r.Read(C.gp.EPSeed[2:])
	r.Read(C.gp.SPSeed[2:])
	r.Read(C.gp.PPSeed[2:])
	return s, nil
}

// Reset the TPM as if the host computer had rebooted.
func (s *Simulator) Reset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(false)
}

// ManufactureReset behaves like Reset() except that the TPM is complete wiped.
// All data (NVData, Hierarchy seeds, etc...) is cleared or reset.
func (s *Simulator) ManufactureReset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(true)
}

// Write executes the command specified by commandBuffer. The command response
// can be retrieved with a subsequent call to Read().
func (s *Simulator) Write(commandBuffer []byte) (int, error) {
	resp, err := runCommand(commandBuffer)
	if err != nil {
		return 0, err
	}
	return s.buf.Write(resp)
}

// Read gets the response of a command previously issued by calling Write().
func (s *Simulator) Read(responseBuffer []byte) (int, error) {
	return s.buf.Read(responseBuffer)
}

// Close cleans up and stops the simulator, Close() should always be called when
// the Simulator is no longer needed, freeing up other callers to use Get().
func (s *Simulator) Close() error {
	lock.Lock()
	defer lock.Unlock()
	inUse = false
	return s.off()
}

func (s *Simulator) on(manufactureReset bool) error {
	// Setup the simulator to receive commands
	C._plat__Signal_PowerOn()
	C._plat__Signal_Reset()
	C._plat__SetNvAvail()
	C._plat__Signal_PhysicalPresenceOn()
	if manufactureReset {
		if rc := C.TPM_Manufacture(1); rc != C.TPM_RC_SUCCESS {
			return fmt.Errorf("manufacture reset failed: code %x", rc)
		}
	}
	// TPM2_Setup must be the first command the TPM receives
	if err := tpm2.Startup(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("startup: %v", err)
	}
	return nil
}

func (s *Simulator) off() error {
	// TPM2_Shutdown must be the first command the TPM receives. We call
	// Shutdown with StartupClear to simulate a full reboot.
	if err := tpm2.Shutdown(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("shutdown: %v", err)
	}
	C._plat__Signal_PhysicalPresenceOff()
	C._plat__ClearNvAvail()
	C._plat__Signal_PowerOff()
	return nil
}

func runCommand(cmd []byte) ([]byte, error) {
	responseSize := C.uint32_t(C.MAX_RESPONSE_SIZE)
	// _plat__RunCommand takes the response buffer as a uint8_t** instead of as
	// a uint8_t*. As Cgo bans go pointers to go pointers, we must allocate the
	// response buffer with malloc().
	response := C.malloc(C.size_t(responseSize))
	defer C.free(response)
	// Make a copy of the response pointer, so we can be sure _plat__RunCommand
	// doesn't modify the pointer (it _is_ expected to modify the buffer).
	responsePtr := (*C.uint8_t)(response)

	C._plat__RunCommand(C.uint32_t(len(cmd)), (*C.uint8_t)(&cmd[0]),
		&responseSize, &responsePtr)
	// As long as NO_FAIL_TRACE is not defined, debug error information is
	// written to certain global variables on internal failure.
	if C.g_inFailureMode == C.TRUE {
		return nil, errors.New("unknown internal failure")
	}
	if response != unsafe.Pointer(responsePtr) {
		panic("Response pointer shouldn't be modified on success")
	}
	return C.GoBytes(response, C.int(responseSize)), nil
}
<commit_msg>Add flags to prevent warnings<commit_after>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

// Package simulator provides a go interface to the Microsoft TPM2 simulator.
package simulator

// // Directories containing .h files in the simulator source
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/include
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/include/prototypes
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/include
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/include/prototypes
// // Allows simulator.c to import files without repeating the source repo path.
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/Platform/src
// #cgo CFLAGS: -I ms-tpm-20-ref/TPMCmd/tpm/src
// // Store NVDATA in memory, and we don't care about updates to failedTries.
// #cgo CFLAGS: -DVTPM=NO -DSIMULATION=NO -DUSE_DA_USED=NO
// // Flags from ms-tpm-20-ref/TPMCmd/configure.ac
// #cgo CFLAGS: -std=gnu11 -Wall -Wformat-security -fstack-protector-all -fPIC
// // Silence known warnings from the reference code and CGO code.
// #cgo CFLAGS: -Wno-missing-braces -Wno-empty-body -Wno-unused-variable
// // Link against the system OpenSSL
// #cgo CFLAGS: -DHASH_LIB=Ossl -DSYM_LIB=Ossl -DMATH_LIB=Ossl
// #cgo LDFLAGS: -lcrypto
//
// #include <stdlib.h>
// #include "Tpm.h"
//
// void sync_seeds() {
//     NV_SYNC_PERSISTENT(EPSeed);
//     NV_SYNC_PERSISTENT(SPSeed);
//     NV_SYNC_PERSISTENT(PPSeed);
// }
import "C"
import (
	"bytes"
	"errors"
	"fmt"
	"math/rand"
	"sync"
	"unsafe"

	"github.com/google/go-tpm/tpm2"
)

// Simulator represents a go-tpm compatible interface to the IBM TPM2 simulator.
// Similar to the file-based (for linux) or syscall-based (for Windows) TPM
// handles, no synchronization is provided; the same simulator handle should not
// be used from multiple threads.
type Simulator struct {
	buf bytes.Buffer
}

// ErrSimulatorInUse indicates another open Simulator already exists.
var ErrSimulatorInUse = errors.New("simulator is being used by another caller")

// The simulator is a global resource, so we use the variables below to make
// sure we only ever have one open reference to the Simulator at a time.
var (
	lock  sync.Mutex
	inUse bool
)

// Get the pointer to an initialized, powered on, and started simulator. As only
// one simulator may be running at a time, a second call to Get() will return
// ErrSimulatorInUse until the first Simulator is Closed.
func Get() (*Simulator, error) {
	lock.Lock()
	defer lock.Unlock()
	if inUse {
		return nil, ErrSimulatorInUse
	}

	simulator := &Simulator{}
	if err := simulator.on(true); err != nil {
		return nil, err
	}
	inUse = true
	return simulator, nil
}

// GetWithFixedSeedInsecure behaves like Get() expect that all of the internal
// hierarchy seeds are derived from the input seed. Note that this function
// compromises the security of the keys/seeds and should only be used for tests.
func GetWithFixedSeedInsecure(seed int64) (*Simulator, error) {
	r := rand.New(rand.NewSource(seed))
	s, err := Get()
	if err != nil {
		return nil, err
	}

	// The first two bytes of the seed encode the size (so we don't overwrite)
	r.Read(C.gp.EPSeed[2:])
	r.Read(C.gp.SPSeed[2:])
	r.Read(C.gp.PPSeed[2:])
	return s, nil
}

// Reset the TPM as if the host computer had rebooted.
func (s *Simulator) Reset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(false)
}

// ManufactureReset behaves like Reset() except that the TPM is complete wiped.
// All data (NVData, Hierarchy seeds, etc...) is cleared or reset.
func (s *Simulator) ManufactureReset() error {
	if err := s.off(); err != nil {
		return err
	}
	return s.on(true)
}

// Write executes the command specified by commandBuffer. The command response
// can be retrieved with a subsequent call to Read().
func (s *Simulator) Write(commandBuffer []byte) (int, error) {
	resp, err := runCommand(commandBuffer)
	if err != nil {
		return 0, err
	}
	return s.buf.Write(resp)
}

// Read gets the response of a command previously issued by calling Write().
func (s *Simulator) Read(responseBuffer []byte) (int, error) {
	return s.buf.Read(responseBuffer)
}

// Close cleans up and stops the simulator, Close() should always be called when
// the Simulator is no longer needed, freeing up other callers to use Get().
func (s *Simulator) Close() error {
	lock.Lock()
	defer lock.Unlock()
	inUse = false
	return s.off()
}

func (s *Simulator) on(manufactureReset bool) error {
	// Setup the simulator to receive commands
	C._plat__Signal_PowerOn()
	C._plat__Signal_Reset()
	C._plat__SetNvAvail()
	C._plat__Signal_PhysicalPresenceOn()
	if manufactureReset {
		if rc := C.TPM_Manufacture(1); rc != C.TPM_RC_SUCCESS {
			return fmt.Errorf("manufacture reset failed: code %x", rc)
		}
	}
	// TPM2_Setup must be the first command the TPM receives
	if err := tpm2.Startup(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("startup: %v", err)
	}
	return nil
}

func (s *Simulator) off() error {
	// TPM2_Shutdown must be the first command the TPM receives. We call
	// Shutdown with StartupClear to simulate a full reboot.
	if err := tpm2.Shutdown(s, tpm2.StartupClear); err != nil {
		return fmt.Errorf("shutdown: %v", err)
	}
	C._plat__Signal_PhysicalPresenceOff()
	C._plat__ClearNvAvail()
	C._plat__Signal_PowerOff()
	return nil
}

func runCommand(cmd []byte) ([]byte, error) {
	responseSize := C.uint32_t(C.MAX_RESPONSE_SIZE)
	// _plat__RunCommand takes the response buffer as a uint8_t** instead of as
	// a uint8_t*. As Cgo bans go pointers to go pointers, we must allocate the
	// response buffer with malloc().
	response := C.malloc(C.size_t(responseSize))
	defer C.free(response)
	// Make a copy of the response pointer, so we can be sure _plat__RunCommand
	// doesn't modify the pointer (it _is_ expected to modify the buffer).
	responsePtr := (*C.uint8_t)(response)

	C._plat__RunCommand(C.uint32_t(len(cmd)), (*C.uint8_t)(&cmd[0]),
		&responseSize, &responsePtr)
	// As long as NO_FAIL_TRACE is not defined, debug error information is
	// written to certain global variables on internal failure.
	if C.g_inFailureMode == C.TRUE {
		return nil, errors.New("unknown internal failure")
	}
	if response != unsafe.Pointer(responsePtr) {
		panic("Response pointer shouldn't be modified on success")
	}
	return C.GoBytes(response, C.int(responseSize)), nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func wrapper() func() int {
	x := 0
	return func() int {
		x++
		return x
	}
}

func main() {
	increment := wrapper()

	fmt.Println(increment())
	fmt.Println(increment())

	increment = wrapper()

	fmt.Println(increment())
	fmt.Println(increment())
}
<commit_msg>Udemy - Scopes<commit_after>package main

import "fmt"

func wrapper(x int) func() int {
	return func() int {
		x++
		return x
	}
}

func main() {
	increment := wrapper(0)

	fmt.Println(increment())
	fmt.Println(increment())

	increment = wrapper(1)

	fmt.Println(increment())
	fmt.Println(increment())
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kvenc

import (
	"bytes"
	"fmt"
	"strings"
	"sync/atomic"

	"github.com/juju/errors"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/meta/autoid"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/plan"
	"github.com/pingcap/tidb/session"
	"github.com/pingcap/tidb/store/mockstore"
	"github.com/pingcap/tidb/tablecodec"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
)

var _ KvEncoder = &kvEncoder{}
var mockConnID uint64

// KvPair is a key-value pair.
type KvPair struct {
	// Key is the key of the pair.
	Key []byte
	// Val is the value of the pair. if the op is delete, the len(Val) == 0
	Val []byte
}

// KvEncoder is an encoder that transfer sql to key-value pairs.
type KvEncoder interface {
	// Encode transfers sql to kv pairs.
	// Before use Encode() method, please make sure you already created schame by calling ExecDDLSQL() method.
	// NOTE: now we just support transfers insert statement to kv pairs.
	// (if we wanna support other statement, we need to add a kv.Storage parameter,
	// and pass tikv store in.)
	// return encoded kvs array that generate by sql, and affectRows count.
	Encode(sql string, tableID int64) (kvPairs []KvPair, affectedRows uint64, err error)

	// PrepareStmt prepare query statement, and return statement id.
	// Pass stmtID into EncodePrepareStmt to execute a prepare statement.
	PrepareStmt(query string) (stmtID uint32, err error)

	// EncodePrepareStmt transfer prepare query to kv pairs.
	// stmtID is generated by PrepareStmt.
	EncodePrepareStmt(tableID int64, stmtID uint32, param ...interface{}) (kvPairs []KvPair, affectedRows uint64, err error)

	// ExecDDLSQL executes ddl sql, you must use it to create schema infos.
	ExecDDLSQL(sql string) error

	// EncodeMetaAutoID encode the table meta info, autoID to coresponding key-value pair.
	EncodeMetaAutoID(dbID, tableID, autoID int64) (KvPair, error)

	// SetSystemVariable set system variable name = value.
	SetSystemVariable(name string, value string) error

	// GetSystemVariable get the system variable value of name.
	GetSystemVariable(name string) (string, bool)

	// Close cleanup the kvEncoder.
	Close() error
}

type kvEncoder struct {
	store kv.Storage
	dom   *domain.Domain
	se    session.Session
}

// New new a KvEncoder
func New(dbName string, idAlloc autoid.Allocator) (KvEncoder, error) {
	kvEnc := &kvEncoder{}
	err := kvEnc.initial(dbName, idAlloc)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return kvEnc, nil
}

func (e *kvEncoder) Close() error {
	e.dom.Close()
	if err := e.store.Close(); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (e *kvEncoder) Encode(sql string, tableID int64) (kvPairs []KvPair, affectedRows uint64, err error) {
	e.se.GetSessionVars().SetStatusFlag(mysql.ServerStatusInTrans, true)
	defer func() {
		err1 := e.se.RollbackTxn(context.Background())
		if err1 != nil {
			log.Error(errors.ErrorStack(err1))
		}
	}()

	_, err = e.se.Execute(context.Background(), sql)
	if err != nil {
		return nil, 0, errors.Trace(err)
	}

	return e.getKvPairsInMemBuffer(tableID)
}

func (e *kvEncoder) getKvPairsInMemBuffer(tableID int64) (kvPairs []KvPair, affectedRows uint64, err error) {
	txnMemBuffer := e.se.Txn().GetMemBuffer()
	kvPairs = make([]KvPair, 0, txnMemBuffer.Len())
	err = kv.WalkMemBuffer(txnMemBuffer, func(k kv.Key, v []byte) error {
		if bytes.HasPrefix(k, tablecodec.TablePrefix()) {
			k = tablecodec.ReplaceRecordKeyTableID(k, tableID)
		}
		kvPairs = append(kvPairs, KvPair{Key: k, Val: v})
		return nil
	})

	if err != nil {
		return nil, 0, errors.Trace(err)
	}
	return kvPairs, e.se.GetSessionVars().StmtCtx.AffectedRows(), nil
}

func (e *kvEncoder) PrepareStmt(query string) (stmtID uint32, err error) {
	stmtID, _, _, err = e.se.PrepareStmt(query)
	return
}

func (e *kvEncoder) EncodePrepareStmt(tableID int64, stmtID uint32, param ...interface{}) (kvPairs []KvPair, affectedRows uint64, err error) {
	e.se.GetSessionVars().SetStatusFlag(mysql.ServerStatusInTrans, true)
	defer func() {
		err1 := e.se.RollbackTxn(context.Background())
		if err1 != nil {
			log.Error(errors.ErrorStack(err1))
		}
	}()

	_, err = e.se.ExecutePreparedStmt(context.Background(), stmtID, param...)
	if err != nil {
		return nil, 0, errors.Trace(err)
	}

	return e.getKvPairsInMemBuffer(tableID)
}

func (e *kvEncoder) EncodeMetaAutoID(dbID, tableID, autoID int64) (KvPair, error) {
	mockTxn := kv.NewMockTxn()
	m := meta.NewMeta(mockTxn)
	k, v := m.GenAutoTableIDIDKeyValue(dbID, tableID, autoID)
	return KvPair{Key: k, Val: v}, nil
}

func (e *kvEncoder) ExecDDLSQL(sql string) error {
	_, err := e.se.Execute(context.Background(), sql)
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (e *kvEncoder) SetSystemVariable(name string, value string) error {
	name = strings.ToLower(name)
	if e.se != nil {
		return e.se.GetSessionVars().SetSystemVar(name, value)
	}
	return errors.Errorf("e.se is nil, please new KvEncoder by kvencoder.New().")
}

func (e *kvEncoder) GetSystemVariable(name string) (string, bool) {
	name = strings.ToLower(name)
	if e.se == nil {
		return "", false
	}

	return e.se.GetSessionVars().GetSystemVar(name)
}

func newMockTikvWithBootstrap() (kv.Storage, *domain.Domain, error) {
	store, err := mockstore.NewMockTikvStore()
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	session.SetSchemaLease(0)
	dom, err := session.BootstrapSession(store)
	return store, dom, errors.Trace(err)
}

func (e *kvEncoder) initial(dbName string, idAlloc autoid.Allocator) (err error) {
	var (
		store kv.Storage
		dom   *domain.Domain
		se    session.Session
	)
	defer func() {
		if err == nil {
			return
		}
		if store != nil {
			if err1 := store.Close(); err1 != nil {
				log.Error(errors.ErrorStack(err1))
			}
		}
		if dom != nil {
			dom.Close()
		}
		if se != nil {
			se.Close()
		}
	}()

	plan.PreparedPlanCacheEnabled = true
	plan.PreparedPlanCacheCapacity = 10
	// disable stats update.
	session.SetStatsLease(0)
	store, dom, err = newMockTikvWithBootstrap()
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se, err = session.CreateSession(store)
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se.SetConnectionID(atomic.AddUint64(&mockConnID, 1))
	_, err = se.Execute(context.Background(), fmt.Sprintf("create database if not exists %s", dbName))
	if err != nil {
		err = errors.Trace(err)
		return
	}
	_, err = se.Execute(context.Background(), fmt.Sprintf("use %s", dbName))
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se.GetSessionVars().IDAllocator = idAlloc
	se.GetSessionVars().ImportingData = true
	se.GetSessionVars().SkipUTF8Check = true
	e.se = se
	e.store = store
	e.dom = dom
	return nil
}
<commit_msg>kvencoder: fix data race on plan.PreparedPlanCacheCapacity (#6159)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kvenc

import (
	"bytes"
	"fmt"
	"strings"
	"sync/atomic"

	"github.com/juju/errors"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/meta"
	"github.com/pingcap/tidb/meta/autoid"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/session"
	"github.com/pingcap/tidb/store/mockstore"
	"github.com/pingcap/tidb/tablecodec"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
)

var _ KvEncoder = &kvEncoder{}
var mockConnID uint64

// KvPair is a key-value pair.
type KvPair struct {
	// Key is the key of the pair.
	Key []byte
	// Val is the value of the pair. if the op is delete, the len(Val) == 0
	Val []byte
}

// KvEncoder is an encoder that transfer sql to key-value pairs.
type KvEncoder interface {
	// Encode transfers sql to kv pairs.
	// Before use Encode() method, please make sure you already created schame by calling ExecDDLSQL() method.
	// NOTE: now we just support transfers insert statement to kv pairs.
	// (if we wanna support other statement, we need to add a kv.Storage parameter,
	// and pass tikv store in.)
	// return encoded kvs array that generate by sql, and affectRows count.
	Encode(sql string, tableID int64) (kvPairs []KvPair, affectedRows uint64, err error)

	// PrepareStmt prepare query statement, and return statement id.
	// Pass stmtID into EncodePrepareStmt to execute a prepare statement.
	PrepareStmt(query string) (stmtID uint32, err error)

	// EncodePrepareStmt transfer prepare query to kv pairs.
	// stmtID is generated by PrepareStmt.
	EncodePrepareStmt(tableID int64, stmtID uint32, param ...interface{}) (kvPairs []KvPair, affectedRows uint64, err error)

	// ExecDDLSQL executes ddl sql, you must use it to create schema infos.
	ExecDDLSQL(sql string) error

	// EncodeMetaAutoID encode the table meta info, autoID to coresponding key-value pair.
	EncodeMetaAutoID(dbID, tableID, autoID int64) (KvPair, error)

	// SetSystemVariable set system variable name = value.
	SetSystemVariable(name string, value string) error

	// GetSystemVariable get the system variable value of name.
	GetSystemVariable(name string) (string, bool)

	// Close cleanup the kvEncoder.
	Close() error
}

type kvEncoder struct {
	store kv.Storage
	dom   *domain.Domain
	se    session.Session
}

// New new a KvEncoder
func New(dbName string, idAlloc autoid.Allocator) (KvEncoder, error) {
	kvEnc := &kvEncoder{}
	err := kvEnc.initial(dbName, idAlloc)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return kvEnc, nil
}

func (e *kvEncoder) Close() error {
	e.dom.Close()
	if err := e.store.Close(); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (e *kvEncoder) Encode(sql string, tableID int64) (kvPairs []KvPair, affectedRows uint64, err error) {
	e.se.GetSessionVars().SetStatusFlag(mysql.ServerStatusInTrans, true)
	defer func() {
		err1 := e.se.RollbackTxn(context.Background())
		if err1 != nil {
			log.Error(errors.ErrorStack(err1))
		}
	}()

	_, err = e.se.Execute(context.Background(), sql)
	if err != nil {
		return nil, 0, errors.Trace(err)
	}

	return e.getKvPairsInMemBuffer(tableID)
}

func (e *kvEncoder) getKvPairsInMemBuffer(tableID int64) (kvPairs []KvPair, affectedRows uint64, err error) {
	txnMemBuffer := e.se.Txn().GetMemBuffer()
	kvPairs = make([]KvPair, 0, txnMemBuffer.Len())
	err = kv.WalkMemBuffer(txnMemBuffer, func(k kv.Key, v []byte) error {
		if bytes.HasPrefix(k, tablecodec.TablePrefix()) {
			k = tablecodec.ReplaceRecordKeyTableID(k, tableID)
		}
		kvPairs = append(kvPairs, KvPair{Key: k, Val: v})
		return nil
	})

	if err != nil {
		return nil, 0, errors.Trace(err)
	}
	return kvPairs, e.se.GetSessionVars().StmtCtx.AffectedRows(), nil
}

func (e *kvEncoder) PrepareStmt(query string) (stmtID uint32, err error) {
	stmtID, _, _, err = e.se.PrepareStmt(query)
	return
}

func (e *kvEncoder) EncodePrepareStmt(tableID int64, stmtID uint32, param ...interface{}) (kvPairs []KvPair, affectedRows uint64, err error) {
	e.se.GetSessionVars().SetStatusFlag(mysql.ServerStatusInTrans, true)
	defer func() {
		err1 := e.se.RollbackTxn(context.Background())
		if err1 != nil {
			log.Error(errors.ErrorStack(err1))
		}
	}()

	_, err = e.se.ExecutePreparedStmt(context.Background(), stmtID, param...)
	if err != nil {
		return nil, 0, errors.Trace(err)
	}

	return e.getKvPairsInMemBuffer(tableID)
}

func (e *kvEncoder) EncodeMetaAutoID(dbID, tableID, autoID int64) (KvPair, error) {
	mockTxn := kv.NewMockTxn()
	m := meta.NewMeta(mockTxn)
	k, v := m.GenAutoTableIDIDKeyValue(dbID, tableID, autoID)
	return KvPair{Key: k, Val: v}, nil
}

func (e *kvEncoder) ExecDDLSQL(sql string) error {
	_, err := e.se.Execute(context.Background(), sql)
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (e *kvEncoder) SetSystemVariable(name string, value string) error {
	name = strings.ToLower(name)
	if e.se != nil {
		return e.se.GetSessionVars().SetSystemVar(name, value)
	}
	return errors.Errorf("e.se is nil, please new KvEncoder by kvencoder.New().")
}

func (e *kvEncoder) GetSystemVariable(name string) (string, bool) {
	name = strings.ToLower(name)
	if e.se == nil {
		return "", false
	}

	return e.se.GetSessionVars().GetSystemVar(name)
}

func newMockTikvWithBootstrap() (kv.Storage, *domain.Domain, error) {
	store, err := mockstore.NewMockTikvStore()
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	session.SetSchemaLease(0)
	dom, err := session.BootstrapSession(store)
	return store, dom, errors.Trace(err)
}

func (e *kvEncoder) initial(dbName string, idAlloc autoid.Allocator) (err error) {
	var (
		store kv.Storage
		dom   *domain.Domain
		se    session.Session
	)
	defer func() {
		if err == nil {
			return
		}
		if store != nil {
			if err1 := store.Close(); err1 != nil {
				log.Error(errors.ErrorStack(err1))
			}
		}
		if dom != nil {
			dom.Close()
		}
		if se != nil {
			se.Close()
		}
	}()

	// disable stats update.
	session.SetStatsLease(0)
	store, dom, err = newMockTikvWithBootstrap()
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se, err = session.CreateSession(store)
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se.SetConnectionID(atomic.AddUint64(&mockConnID, 1))
	_, err = se.Execute(context.Background(), fmt.Sprintf("create database if not exists %s", dbName))
	if err != nil {
		err = errors.Trace(err)
		return
	}
	_, err = se.Execute(context.Background(), fmt.Sprintf("use %s", dbName))
	if err != nil {
		err = errors.Trace(err)
		return
	}

	se.GetSessionVars().IDAllocator = idAlloc
	se.GetSessionVars().ImportingData = true
	se.GetSessionVars().SkipUTF8Check = true
	e.se = se
	e.store = store
	e.dom = dom
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
)

type ServeConfig struct {
	Port string
	Dir  string
	Path string
}

func main() {
	args := os.Args[1:]
	config := getServeConfig(args)
	serve(config)
}

func getServeConfig(args []string) ServeConfig {

	dir, _ := os.Getwd()
	config := ServeConfig{"8000", dir, "/"}

	return config
}

func serve(config ServeConfig) {

	http.Handle(config.Path, http.FileServer(http.Dir(config.Dir)))

	fmt.Println("Serve Config \n Directory : " + path.Base(config.Dir) + " \n On        : http://localhost:" + config.Port + config.Path + "\n")
	log.Println("Starting server on port: " + config.Port)

	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal("Error ListenAndServe", err)
	}
}
<commit_msg>add option to use custom port<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
)

type ServeConfig struct {
	Port string
	Dir  string
	Path string
}

func getServeConfig(args []string) ServeConfig {

	dir, _ := os.Getwd()
	config := ServeConfig{"8000", dir, "/"}

	if(len(args)>0){
		for _,element := range args {
			_,err := strconv.Atoi(element)
			if(err == nil){
				config.Port = element
			}
		}
	}

	return config
}

func serve(config ServeConfig) {

	http.Handle(config.Path, http.FileServer(http.Dir(config.Dir)))

	fmt.Println("Serve Config \n Directory : " + path.Base(config.Dir) + " \n Path      : http://localhost:" + config.Port + config.Path + "\n")
	log.Println("Starting server on port: " + config.Port)

	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		log.Fatal("Error ListenAndServe", err)
	}
}

func main() {
	args := os.Args[1:]
	config := getServeConfig(args)
	serve(config)
}<|endoftext|>