<commit_before>package attspeech

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"reflect"
	"runtime"
	"strings"
	"unicode"
)

const (
	// APIBase is the base URL for the ATT Speech API
	APIBase = "https://api.att.com"
	// STTResource is the speech to text resource
	STTResource = "/speech/v3/speechToText"
	// STTCResource is the speech to text custom resource
	STTCResource = "/speech/v3/speechToTextCustom"
	// TTSResource is the text to speech resource
	TTSResource = "/speech/v3/textToSpeech"
	// OauthResource is the oauth resource
	OauthResource = "/oauth/access_token"
	// UserAgent is the user agent use for the HTTP client
	UserAgent = "GoATTSpeechLib"
	// Version is the version of the ATT Speech API
	Version = "0.1"
)

/*
New creates a new AttSpeechClient

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
*/
func New(id string, secret string, apiBase string) *Client {
	client := &Client{
		STTResource:   STTResource,
		STTCResource:  STTCResource,
		TTSResource:   TTSResource,
		OauthResource: OauthResource,
		ID:            id,
		Secret:        secret,
		Scope:         [3]string{"SPEECH", "STTC", "TTS"},
	}
	if apiBase == "" {
		client.APIBase = APIBase
	} else {
		client.APIBase = apiBase
	}
	return client
}

/*
SetAuthTokens sets the provided authorization tokens for the client

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
*/
func (client *Client) SetAuthTokens() error {
	data := "grant_type=client_credentials&"
	data += "client_id=" + client.ID + "&"
	data += "client_secret=" + client.Secret + "&"
	data += "scope="

	m := make(map[string]*Token)
	for _, scope := range client.Scope {
		req, _ := http.NewRequest("POST", client.APIBase+OauthResource+"?"+data+scope, nil)
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}
		body, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		token := &Token{}
		err = json.Unmarshal(body, token)
		if err != nil {
			return err
		}
		m[scope] = token
	}
	client.Tokens = m
	return nil
}

/*
SpeechToText converts an audio file to text

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(STTResource)
	apiRequest.Data = data // where data is audio content as *bytes.Buffer
	apiRequest.ContentType = "audio/wav"
	result, apiError, err := client.SpeechToText(apiRequest)

More details available here:

	http://developer.att.com/apis/speech/docs#resources-speech-to-text
*/
func (client *Client) SpeechToText(apiRequest *APIRequest) (*Recognition, error) {
	if apiRequest.ContentType == "" {
		return nil, errors.New("a content type must be provided")
	}
	if apiRequest.Data == nil {
		return nil, errors.New("data to convert to text must be provided")
	}

	body, statusCode, err := client.post(client.STTResource, apiRequest.Data, apiRequest)
	if err != nil {
		return nil, err
	}
	if statusCode == 200 {
		recognition := &Recognition{}
		err := json.Unmarshal(body, recognition)
		if err != nil {
			return nil, err
		}
		return recognition, nil
	}
	apiError := &APIError{}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
SpeechToTextCustom converts an audio file to text

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(STTResource)
	apiRequest.Data = data // where data is audio content as *bytes.Buffer
	apiRequest.ContentType = "audio/wav"
	apiRequest.Filename = "test.wav"
	result, apiError, err := client.SpeechToTextCustom(apiRequest, "<some srgs XML>", "<some pls XML>")

More details available here:

	http://developer.att.com/apis/speech/docs#resources-speech-to-text-custom

*/
func (client *Client) SpeechToTextCustom(apiRequest *APIRequest, grammar string, dictionary string) (*Recognition, error) {
	if grammar == "" {
		return nil, errors.New("a grammar must be provided")
	}
	if apiRequest.Data == nil {
		return nil, errors.New("data must be provided")
	}
	if apiRequest.Filename == "" {
		return nil, errors.New("filename must be provided")
	}
	if apiRequest.ContentType == "" {
		return nil, errors.New("content type must be provided")
	}

	apiRequest.Data, apiRequest.ContentType = buildForm(apiRequest, grammar, dictionary)
	body, statusCode, err := client.post(client.STTCResource, apiRequest.Data, apiRequest)
	if err != nil {
		return nil, err
	}
	apiError := &APIError{}
	if statusCode == 200 {
		recognition := &Recognition{}
		err := json.Unmarshal(body, recognition)
		if err != nil {
			fmt.Println(err)
			return nil, apiError.generateErr()
		}
		return recognition, nil
	}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
TextToSpeech converts text to a speech file

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()

	apiRequest := client.NewAPIRequest(TTSResource)
	apiRequest.Accept = "audio/x-wav",
	apiRequest.VoiceName = "crystal",
	apiRequest.Text = "I want to be an airborne ranger, I want to live the life of danger.",
	data, err := client.TextToSpeech(apiRequest)

More details available here:

	http://developer.att.com/apis/speech/docs#resources-text-to-speech
*/
func (client *Client) TextToSpeech(apiRequest *APIRequest) ([]byte, error) {
	if apiRequest.Text == "" {
		return nil, errors.New("text to convert to speech must be provided")
	}

	body, statusCode, err := client.post(client.TTSResource, bytes.NewBuffer([]byte(apiRequest.Text)), apiRequest)
	if err != nil {
		return nil, err
	}
	if statusCode == 200 {
		return body, nil
	}
	apiError := &APIError{}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
NewAPIRequest sets the common headers for TTS and STT

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(TTSResource)

Note, when setting apiRequest.XArg, always append with '+=', unless you specifically
intend to overwrite the defaults for ClientApp, ClientVersion, DeviceType and DeviceOs
*/
func (client *Client) NewAPIRequest(resource string) *APIRequest {
	apiRequest := &APIRequest{}
	apiRequest.Accept = "application/json"
	apiRequest.UserAgent = "Golang net/http"
	apiRequest.XArg = "ClientApp=GoLibForATTSpeech,"
	apiRequest.XArg += "ClientVersion=" + Version + ","
	apiRequest.XArg += "DeviceType=" + runtime.GOARCH + ","
	apiRequest.XArg += "DeviceOs=" + runtime.GOOS

	switch resource {
	case client.STTResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["SPEECH"].AccessToken
		apiRequest.TransferEncoding = "chunked"
	case client.STTCResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["STTC"].AccessToken
	case client.TTSResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["TTS"].AccessToken
		apiRequest.ContentType = "text/plain"
	case client.OauthResource:
		apiRequest.ContentType = "application/x-www-form-urlencoded"
	}
	return apiRequest
}

// post to the AT&T Speech API
func (client *Client) post(resource string, body *bytes.Buffer, apiRequest *APIRequest) ([]byte, int, error) {
	req, err := http.NewRequest("POST", client.APIBase+resource, body)
	if err != nil {
		return nil, 0, err
	}
	apiRequest.setHeaders(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, 0, err
	}
	respBody, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	return respBody, resp.StatusCode, nil
}

// generateErr takes the APIError and turns it into a Go error
func (apiError *APIError) generateErr() error {
	msg := apiError.RequestError.ServiceException.MessageID + " - "
	msg += apiError.RequestError.ServiceException.Text + " - "
	msg += apiError.RequestError.ServiceException.Variables
	if msg == " -  - " {
		msg = apiError.RequestError.PolicyException.MessageID + " - "
		msg += apiError.RequestError.PolicyException.Text + " - "
		msg += apiError.RequestError.PolicyException.Variables

	}
	if msg == " -  - " {
		/*
			#FIXME
			http://developerboards.att.lithium.com/t5/API-Platform/Speech-API-STTC-Error-Returns-Invalid-JSON/td-p/38929
		*/
		msg = "could not parse JSON error from the AT&T Speech API"
	}
	return errors.New(msg)
}

// setHeaders returns the APIRequest as a map
func (apiRequest *APIRequest) setHeaders(req *http.Request) {
	headers := make(map[string]string)
	xarg := ""

	s := reflect.ValueOf(apiRequest).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		name := typeOfT.Field(i).Name
		if name != "Data" && name != "Text" && name != "Filename" {
			if f.Interface().(string) != "" {
				if name == "VoiceName" || name == "Volume" || name == "Tempo" {
					xarg += "," + name + "=" + f.Interface().(string)
				} else {
					headers[toDash(name)] = f.Interface().(string)
				}
			}
		}
	}
	headers["X-Arg"] += xarg
	for key, value := range headers {
		req.Header.Add(key, value)
	}
}

/*
toDash converts an uppercase string into a string
where uppercase letters are sperated by a '-'
*/
func toDash(value string) string {
	var words []string
	l := 0
	for s := value; s != ""; s = s[l:] {
		l = strings.IndexFunc(s[1:], unicode.IsUpper) + 1
		if l <= 0 {
			l = len(s)
		}
		words = append(words, s[:l])
	}
	dashedWord := ""
	numWords := len(words)
	for i := 0; i < numWords; i++ {
		if i == 0 && numWords > 1 {
			dashedWord = words[0] + "-"
		} else {
			dashedWord += words[i]
		}
	}
	return dashedWord
}

//buildForm builds a multipart form to send the file with
func buildForm(apiRequest *APIRequest, grammar string, dictionary string) (*bytes.Buffer, string) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	defer writer.Close()

	if dictionary != "" {
		// Add the dictionary field
		contentDisposition := "form-data; name=\"x-dictionary\"; filename=\"speech_alpha.pls\""
		apiRequest.addField(writer, dictionary, contentDisposition, "application/pls+xml")
	}

	// Add the grammar field
	contentDisposition := "form-data; name=\"x-grammar\""
	apiRequest.addField(writer, grammar, contentDisposition, "application/srgs+xml")

	// Add the file field
	contentDisposition = "form-data; name=\"x-voice\"; filename=\"" + apiRequest.Filename + "\""
	apiRequest.addField(writer, grammar, contentDisposition, apiRequest.ContentType)

	contentType := writer.FormDataContentType()
	contentType = strings.Replace(contentType, "form-data", "x-srgs-audio", 1)
	return body, contentType
}

// addField adds a field to a multipart form
func (apiRequest *APIRequest) addField(writer *multipart.Writer, body string, contentDisposition string, contentType string) {
	header := make(map[string][]string)
	header["Content-Disposition"] = []string{contentDisposition}
	header["Content-Type"] = []string{contentType}
	part, _ := writer.CreatePart(header)
	if contentType == "application/pls+xml" || contentType == "application/srgs+xml" {
		part.Write([]byte(body + "\n"))
	} else {
		io.Copy(part, apiRequest.Data)
	}
}
<commit_msg>Got rid of fmt<commit_after>package attspeech

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"reflect"
	"runtime"
	"strings"
	"unicode"
)

const (
	// APIBase is the base URL for the ATT Speech API
	APIBase = "https://api.att.com"
	// STTResource is the speech to text resource
	STTResource = "/speech/v3/speechToText"
	// STTCResource is the speech to text custom resource
	STTCResource = "/speech/v3/speechToTextCustom"
	// TTSResource is the text to speech resource
	TTSResource = "/speech/v3/textToSpeech"
	// OauthResource is the oauth resource
	OauthResource = "/oauth/access_token"
	// UserAgent is the user agent use for the HTTP client
	UserAgent = "GoATTSpeechLib"
	// Version is the version of the ATT Speech API
	Version = "0.1"
)

/*
New creates a new AttSpeechClient

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
*/
func New(id string, secret string, apiBase string) *Client {
	client := &Client{
		STTResource:   STTResource,
		STTCResource:  STTCResource,
		TTSResource:   TTSResource,
		OauthResource: OauthResource,
		ID:            id,
		Secret:        secret,
		Scope:         [3]string{"SPEECH", "STTC", "TTS"},
	}
	if apiBase == "" {
		client.APIBase = APIBase
	} else {
		client.APIBase = apiBase
	}
	return client
}

/*
SetAuthTokens sets the provided authorization tokens for the client

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
*/
func (client *Client) SetAuthTokens() error {
	data := "grant_type=client_credentials&"
	data += "client_id=" + client.ID + "&"
	data += "client_secret=" + client.Secret + "&"
	data += "scope="

	m := make(map[string]*Token)
	for _, scope := range client.Scope {
		req, _ := http.NewRequest("POST", client.APIBase+OauthResource+"?"+data+scope, nil)
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return err
		}
		body, _ := ioutil.ReadAll(res.Body)
		res.Body.Close()
		token := &Token{}
		err = json.Unmarshal(body, token)
		if err != nil {
			return err
		}
		m[scope] = token
	}
	client.Tokens = m
	return nil
}

/*
SpeechToText converts an audio file to text

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(STTResource)
	apiRequest.Data = data // where data is audio content as *bytes.Buffer
	apiRequest.ContentType = "audio/wav"
	result, apiError, err := client.SpeechToText(apiRequest)

More details available here:

	http://developer.att.com/apis/speech/docs#resources-speech-to-text
*/
func (client *Client) SpeechToText(apiRequest *APIRequest) (*Recognition, error) {
	if apiRequest.ContentType == "" {
		return nil, errors.New("a content type must be provided")
	}
	if apiRequest.Data == nil {
		return nil, errors.New("data to convert to text must be provided")
	}

	body, statusCode, err := client.post(client.STTResource, apiRequest.Data, apiRequest)
	if err != nil {
		return nil, err
	}
	if statusCode == 200 {
		recognition := &Recognition{}
		err := json.Unmarshal(body, recognition)
		if err != nil {
			return nil, err
		}
		return recognition, nil
	}
	apiError := &APIError{}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
SpeechToTextCustom converts an audio file to text

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(STTResource)
	apiRequest.Data = data // where data is audio content as *bytes.Buffer
	apiRequest.ContentType = "audio/wav"
	apiRequest.Filename = "test.wav"
	result, apiError, err := client.SpeechToTextCustom(apiRequest, "<some srgs XML>", "<some pls XML>")

More details available here:

	http://developer.att.com/apis/speech/docs#resources-speech-to-text-custom

*/
func (client *Client) SpeechToTextCustom(apiRequest *APIRequest, grammar string, dictionary string) (*Recognition, error) {
	if grammar == "" {
		return nil, errors.New("a grammar must be provided")
	}
	if apiRequest.Data == nil {
		return nil, errors.New("data must be provided")
	}
	if apiRequest.Filename == "" {
		return nil, errors.New("filename must be provided")
	}
	if apiRequest.ContentType == "" {
		return nil, errors.New("content type must be provided")
	}

	apiRequest.Data, apiRequest.ContentType = buildForm(apiRequest, grammar, dictionary)
	body, statusCode, err := client.post(client.STTCResource, apiRequest.Data, apiRequest)
	if err != nil {
		return nil, err
	}
	apiError := &APIError{}
	if statusCode == 200 {
		recognition := &Recognition{}
		err := json.Unmarshal(body, recognition)
		if err != nil {
			return nil, apiError.generateErr()
		}
		return recognition, nil
	}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
TextToSpeech converts text to a speech file

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()

	apiRequest := client.NewAPIRequest(TTSResource)
	apiRequest.Accept = "audio/x-wav",
	apiRequest.VoiceName = "crystal",
	apiRequest.Text = "I want to be an airborne ranger, I want to live the life of danger.",
	data, err := client.TextToSpeech(apiRequest)

More details available here:

	http://developer.att.com/apis/speech/docs#resources-text-to-speech
*/
func (client *Client) TextToSpeech(apiRequest *APIRequest) ([]byte, error) {
	if apiRequest.Text == "" {
		return nil, errors.New("text to convert to speech must be provided")
	}

	body, statusCode, err := client.post(client.TTSResource, bytes.NewBuffer([]byte(apiRequest.Text)), apiRequest)
	if err != nil {
		return nil, err
	}
	if statusCode == 200 {
		return body, nil
	}
	apiError := &APIError{}
	err = json.Unmarshal(body, apiError)
	if err != nil {
		return nil, apiError.generateErr()
	}
	return nil, apiError.generateErr()
}

/*
NewAPIRequest sets the common headers for TTS and STT

	client := attspeech.New("<id>", "<secret>", "")
	client.SetAuthTokens()
	apiRequest := client.NewAPIRequest(TTSResource)

Note, when setting apiRequest.XArg, always append with '+=', unless you specifically
intend to overwrite the defaults for ClientApp, ClientVersion, DeviceType and DeviceOs
*/
func (client *Client) NewAPIRequest(resource string) *APIRequest {
	apiRequest := &APIRequest{}
	apiRequest.Accept = "application/json"
	apiRequest.UserAgent = "Golang net/http"
	apiRequest.XArg = "ClientApp=GoLibForATTSpeech,"
	apiRequest.XArg += "ClientVersion=" + Version + ","
	apiRequest.XArg += "DeviceType=" + runtime.GOARCH + ","
	apiRequest.XArg += "DeviceOs=" + runtime.GOOS

	switch resource {
	case client.STTResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["SPEECH"].AccessToken
		apiRequest.TransferEncoding = "chunked"
	case client.STTCResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["STTC"].AccessToken
	case client.TTSResource:
		apiRequest.Authorization = "Bearer " + client.Tokens["TTS"].AccessToken
		apiRequest.ContentType = "text/plain"
	case client.OauthResource:
		apiRequest.ContentType = "application/x-www-form-urlencoded"
	}
	return apiRequest
}

// post to the AT&T Speech API
func (client *Client) post(resource string, body *bytes.Buffer, apiRequest *APIRequest) ([]byte, int, error) {
	req, err := http.NewRequest("POST", client.APIBase+resource, body)
	if err != nil {
		return nil, 0, err
	}
	apiRequest.setHeaders(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, 0, err
	}
	respBody, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	return respBody, resp.StatusCode, nil
}

// generateErr takes the APIError and turns it into a Go error
func (apiError *APIError) generateErr() error {
	msg := apiError.RequestError.ServiceException.MessageID + " - "
	msg += apiError.RequestError.ServiceException.Text + " - "
	msg += apiError.RequestError.ServiceException.Variables
	if msg == " -  - " {
		msg = apiError.RequestError.PolicyException.MessageID + " - "
		msg += apiError.RequestError.PolicyException.Text + " - "
		msg += apiError.RequestError.PolicyException.Variables

	}
	if msg == " -  - " {
		/*
			#FIXME
			http://developerboards.att.lithium.com/t5/API-Platform/Speech-API-STTC-Error-Returns-Invalid-JSON/td-p/38929
		*/
		msg = "could not parse JSON error from the AT&T Speech API"
	}
	return errors.New(msg)
}

// setHeaders returns the APIRequest as a map
func (apiRequest *APIRequest) setHeaders(req *http.Request) {
	headers := make(map[string]string)
	xarg := ""

	s := reflect.ValueOf(apiRequest).Elem()
	typeOfT := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		name := typeOfT.Field(i).Name
		if name != "Data" && name != "Text" && name != "Filename" {
			if f.Interface().(string) != "" {
				if name == "VoiceName" || name == "Volume" || name == "Tempo" {
					xarg += "," + name + "=" + f.Interface().(string)
				} else {
					headers[toDash(name)] = f.Interface().(string)
				}
			}
		}
	}
	headers["X-Arg"] += xarg
	for key, value := range headers {
		req.Header.Add(key, value)
	}
}

/*
toDash converts an uppercase string into a string
where uppercase letters are sperated by a '-'
*/
func toDash(value string) string {
	var words []string
	l := 0
	for s := value; s != ""; s = s[l:] {
		l = strings.IndexFunc(s[1:], unicode.IsUpper) + 1
		if l <= 0 {
			l = len(s)
		}
		words = append(words, s[:l])
	}
	dashedWord := ""
	numWords := len(words)
	for i := 0; i < numWords; i++ {
		if i == 0 && numWords > 1 {
			dashedWord = words[0] + "-"
		} else {
			dashedWord += words[i]
		}
	}
	return dashedWord
}

//buildForm builds a multipart form to send the file with
func buildForm(apiRequest *APIRequest, grammar string, dictionary string) (*bytes.Buffer, string) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	defer writer.Close()

	if dictionary != "" {
		// Add the dictionary field
		contentDisposition := "form-data; name=\"x-dictionary\"; filename=\"speech_alpha.pls\""
		apiRequest.addField(writer, dictionary, contentDisposition, "application/pls+xml")
	}

	// Add the grammar field
	contentDisposition := "form-data; name=\"x-grammar\""
	apiRequest.addField(writer, grammar, contentDisposition, "application/srgs+xml")

	// Add the file field
	contentDisposition = "form-data; name=\"x-voice\"; filename=\"" + apiRequest.Filename + "\""
	apiRequest.addField(writer, grammar, contentDisposition, apiRequest.ContentType)

	contentType := writer.FormDataContentType()
	contentType = strings.Replace(contentType, "form-data", "x-srgs-audio", 1)
	return body, contentType
}

// addField adds a field to a multipart form
func (apiRequest *APIRequest) addField(writer *multipart.Writer, body string, contentDisposition string, contentType string) {
	header := make(map[string][]string)
	header["Content-Disposition"] = []string{contentDisposition}
	header["Content-Type"] = []string{contentType}
	part, _ := writer.CreatePart(header)
	if contentType == "application/pls+xml" || contentType == "application/srgs+xml" {
		part.Write([]byte(body + "\n"))
	} else {
		io.Copy(part, apiRequest.Data)
	}
}
<|endoftext|>


<commit_before>package consensus

import (
	"errors"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/gateway"
)

// TestSimpleInitialBlockchainDownload tests that
// threadedInitialBlockchainDownload synchronizes with peers in the simple case
// where there are 8 outbound peers with the same blockchain.
func TestSimpleInitialBlockchainDownload(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	// Create 8 remote peers.
	remoteCSTs := make([]*consensusSetTester, 8)
	for i := range remoteCSTs {
		cst, err := blankConsensusSetTester(fmt.Sprintf("TestSimpleInitialBlockchainDownload - %v", i))
		if err != nil {
			t.Fatal(err)
		}
		defer cst.Close()
		remoteCSTs[i] = cst
	}
	// Create the "local" peer.
	localCST, err := blankConsensusSetTester("TestSimpleInitialBlockchainDownload - local")
	if err != nil {
		t.Fatal(err)
	}
	defer localCST.Close()
	for _, cst := range remoteCSTs {
		err = localCST.cs.gateway.Connect(cst.cs.gateway.Address())
		if err != nil {
			t.Fatal(err)
		}
	}
	// Give the OnConnectRPCs time to finish.
	time.Sleep(1 * time.Second)

	// Test IBD when all peers have only the genesis block.
	doneChan := make(chan struct{})
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when all remote peers have the same longest chain.
	for i := 0; i < 20; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when not starting from the genesis block.
	for i := 0; i < 4; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when the remote peers are on a longer fork.
	for i := 0; i < 5; i++ {
		b, err := localCST.miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		err = localCST.cs.managedAcceptBlock(b)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := 0; i < 10; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when the remote peers are on a shorter fork.
	for i := 0; i < 10; i++ {
		b, err := localCST.miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		err = localCST.cs.managedAcceptBlock(b)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := 0; i < 5; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	localCurrentBlock := localCST.cs.CurrentBlock()
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != localCurrentBlock.ID() {
		t.Fatalf("local was on a longer fork and should not have reorged")
	}
	if localCST.cs.CurrentBlock().ID() == remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("ibd syncing is one way, and a longer fork on the local cs should not cause a reorg on the remote cs's")
	}
}

type mockGatewayRPCError struct {
	modules.Gateway
	rpcErrs map[modules.NetAddress]error
}

func (g *mockGatewayRPCError) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	return g.rpcErrs[addr]
}

// TestInitialBlockChainDownloadDisconnects tests that
// threadedInitialBlockchainDownload only disconnects from peers that error
// with anything but a timeout.
func TestInitialBlockchainDownloadDisconnects(t *testing.T) {
	testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDisconnects")
	g, err := gateway.New("localhost:0", filepath.Join(testdir, "local", modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	defer g.Close()
	mg := mockGatewayRPCError{
		Gateway: g,
		rpcErrs: make(map[modules.NetAddress]error),
	}
	localCS, err := New(&mg, filepath.Join(testdir, "local", modules.ConsensusDir))
	if err != nil {
		t.Fatal(err)
	}
	defer localCS.Close()

	rpcErrs := []error{
		// rpcErrs that should cause a a disconnect.
		io.EOF,
		errors.New("random error"),
		errSendBlocksStalled,
		// rpcErrs that should not cause a disconnect.
		mockNetError{
			error:   errors.New("mock timeout error"),
			timeout: true,
		},
		// Need at least minNumOutbound peers that return nil for
		// threadedInitialBlockchainDownload to mark IBD done.
		nil, nil, nil, nil, nil,
	}
	for i, rpcErr := range rpcErrs {
		g, err := gateway.New("localhost:0", filepath.Join(testdir, "remote - "+strconv.Itoa(i), modules.GatewayDir))
		if err != nil {
			t.Fatal(err)
		}
		defer g.Close()
		err = localCS.gateway.Connect(g.Address())
		if err != nil {
			t.Fatal(err)
		}
		mg.rpcErrs[g.Address()] = rpcErr
	}
	// Sleep to to give the OnConnectRPCs time to finish.
	time.Sleep(500 * time.Millisecond)
	// Do IBD.
	localCS.threadedInitialBlockchainDownload()
	// Check that localCS disconnected from peers that errored but did not time out during SendBlocks.
	for _, p := range localCS.gateway.Peers() {
		err = mg.rpcErrs[p.NetAddress]
		if err == nil {
			continue
		}
		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
			continue
		}
		t.Fatalf("threadedInitialBlockchainDownload didn't disconnect from a peer that returned '%v'", err)
	}
	if len(localCS.gateway.Peers()) != 6 {
		t.Error("threadedInitialBlockchainDownload disconnected from peers that timedout or didn't error")
	}
}
<commit_msg>Test done rules for IBD<commit_after>package consensus

import (
	"errors"
	"fmt"
	"io"
	"net"
	"path/filepath"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/gateway"
)

// TestSimpleInitialBlockchainDownload tests that
// threadedInitialBlockchainDownload synchronizes with peers in the simple case
// where there are 8 outbound peers with the same blockchain.
func TestSimpleInitialBlockchainDownload(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	// Create 8 remote peers.
	remoteCSTs := make([]*consensusSetTester, 8)
	for i := range remoteCSTs {
		cst, err := blankConsensusSetTester(fmt.Sprintf("TestSimpleInitialBlockchainDownload - %v", i))
		if err != nil {
			t.Fatal(err)
		}
		defer cst.Close()
		remoteCSTs[i] = cst
	}
	// Create the "local" peer.
	localCST, err := blankConsensusSetTester("TestSimpleInitialBlockchainDownload - local")
	if err != nil {
		t.Fatal(err)
	}
	defer localCST.Close()
	for _, cst := range remoteCSTs {
		err = localCST.cs.gateway.Connect(cst.cs.gateway.Address())
		if err != nil {
			t.Fatal(err)
		}
	}
	// Give the OnConnectRPCs time to finish.
	time.Sleep(1 * time.Second)

	// Test IBD when all peers have only the genesis block.
	doneChan := make(chan struct{})
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when all remote peers have the same longest chain.
	for i := 0; i < 20; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when not starting from the genesis block.
	for i := 0; i < 4; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when the remote peers are on a longer fork.
	for i := 0; i < 5; i++ {
		b, err := localCST.miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		err = localCST.cs.managedAcceptBlock(b)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := 0; i < 10; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("current block ids do not match: expected '%v', got '%v'", remoteCSTs[0].cs.CurrentBlock().ID(), localCST.cs.CurrentBlock().ID())
	}

	// Test IBD when the remote peers are on a shorter fork.
	for i := 0; i < 10; i++ {
		b, err := localCST.miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		err = localCST.cs.managedAcceptBlock(b)
		if err != nil {
			t.Fatal(err)
		}
	}
	for i := 0; i < 5; i++ {
		b, err := remoteCSTs[0].miner.FindBlock()
		if err != nil {
			t.Fatal(err)
		}
		for _, cst := range remoteCSTs {
			err = cst.cs.managedAcceptBlock(b)
			if err != nil {
				t.Fatal(err)
			}
		}
	}
	localCurrentBlock := localCST.cs.CurrentBlock()
	go func() {
		localCST.cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(5 * time.Second):
		t.Fatal("initialBlockchainDownload never completed")
	}
	if localCST.cs.CurrentBlock().ID() != localCurrentBlock.ID() {
		t.Fatalf("local was on a longer fork and should not have reorged")
	}
	if localCST.cs.CurrentBlock().ID() == remoteCSTs[0].cs.CurrentBlock().ID() {
		t.Fatalf("ibd syncing is one way, and a longer fork on the local cs should not cause a reorg on the remote cs's")
	}
}

type mockGatewayRPCError struct {
	modules.Gateway
	rpcErrs map[modules.NetAddress]error
	mu      sync.Mutex
}

func (g *mockGatewayRPCError) RPC(addr modules.NetAddress, name string, fn modules.RPCFunc) error {
	g.mu.Lock()
	defer g.mu.Unlock()
	return g.rpcErrs[addr]
}

// TestInitialBlockChainDownloadDisconnects tests that
// threadedInitialBlockchainDownload only disconnects from peers that error
// with anything but a timeout.
func TestInitialBlockchainDownloadDisconnects(t *testing.T) {
	testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDisconnects")
	g, err := gateway.New("localhost:0", filepath.Join(testdir, "local", modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	defer g.Close()
	mg := mockGatewayRPCError{
		Gateway: g,
		rpcErrs: make(map[modules.NetAddress]error),
	}
	localCS, err := New(&mg, filepath.Join(testdir, "local", modules.ConsensusDir))
	if err != nil {
		t.Fatal(err)
	}
	defer localCS.Close()

	rpcErrs := []error{
		// rpcErrs that should cause a a disconnect.
		io.EOF,
		errors.New("random error"),
		errSendBlocksStalled,
		// rpcErrs that should not cause a disconnect.
		mockNetError{
			error:   errors.New("mock timeout error"),
			timeout: true,
		},
		// Need at least minNumOutbound peers that return nil for
		// threadedInitialBlockchainDownload to mark IBD done.
		nil, nil, nil, nil, nil,
	}
	for i, rpcErr := range rpcErrs {
		g, err := gateway.New("localhost:0", filepath.Join(testdir, "remote - "+strconv.Itoa(i), modules.GatewayDir))
		if err != nil {
			t.Fatal(err)
		}
		defer g.Close()
		err = localCS.gateway.Connect(g.Address())
		if err != nil {
			t.Fatal(err)
		}
		mg.rpcErrs[g.Address()] = rpcErr
	}
	// Sleep to to give the OnConnectRPCs time to finish.
	time.Sleep(500 * time.Millisecond)
	// Do IBD.
	localCS.threadedInitialBlockchainDownload()
	// Check that localCS disconnected from peers that errored but did not time out during SendBlocks.
	for _, p := range localCS.gateway.Peers() {
		err = mg.rpcErrs[p.NetAddress]
		if err == nil {
			continue
		}
		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
			continue
		}
		t.Fatalf("threadedInitialBlockchainDownload didn't disconnect from a peer that returned '%v'", err)
	}
	if len(localCS.gateway.Peers()) != 6 {
		t.Error("threadedInitialBlockchainDownload disconnected from peers that timedout or didn't error")
	}
}

// TestInitialBlockchainDownloadDoneRules tests that
// threadedInitialBlockchainDownload only terminates under the appropriate
// conditions. Appropriate conditions are:
//  - at least minNumOutbound synced outbound peers
//  - or at least 1 synced outbound peer and minIBDWaitTime has passed since beginning IBD.
func TestInitialBlockchainDownloadDoneRules(t *testing.T) {
	testdir := build.TempDir(modules.ConsensusDir, "TestInitialBlockchainDownloadDoneRules")
	g, err := gateway.New("localhost:0", filepath.Join(testdir, "local", modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	defer g.Close()
	mg := mockGatewayRPCError{
		Gateway: g,
		rpcErrs: make(map[modules.NetAddress]error),
	}
	cs, err := New(&mg, filepath.Join(testdir, "local", modules.ConsensusDir))
	if err != nil {
		t.Fatal(err)
	}
	defer cs.Close()

	doneChan := make(chan struct{})

	// Test when there are 0 peers.
	go func() {
		cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
		t.Error("threadedInitialBlockchainDownload finished with 0 synced peers")
	case <-time.After(minIBDWaitTime * 11 / 10):
	}

	// Test when there are only inbound peers.
	inboundCSTs := make([]*consensusSetTester, 8)
	for i := 0; i < len(inboundCSTs); i++ {
		inboundCST, err := blankConsensusSetTester(filepath.Join("TestInitialBlockchainDownloadDoneRules", fmt.Sprintf("remote - inbound %v", i)))
		if err != nil {
			t.Fatal(err)
		}
		defer inboundCST.Close()

		inboundCST.cs.gateway.Connect(cs.gateway.Address())
	}
	select {
	case <-doneChan:
		t.Error("threadedInitialBlockchainDownload finished with only inbound peers")
	case <-time.After(minIBDWaitTime * 11 / 10):
	}

	// Test when there is 1 peer that isn't synced.
	gatewayTimesout, err := gateway.New("localhost:0", filepath.Join(testdir, "remote - timesout", modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	defer gatewayTimesout.Close()
	mg.mu.Lock()
	mg.rpcErrs[gatewayTimesout.Address()] = mockNetError{
		error:   errors.New("mock timeout error"),
		timeout: true,
	}
	mg.mu.Unlock()
	err = cs.gateway.Connect(gatewayTimesout.Address())
	if err != nil {
		t.Fatal(err)
	}
	select {
	case <-doneChan:
		t.Error("threadedInitialBlockchainDownload finished with 0 synced peers")
	case <-time.After(minIBDWaitTime * 11 / 10):
	}

	// Test when there is 1 peer that is synced and one that is not synced.
	gatewayNoTimeout, err := gateway.New("localhost:0", filepath.Join(testdir, "remote - no timeout", modules.GatewayDir))
	if err != nil {
		t.Fatal(err)
	}
	defer gatewayNoTimeout.Close()
	mg.mu.Lock()
	mg.rpcErrs[gatewayNoTimeout.Address()] = nil
	mg.mu.Unlock()
	err = cs.gateway.Connect(gatewayNoTimeout.Address())
	if err != nil {
		t.Fatal(err)
	}
	select {
	case <-doneChan:
	case <-time.After(minIBDWaitTime * 11 / 10):
		t.Fatal("threadedInitialBlockchainDownload never finished with 1 synced peer")
	}

	// Test when there are >= minNumOutbound peers, but < minNumOutbound peers are synced.
	gatewayTimesouts := make([]modules.Gateway, minNumOutbound-1)
	for i := 0; i < len(gatewayTimesouts); i++ {
		tmpG, err := gateway.New("localhost:0", filepath.Join(testdir, fmt.Sprintf("remote - timesout %v", i), modules.GatewayDir))
		if err != nil {
			t.Fatal(err)
		}
		defer tmpG.Close()
		mg.mu.Lock()
		mg.rpcErrs[tmpG.Address()] = mockNetError{
			error:   errors.New("mock timeout error"),
			timeout: true,
		}
		mg.mu.Unlock()
		gatewayTimesouts[i] = tmpG
		err = cs.gateway.Connect(gatewayTimesouts[i].Address())
		if err != nil {
			t.Fatal(err)
		}
	}
	go func() {
		cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
		t.Fatal("threadedInitialBlockchainDownload finished before minIBDWaitTime")
	case <-time.After(minIBDWaitTime):
	}
	select {
	case <-doneChan:
	case <-time.After(minIBDWaitTime):
		t.Fatal("threadedInitialBlockchainDownload didn't finish after minIBDWaitTime")
	}

	// Test when there are >= minNumOutbound peers and >= minNumOutbound peers are synced.
	gatewayNoTimeouts := make([]modules.Gateway, minNumOutbound-1)
	for i := 0; i < len(gatewayNoTimeouts); i++ {
		tmpG, err := gateway.New("localhost:0", filepath.Join(testdir, fmt.Sprintf("remote - no timeout %v", i), modules.GatewayDir))
		if err != nil {
			t.Fatal(err)
		}
		defer tmpG.Close()
		mg.mu.Lock()
		mg.rpcErrs[tmpG.Address()] = nil
		mg.mu.Unlock()
		gatewayNoTimeouts[i] = tmpG
		err = cs.gateway.Connect(gatewayNoTimeouts[i].Address())
		if err != nil {
			t.Fatal(err)
		}
	}
	go func() {
		cs.threadedInitialBlockchainDownload()
		doneChan <- struct{}{}
	}()
	select {
	case <-doneChan:
	case <-time.After(minIBDWaitTime):
		t.Fatal("threadedInitialBlockchainDownload didn't finish in less than minIBDWaitTime")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Scott Mansfield
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reader

import (
	"bytes"
	"testing"

	"github.com/ScottMansfield/nanolog"
)

func TestReader(t *testing.T) {
	inbuf := &bytes.Buffer{}
	nanolog.SetWriter(inbuf)

	// This should exercise every different type in one log line
	h := nanolog.AddLogger("%b %s %i %i8 %i16 %i32 %i64 %u %u8 %u16 %u32 %u64 %f32 %f64 %c64 %c128")
	nanolog.Log(h,
		true, "",
		int(4), int8(4), int16(4), int32(4), int64(4),
		uint(4), uint8(4), uint16(4), uint32(4), uint64(4),
		float32(4), float64(4),
		complex(float32(4), float32(4)), complex(float64(4), float64(4)),
	)
	nanolog.Flush()

	outbuf := &bytes.Buffer{}
	r := New(inbuf, outbuf)
	if err := r.Inflate(); err != nil {
		t.Fatalf("Got error during inflate: %v", err)
	}
}
<commit_msg>Adding another reader test to exercise failure modes<commit_after>// Copyright 2017 Scott Mansfield
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reader

import (
	"bytes"
	"fmt"
	"testing"

	"io"

	"github.com/ScottMansfield/nanolog"
)

func TestReader(t *testing.T) {
	inbuf := &bytes.Buffer{}
	nanolog.SetWriter(inbuf)

	// This should exercise every different type in one log line
	h := nanolog.AddLogger("%b %s %i %i8 %i16 %i32 %i64 %u %u8 %u16 %u32 %u64 %f32 %f64 %c64 %c128")
	nanolog.Log(h,
		true, "",
		int(4), int8(4), int16(4), int32(4), int64(4),
		uint(4), uint8(4), uint16(4), uint32(4), uint64(4),
		float32(4), float64(4),
		complex(float32(4), float32(4)), complex(float64(4), float64(4)),
	)
	nanolog.Flush()

	outbuf := &bytes.Buffer{}
	r := New(inbuf, outbuf)
	if err := r.Inflate(); err != nil {
		t.Fatalf("Got error during inflate: %v", err)
	}
}

func TestReaderEarlyExits(t *testing.T) {
	inbuf := &bytes.Buffer{}
	nanolog.SetWriter(inbuf)

	// This should exercise every different type in one log line
	h := nanolog.AddLogger("%b %s %i %i8 %i16 %i32 %i64 %u %u8 %u16 %u32 %u64 %f32 %f64 %c64 %c128")
	nanolog.Log(h,
		false, "",
		int(4), int8(4), int16(4), int32(4), int64(4),
		uint(4), uint8(4), uint16(4), uint32(4), uint64(4),
		float32(4), float64(4),
		complex(float32(4), float32(4)), complex(float64(4), float64(4)),
	)
	nanolog.Flush()

	outbuf := &bytes.Buffer{}

	for i := 0; i < inbuf.Len(); i++ {
		t.Run(fmt.Sprintf("Exit%d", i), func(t *testing.T) {
			defer func() {
				if r := recover(); r != nil {
					t.Logf("Panic!: %v", r)
				} else {
					t.Logf("No panic.")
				}
			}()

			outbuf.Reset()
			inbufcopy := bytes.NewReader(inbuf.Bytes())
			r := New(io.LimitReader(inbufcopy, int64(i)), outbuf)
			if err := r.Inflate(); err != nil {
				t.Logf("Got error during inflate: %v", err)
			}
		})
	}
}
<|endoftext|>


<commit_before>package deploy

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	kube "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	rest "k8s.io/kubernetes/pkg/client/restclient"
	kubecli "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	"k8s.io/kubernetes/pkg/kubectl/cmd/config"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

const DefaultContext = ""

// KubeCluster is able to deploy to Kubernetes clusters. This is a very simple implementation with no error recovery.
type KubeCluster struct {
	client  *kubecli.Client
	context string
}

// NewKubeClusterFromContext creates a KubeCluster using a Kubernetes client with the configuration of the given context.
// If the context name is empty, the default context will be used.
func NewKubeClusterFromContext(name string) (*KubeCluster, error) {
	rules := defaultLoadingRules()

	overrides := &clientcmd.ConfigOverrides{
		CurrentContext: name,
	}

	config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)

	clientConfig, err := config.ClientConfig()
	if err != nil {
		if len(name) == 0 {
			return nil, fmt.Errorf("could not use default context: %v", err)
		}
		return nil, fmt.Errorf("could not use context `%s`: %v", name, err)
	}

	client, err := kubecli.New(clientConfig)
	if err != nil {
		return nil, fmt.Errorf("could not create Kubernetes client: %v", err)
	}

	return &KubeCluster{
		client:  client,
		context: name,
	}, nil
}

// Context returns the kubectl context being used
func (c *KubeCluster) Context() string {
	return c.context
}

// Deploy creates/updates the Deployment's objects on the Kubernetes cluster.
// Currently no error recovery is implemented; if there is an error the deployment process will immediately halt and return the error.
// If update is not set, will error if objects exist. If deleteModifiedPods is set, pods of modified RCs will be deleted.
func (c *KubeCluster) Deploy(dep *Deployment, update, deleteModifiedPods bool) error {
	if c.client == nil {
		return errors.New("client not setup (was nil)")
	}

	// create namespaces before everything else
	for _, ns := range dep.namespaces {
		_, err := c.client.Namespaces().Create(ns)
		if err != nil && !alreadyExists(err) {
			return err
		}
	}

	// TODO: add continue on error and error lists
	for _, obj := range dep.Objects() {
		// don't create namespaces again
		if _, isNamespace := obj.(*kube.Namespace); isNamespace {
			continue
		}

		err := c.deploy(obj, update)
		if err != nil {
			return err
		}

		if rc, isRC := obj.(*kube.ReplicationController); isRC && deleteModifiedPods {
			err = c.deletePods(rc)
			if err != nil {
				return fmt.Errorf("could not delete pods for rc `%s/%s`: %v", rc.Namespace, rc.Name, err)
			}
		}
	}

	printLoadBalancers(c.client, dep.services)

	// deployed successfully
	return nil
}

// deploy creates the object on the connected Kubernetes instance. Errors if object exists and not updating.
func (c *KubeCluster) deploy(obj KubeObject, update bool) error {
	if obj == nil {
		return errors.New("tried to deploy nil object")
	}

	mapping, err := mapping(obj)
	if err != nil {
		return err
	}

	if update {
		_, err := c.update(obj, true, mapping)
		if err != nil {
			return err
		}
		return nil
	}

	_, err = c.create(obj, mapping)
	return err
}

// update replaces the currently deployed version with a new one. If the objects already match then nothing is done.
func (c *KubeCluster) update(obj KubeObject, create bool, mapping *meta.RESTMapping) (KubeObject, error) {
	meta := obj.GetObjectMeta()

	deployed, err := c.get(meta.GetNamespace(), meta.GetName(), true, mapping)
	if doesNotExist(err) && create {
		return c.create(obj, mapping)
	} else if err != nil {
		return nil, err
	}

	// TODO: need a better way to handle resource versioning
	// set resource version on local to same as remote
	deployedVersion := deployed.GetObjectMeta().GetResourceVersion()
	meta.SetResourceVersion(deployedVersion)

	copyImmutables(deployed, obj)

	// if local matches deployed, do nothing
	if kube.Semantic.DeepEqual(obj, deployed) {
		return deployed, nil
	}

	patch, err := diff(deployed, obj)
	if err != nil {
		return nil, fmt.Errorf("could not create diff: %v", err)
	}

	req := c.client.RESTClient.Patch(kube.StrategicMergePatchType).
		Name(meta.GetName()).
		Body(patch)

	setRequestObjectInfo(req, meta.GetNamespace(), mapping)

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("update", meta.GetNamespace(), meta.GetName(), mapping, err)
	}

	return asKubeObject(runtimeObj)
}

// get retrieves the object from the cluster.
func (c *KubeCluster) get(namespace, name string, export bool, mapping *meta.RESTMapping) (KubeObject, error) {
	req := c.client.RESTClient.Get().Name(name)
	setRequestObjectInfo(req, namespace, mapping)

	if export {
		req.Param("export", "true")
	}

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("get", namespace, name, mapping, err)
	}

	return asKubeObject(runtimeObj)
}

// create adds the object to the cluster.
func (c *KubeCluster) create(obj KubeObject, mapping *meta.RESTMapping) (KubeObject, error) {
	meta := obj.GetObjectMeta()
	req := c.client.RESTClient.Post().Body(obj)

	setRequestObjectInfo(req, meta.GetNamespace(), mapping)

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("create", meta.GetName(), meta.GetNamespace(), mapping, err)
	}

	return asKubeObject(runtimeObj)
}

func (c *KubeCluster) deletePods(rc *kube.ReplicationController) error {
	if rc == nil {
		return errors.New("rc was nil")
	}

	// list pods
	opts := kube.ListOptions{
		LabelSelector: labels.Set(rc.Spec.Selector).AsSelector(),
	}
	podList, err := c.client.Pods(rc.Namespace).List(opts)
	if err != nil {
		return err
	}

	// delete pods
	for _, pod := range podList.Items {
		err := c.client.Pods(pod.Namespace).Delete(pod.Name, nil)
		if err != nil {
			return err
		}
	}
	return nil
}

// setRequestObjectInfo adds necessary type information to requests.
func setRequestObjectInfo(req *rest.Request, namespace string, mapping *meta.RESTMapping) {
	// if namespace scoped resource, set namespace
	req.NamespaceIfScoped(namespace, isNamespaceScoped(mapping))

	// set resource name
	req.Resource(mapping.Resource)
}

// alreadyExists checks if the error is for a resource already existing.
func alreadyExists(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "already exists")
}

// doesNotExist checks if the error is for a non-existent resource.
func doesNotExist(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "not found")
}

// mapping returns the appropriate RESTMapping for the object.
func mapping(obj KubeObject) (*meta.RESTMapping, error) {
	gvk, err := kube.Scheme.ObjectKind(obj)
	if err != nil {
		return nil, err
	}

	mapping, err := kube.RESTMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
	if err != nil {
		return nil, fmt.Errorf("could not create RESTMapping for %s: %v", gvk, err)
	}
	return mapping, nil
}

// isNamespaceScoped returns if the mapping is scoped by Namespace.
func isNamespaceScoped(mapping *meta.RESTMapping) bool {
	return mapping.Scope.Name() == meta.RESTScopeNameNamespace
}

// defaultLoadingRules use the same rules (as of 2/17/16) as kubectl.
func defaultLoadingRules() *clientcmd.ClientConfigLoadingRules {
	opts := config.NewDefaultPathOptions()

	loadingRules := opts.LoadingRules
	loadingRules.Precedence = opts.GetLoadingPrecedence()
	return loadingRules
}

// diff creates a patch.
func diff(original, modified runtime.Object) (patch []byte, err error) {
	origBytes, err := json.Marshal(original)
	if err != nil {
		return nil, err
	}

	modBytes, err := json.Marshal(modified)
	if err != nil {
		return nil, err
	}

	return strategicpatch.CreateTwoWayMergePatch(origBytes, modBytes, original)
}

// asKubeObject attempts use the object as a KubeObject. It will return an error if not possible.
func asKubeObject(runtimeObj runtime.Object) (KubeObject, error) {
	kubeObj, ok := runtimeObj.(KubeObject)
	if !ok {
		return nil, errors.New("was unable to use runtime.Object as deploy.KubeObject")
	}
	return kubeObj, nil
}

func resourceError(action, namespace, name string, mapping *meta.RESTMapping, err error) error {
	if mapping == nil || mapping.GroupVersionKind.IsEmpty() {
		return fmt.Errorf("could not %s '%s/%s': %v", action, namespace, name, err)
	}
	gvk := mapping.GroupVersionKind
	return fmt.Errorf("could not %s '%s/%s' (%s): %v", action, namespace, name, gvk.Kind, err)
}

// copyImmutables sets any immutable fields from src on dst. Will panic if objects not of same type.
func copyImmutables(src, dst KubeObject) {
	if src == nil || dst == nil {
		return
	}

	// each type has specific fields that must be copied
	switch src := src.(type) {
	case *kube.Service:
		dst := dst.(*kube.Service)
		dst.Spec.ClusterIP = src.Spec.ClusterIP
	}
}

func printLoadBalancers(client *kubecli.Client, services []*kube.Service) {
	if len(services) == 0 {
		return
	}

	first := true
	completed := map[string]bool{}

	// checks when we've seen every service
	done := func() bool {
		for _, s := range services {
			if s.Spec.Type == kube.ServiceTypeLoadBalancer && !completed[s.Name] {
				return false
			}
		}
		return true
	}

	for {
		if done() {
			return
		}

		if first {
			fmt.Println("Waiting for load balancer deployment...")
			first = false
		}

		for _, s := range services {
			if s.Spec.Type == kube.ServiceTypeLoadBalancer && !completed[s.Name] {
				clusterVers, err := client.Services(s.Namespace).Get(s.Name)
				if err != nil {
					fmt.Printf("Error getting service `%s`: %v\n", s.Name, err)
				}

				loadBalancers := clusterVers.Status.LoadBalancer.Ingress
				if len(loadBalancers) == 1 {
					completed[s.Name] = true
					fmt.Printf("Service '%s/%s' available at: \t%s\n", s.Namespace, s.Name, loadBalancers[0].IP)
				}
			}
		}
	}
}
<commit_msg>rate limited LoadBalancer check to prevent warning<commit_after>package deploy

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	kube "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	rest "k8s.io/kubernetes/pkg/client/restclient"
	kubecli "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	"k8s.io/kubernetes/pkg/kubectl/cmd/config"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

const DefaultContext = ""

// KubeCluster is able to deploy to Kubernetes clusters. This is a very simple implementation with no error recovery.
type KubeCluster struct {
	client  *kubecli.Client
	context string
}

// NewKubeClusterFromContext creates a KubeCluster using a Kubernetes client with the configuration of the given context.
// If the context name is empty, the default context will be used.
func NewKubeClusterFromContext(name string) (*KubeCluster, error) {
	rules := defaultLoadingRules()

	overrides := &clientcmd.ConfigOverrides{
		CurrentContext: name,
	}

	config := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)

	clientConfig, err := config.ClientConfig()
	if err != nil {
		if len(name) == 0 {
			return nil, fmt.Errorf("could not use default context: %v", err)
		}
		return nil, fmt.Errorf("could not use context `%s`: %v", name, err)
	}

	client, err := kubecli.New(clientConfig)
	if err != nil {
		return nil, fmt.Errorf("could not create Kubernetes client: %v", err)
	}

	return &KubeCluster{
		client:  client,
		context: name,
	}, nil
}

// Context returns the kubectl context being used
func (c *KubeCluster) Context() string {
	return c.context
}

// Deploy creates/updates the Deployment's objects on the Kubernetes cluster.
// Currently no error recovery is implemented; if there is an error the deployment process will immediately halt and return the error.
// If update is not set, will error if objects exist. If deleteModifiedPods is set, pods of modified RCs will be deleted.
func (c *KubeCluster) Deploy(dep *Deployment, update, deleteModifiedPods bool) error {
	if c.client == nil {
		return errors.New("client not setup (was nil)")
	}

	// create namespaces before everything else
	for _, ns := range dep.namespaces {
		_, err := c.client.Namespaces().Create(ns)
		if err != nil && !alreadyExists(err) {
			return err
		}
	}

	// TODO: add continue on error and error lists
	for _, obj := range dep.Objects() {
		// don't create namespaces again
		if _, isNamespace := obj.(*kube.Namespace); isNamespace {
			continue
		}

		err := c.deploy(obj, update)
		if err != nil {
			return err
		}

		if rc, isRC := obj.(*kube.ReplicationController); isRC && deleteModifiedPods {
			err = c.deletePods(rc)
			if err != nil {
				return fmt.Errorf("could not delete pods for rc `%s/%s`: %v", rc.Namespace, rc.Name, err)
			}
		}
	}

	printLoadBalancers(c.client, dep.services)

	// deployed successfully
	return nil
}

// deploy creates the object on the connected Kubernetes instance. Errors if object exists and not updating.
func (c *KubeCluster) deploy(obj KubeObject, update bool) error {
	if obj == nil {
		return errors.New("tried to deploy nil object")
	}

	mapping, err := mapping(obj)
	if err != nil {
		return err
	}

	if update {
		_, err := c.update(obj, true, mapping)
		if err != nil {
			return err
		}
		return nil
	}

	_, err = c.create(obj, mapping)
	return err
}

// update replaces the currently deployed version with a new one. If the objects already match then nothing is done.
func (c *KubeCluster) update(obj KubeObject, create bool, mapping *meta.RESTMapping) (KubeObject, error) {
	meta := obj.GetObjectMeta()

	deployed, err := c.get(meta.GetNamespace(), meta.GetName(), true, mapping)
	if doesNotExist(err) && create {
		return c.create(obj, mapping)
	} else if err != nil {
		return nil, err
	}

	// TODO: need a better way to handle resource versioning
	// set resource version on local to same as remote
	deployedVersion := deployed.GetObjectMeta().GetResourceVersion()
	meta.SetResourceVersion(deployedVersion)

	copyImmutables(deployed, obj)

	// if local matches deployed, do nothing
	if kube.Semantic.DeepEqual(obj, deployed) {
		return deployed, nil
	}

	patch, err := diff(deployed, obj)
	if err != nil {
		return nil, fmt.Errorf("could not create diff: %v", err)
	}

	req := c.client.RESTClient.Patch(kube.StrategicMergePatchType).
		Name(meta.GetName()).
		Body(patch)

	setRequestObjectInfo(req, meta.GetNamespace(), mapping)

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("update", meta.GetNamespace(), meta.GetName(), mapping, err)
	}

	return asKubeObject(runtimeObj)
}

// get retrieves the object from the cluster.
func (c *KubeCluster) get(namespace, name string, export bool, mapping *meta.RESTMapping) (KubeObject, error) {
	req := c.client.RESTClient.Get().Name(name)
	setRequestObjectInfo(req, namespace, mapping)

	if export {
		req.Param("export", "true")
	}

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("get", namespace, name, mapping, err)
	}

	return asKubeObject(runtimeObj)
}

// create adds the object to the cluster.
func (c *KubeCluster) create(obj KubeObject, mapping *meta.RESTMapping) (KubeObject, error) {
	meta := obj.GetObjectMeta()
	req := c.client.RESTClient.Post().Body(obj)

	setRequestObjectInfo(req, meta.GetNamespace(), mapping)

	runtimeObj, err := req.Do().Get()
	if err != nil {
		return nil, resourceError("create", meta.GetName(), meta.GetNamespace(), mapping, err)
	}

	return asKubeObject(runtimeObj)
}

func (c *KubeCluster) deletePods(rc *kube.ReplicationController) error {
	if rc == nil {
		return errors.New("rc was nil")
	}

	// list pods
	opts := kube.ListOptions{
		LabelSelector: labels.Set(rc.Spec.Selector).AsSelector(),
	}
	podList, err := c.client.Pods(rc.Namespace).List(opts)
	if err != nil {
		return err
	}

	// delete pods
	for _, pod := range podList.Items {
		err := c.client.Pods(pod.Namespace).Delete(pod.Name, nil)
		if err != nil {
			return err
		}
	}
	return nil
}

// setRequestObjectInfo adds necessary type information to requests.
func setRequestObjectInfo(req *rest.Request, namespace string, mapping *meta.RESTMapping) {
	// if namespace scoped resource, set namespace
	req.NamespaceIfScoped(namespace, isNamespaceScoped(mapping))

	// set resource name
	req.Resource(mapping.Resource)
}

// alreadyExists checks if the error is for a resource already existing.
func alreadyExists(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "already exists")
}

// doesNotExist checks if the error is for a non-existent resource.
func doesNotExist(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "not found")
}

// mapping returns the appropriate RESTMapping for the object.
func mapping(obj KubeObject) (*meta.RESTMapping, error) {
	gvk, err := kube.Scheme.ObjectKind(obj)
	if err != nil {
		return nil, err
	}

	mapping, err := kube.RESTMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
	if err != nil {
		return nil, fmt.Errorf("could not create RESTMapping for %s: %v", gvk, err)
	}
	return mapping, nil
}

// isNamespaceScoped returns if the mapping is scoped by Namespace.
func isNamespaceScoped(mapping *meta.RESTMapping) bool {
	return mapping.Scope.Name() == meta.RESTScopeNameNamespace
}

// defaultLoadingRules use the same rules (as of 2/17/16) as kubectl.
func defaultLoadingRules() *clientcmd.ClientConfigLoadingRules {
	opts := config.NewDefaultPathOptions()

	loadingRules := opts.LoadingRules
	loadingRules.Precedence = opts.GetLoadingPrecedence()
	return loadingRules
}

// diff creates a patch.
func diff(original, modified runtime.Object) (patch []byte, err error) {
	origBytes, err := json.Marshal(original)
	if err != nil {
		return nil, err
	}

	modBytes, err := json.Marshal(modified)
	if err != nil {
		return nil, err
	}

	return strategicpatch.CreateTwoWayMergePatch(origBytes, modBytes, original)
}

// asKubeObject attempts use the object as a KubeObject. It will return an error if not possible.
func asKubeObject(runtimeObj runtime.Object) (KubeObject, error) {
	kubeObj, ok := runtimeObj.(KubeObject)
	if !ok {
		return nil, errors.New("was unable to use runtime.Object as deploy.KubeObject")
	}
	return kubeObj, nil
}

func resourceError(action, namespace, name string, mapping *meta.RESTMapping, err error) error {
	if mapping == nil || mapping.GroupVersionKind.IsEmpty() {
		return fmt.Errorf("could not %s '%s/%s': %v", action, namespace, name, err)
	}
	gvk := mapping.GroupVersionKind
	return fmt.Errorf("could not %s '%s/%s' (%s): %v", action, namespace, name, gvk.Kind, err)
}

// copyImmutables sets any immutable fields from src on dst. Will panic if objects not of same type.
func copyImmutables(src, dst KubeObject) {
	if src == nil || dst == nil {
		return
	}

	// each type has specific fields that must be copied
	switch src := src.(type) {
	case *kube.Service:
		dst := dst.(*kube.Service)
		dst.Spec.ClusterIP = src.Spec.ClusterIP
	}
}

func printLoadBalancers(client *kubecli.Client, services []*kube.Service) {
	if len(services) == 0 {
		return
	}

	first := true
	completed := map[string]bool{}

	// checks when we've seen every service
	done := func() bool {
		for _, s := range services {
			if s.Spec.Type == kube.ServiceTypeLoadBalancer && !completed[s.Name] {
				return false
			}
		}
		return true
	}

	for {
		if done() {
			return
		}

		if first {
			fmt.Println("Waiting for load balancer deployment...")
			first = false
		}

		for _, s := range services {
			if s.Spec.Type == kube.ServiceTypeLoadBalancer && !completed[s.Name] {
				clusterVers, err := client.Services(s.Namespace).Get(s.Name)
				if err != nil {
					fmt.Printf("Error getting service `%s`: %v\n", s.Name, err)
				}

				loadBalancers := clusterVers.Status.LoadBalancer.Ingress
				if len(loadBalancers) == 1 {
					completed[s.Name] = true
					fmt.Printf("Service '%s/%s' available at: \t%s\n", s.Namespace, s.Name, loadBalancers[0].IP)
				}
			}
		}

		// prevents warning about throttling
		time.Sleep(250 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Get attributes for the inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCKS_REQUIRED(in)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	in inode.Inode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = in.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	in.Lock()
	defer in.Unlock()

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = fs.getAttributes(ctx, in); err != nil {
		return
	}

	return
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	in.Lock()
	defer in.Unlock()

	// Grab its attributes.
	resp.Attributes, err = fs.getAttributes(ctx, in)
	if err != nil {
		return
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.inodes[req.Inode].(*inode.DirInode)
	in.Lock()
	defer in.Unlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<commit_msg>Added missing annotations.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Get attributes for the inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// EXCLUSIVE_LOCKS_REQUIRED(in)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	in inode.Inode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = in.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	in.Lock()
	defer in.Unlock()

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = fs.getAttributes(ctx, in); err != nil {
		return
	}

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	in.Lock()
	defer in.Unlock()

	// Grab its attributes.
	resp.Attributes, err = fs.getAttributes(ctx, in)
	if err != nil {
		return
	}

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.inodes[req.Inode].(*inode.DirInode)
	in.Lock()
	defer in.Unlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/miku/ntto"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"
)

func Worker(queue chan *string, out chan *ntto.Triple, wg *sync.WaitGroup, ignore *bool) {
	defer wg.Done()
	for b := range queue {
		triple, err := ntto.ParseNTriple(*b)
		if err != nil {
			if !*ignore {
				log.Fatalln(err)
			} else {
				log.Println(err)
			}
		}
		out <- triple
	}
}

func Marshaller(writer io.Writer, in chan *ntto.Triple, done chan bool, ignore *bool) {
	for triple := range in {
		b, err := json.Marshal(triple)
		if err != nil {
			if !*ignore {
				log.Fatalln(err)
			} else {
				log.Println(err)
			}
		}
		writer.Write(b)
		writer.Write([]byte("\n"))
	}
	done <- true
}

func main() {

	executive := "replace"
	_, err := exec.LookPath("replace")
	if err != nil {
		executive = "perl"
	}

	_, err = exec.LookPath("perl")
	if err != nil {
		log.Fatalln("This program requires perl or replace.")
		os.Exit(1)
	}

	abbreviate := flag.Bool("a", false, "abbreviate n-triples using rules")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	dumpCommand := flag.Bool("c", false, "dump constructed sed command and exit")
	dumpRules := flag.Bool("d", false, "dump rules and exit")
	ignore := flag.Bool("i", false, "ignore conversion errors")
	jsonOutput := flag.Bool("j", false, "convert nt to json")
	nullValue := flag.String("n", "<NULL>", "string to indicate empty string replacement")
	outFile := flag.String("o", "", "output file to write result to")
	rulesFile := flag.String("r", "", "path to rules file, use built-in if none given")
	version := flag.Bool("v", false, "prints current version and exits")
	numWorkers := flag.Int("w", runtime.NumCPU(), "parallelism measure")

	flag.Parse()

	runtime.GOMAXPROCS(*numWorkers)

	var PrintUsage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] FILE\n", os.Args[0])
		flag.PrintDefaults()
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(ntto.AppVersion)
		os.Exit(0)
	}

	var rules []ntto.Rule

	if *rulesFile == "" {
		rules, err = ntto.ParseRules(ntto.DefaultRules)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		b, err := ioutil.ReadFile(*rulesFile)
		if err != nil {
			log.Fatalln(err)
		}
		rules, err = ntto.ParseRules(string(b))
		if err != nil {
			log.Fatalln(err)
		}
	}

	if *dumpRules {
		fmt.Println(ntto.DumpRules(rules))
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		PrintUsage()
		os.Exit(1)
	}

	filename := flag.Args()[0]
	var output string

	if *abbreviate {
		if *outFile == "" {
			tmp, err := ioutil.TempFile("", "ntto-")
			output = tmp.Name()
			log.Printf("No explicit [-o]utput given, writing to %s\n", output)
			if err != nil {
				log.Fatalln(err)
			}
		} else {
			output = *outFile
		}

		var command string
		if executive == "perl" {
			command = fmt.Sprintf("%s > %s", ntto.SedifyNull(rules, *numWorkers, filename, *nullValue), output)
		} else {
			command = fmt.Sprintf("%s > %s", ntto.ReplacifyNull(rules, filename, *nullValue), output)
		}
		if *dumpCommand {
			fmt.Println(command)
			os.Exit(0)
		}
		_, err = exec.Command("sh", "-c", command).Output()
		if err != nil {
			log.Fatalln(err)
		}
		// set filename to abbreviated output, so we can use combine -j -a
		filename = output
	}

	if *jsonOutput {
		var file *os.File
		if filename == "-" {
			file = os.Stdin
		} else {
			file, err = os.Open(filename)
			defer file.Close()
			if err != nil {
				log.Fatalln(err)
			}
		}

		queue := make(chan *string)
		results := make(chan *ntto.Triple)
		done := make(chan bool)

		writer := bufio.NewWriter(os.Stdout)
		defer writer.Flush()
		go Marshaller(writer, results, done, ignore)

		var wg sync.WaitGroup
		for i := 0; i < *numWorkers; i++ {
			wg.Add(1)
			go Worker(queue, results, &wg, ignore)
		}

		reader := bufio.NewReader(file)

		for {
			b, _, err := reader.ReadLine()
			if err != nil || b == nil {
				break
			}
			line := string(b)
			queue <- &line
		}
		close(queue)
		wg.Wait()
		close(results)
		select {
		case <-time.After(1e9):
			break
		case <-done:
			break
		}
		// remove abbreviated tempfile output, if possible
		if *outFile == "" {
			_ = os.Remove(output)
		}
	}
}
<commit_msg>sort imports according to goimports<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"runtime"
	"runtime/pprof"
	"sync"
	"time"

	"github.com/miku/ntto"
)

func Worker(queue chan *string, out chan *ntto.Triple, wg *sync.WaitGroup, ignore *bool) {
	defer wg.Done()
	for b := range queue {
		triple, err := ntto.ParseNTriple(*b)
		if err != nil {
			if !*ignore {
				log.Fatalln(err)
			} else {
				log.Println(err)
			}
		}
		out <- triple
	}
}

func Marshaller(writer io.Writer, in chan *ntto.Triple, done chan bool, ignore *bool) {
	for triple := range in {
		b, err := json.Marshal(triple)
		if err != nil {
			if !*ignore {
				log.Fatalln(err)
			} else {
				log.Println(err)
			}
		}
		writer.Write(b)
		writer.Write([]byte("\n"))
	}
	done <- true
}

func main() {

	executive := "replace"
	_, err := exec.LookPath("replace")
	if err != nil {
		executive = "perl"
	}

	_, err = exec.LookPath("perl")
	if err != nil {
		log.Fatalln("This program requires perl or replace.")
		os.Exit(1)
	}

	abbreviate := flag.Bool("a", false, "abbreviate n-triples using rules")
	cpuprofile := flag.String("cpuprofile", "", "write cpu profile to file")
	dumpCommand := flag.Bool("c", false, "dump constructed sed command and exit")
	dumpRules := flag.Bool("d", false, "dump rules and exit")
	ignore := flag.Bool("i", false, "ignore conversion errors")
	jsonOutput := flag.Bool("j", false, "convert nt to json")
	nullValue := flag.String("n", "<NULL>", "string to indicate empty string replacement")
	outFile := flag.String("o", "", "output file to write result to")
	rulesFile := flag.String("r", "", "path to rules file, use built-in if none given")
	version := flag.Bool("v", false, "prints current version and exits")
	numWorkers := flag.Int("w", runtime.NumCPU(), "parallelism measure")

	flag.Parse()

	runtime.GOMAXPROCS(*numWorkers)

	var PrintUsage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] FILE\n", os.Args[0])
		flag.PrintDefaults()
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *version {
		fmt.Println(ntto.AppVersion)
		os.Exit(0)
	}

	var rules []ntto.Rule

	if *rulesFile == "" {
		rules, err = ntto.ParseRules(ntto.DefaultRules)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		b, err := ioutil.ReadFile(*rulesFile)
		if err != nil {
			log.Fatalln(err)
		}
		rules, err = ntto.ParseRules(string(b))
		if err != nil {
			log.Fatalln(err)
		}
	}

	if *dumpRules {
		fmt.Println(ntto.DumpRules(rules))
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		PrintUsage()
		os.Exit(1)
	}

	filename := flag.Args()[0]
	var output string

	if *abbreviate {
		if *outFile == "" {
			tmp, err := ioutil.TempFile("", "ntto-")
			output = tmp.Name()
			log.Printf("No explicit [-o]utput given, writing to %s\n", output)
			if err != nil {
				log.Fatalln(err)
			}
		} else {
			output = *outFile
		}

		var command string
		if executive == "perl" {
			command = fmt.Sprintf("%s > %s", ntto.SedifyNull(rules, *numWorkers, filename, *nullValue), output)
		} else {
			command = fmt.Sprintf("%s > %s", ntto.ReplacifyNull(rules, filename, *nullValue), output)
		}
		if *dumpCommand {
			fmt.Println(command)
			os.Exit(0)
		}
		_, err = exec.Command("sh", "-c", command).Output()
		if err != nil {
			log.Fatalln(err)
		}
		// set filename to abbreviated output, so we can use combine -j -a
		filename = output
	}

	if *jsonOutput {
		var file *os.File
		if filename == "-" {
			file = os.Stdin
		} else {
			file, err = os.Open(filename)
			defer file.Close()
			if err != nil {
				log.Fatalln(err)
			}
		}

		queue := make(chan *string)
		results := make(chan *ntto.Triple)
		done := make(chan bool)

		writer := bufio.NewWriter(os.Stdout)
		defer writer.Flush()
		go Marshaller(writer, results, done, ignore)

		var wg sync.WaitGroup
		for i := 0; i < *numWorkers; i++ {
			wg.Add(1)
			go Worker(queue, results, &wg, ignore)
		}

		reader := bufio.NewReader(file)

		for {
			b, _, err := reader.ReadLine()
			if err != nil || b == nil {
				break
			}
			line := string(b)
			queue <- &line
		}
		close(queue)
		wg.Wait()
		close(results)
		select {
		case <-time.After(1e9):
			break
		case <-done:
			break
		}
		// remove abbreviated tempfile output, if possible
		if *outFile == "" {
			_ = os.Remove(output)
		}
	}
}
<|endoftext|>