<commit_before>// Copyright (c) 2017 J. Hartsfield

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// Package inboxer is a Go library for checking email using the google Gmail
// API.
package inboxer

// SCOPE:
// TODO:
// channels and go routines
// Mark as read/unread/important/spam
// Get emails by label
// Get emails by query
//
// tests
// Watch inbox
// DOCS
// README.md
// how-to: add client credentials (for readme)
// Get Previews/snippet (put in docs)
//
// WORKS:
// Get emails by date
// Get emails by sender
// Get emails by recipient
// Get emails by subject
// Get emails by mailing-list
// Get emails by thread-topic
// Get labels
// Check for unread messages
// Convert date to human readable format
// Get Body
//
// DONE:
// LICENSE

import (
	"encoding/base64"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	gmail "google.golang.org/api/gmail/v1"
)

// GetBody gets, decodes, and returns the body of the email. It returns an
// error if decoding goes wrong. mimeType is used to indicate whether you want
// the plain text or html encoding ("text/html", "text/plain").
func GetBody(msg *gmail.Message, mimeType string) (string, error) {
	for _, v := range msg.Payload.Parts {
		if v.MimeType == "multipart/alternative" {
			for _, l := range v.Parts {
				if l.MimeType == mimeType && l.Body.Size >= 1 {
					dec, err := decodeEmailBody(l.Body.Data)
					if err != nil {
						return "", err
					}
					return dec, nil
				}
			}
		}
		if v.MimeType == mimeType && v.Body.Size >= 1 {
			dec, err := decodeEmailBody(v.Body.Data)
			if err != nil {
				return "", err
			}
			return dec, nil
		}
	}
	return "", errors.New("Couldn't Read Body")
}

// CheckForUnreadByLabel checks for unread mail maching the speciified label.
// NOTE: When checking your inbox for unread messages, it's not uncommon for
// it return thousands of unread messages that you don't know about. To see them
// in gmail, search your mail for "label:unread". For CheckForUnreadByLabel to
// work properly you need to mark all mail as read either through gmail or
// through the MarkAllAsRead() function found in this library.
func CheckForUnreadByLabel(srv *gmail.Service, label string) (int64, error) {
	inbox, err := srv.Users.Labels.Get("me", label).Do()
	if err != nil {
		return -1, err
	}

	// fmt.Println(label.MessagesUnread)
	if inbox.MessagesUnread == 0 && inbox.ThreadsUnread == 0 {
		return 0, nil
	}

	return inbox.MessagesUnread + inbox.ThreadsUnread, nil
}

// CheckForUnread checks for mail labeled "UNREAD".
// NOTE: When checking your inbox for unread messages, it's not uncommon for
// it return thousands of unread messages that you don't know about. To see them
// in gmail, search your mail for "label:unread". For CheckForUnread to
// work properly you need to mark all mail as read either through gmail or
// through the MarkAllAsRead() function found in this library.
func CheckForUnread(srv *gmail.Service) (int64, error) {
	inbox, err := srv.Users.Labels.Get("me", "UNREAD").Do()
	if err != nil {
		return -1, err
	}

	// fmt.Println(label.MessagesUnread)
	if inbox.MessagesUnread == 0 && inbox.ThreadsUnread == 0 {
		return 0, nil
	}

	return inbox.MessagesUnread + inbox.ThreadsUnread, nil
}

// GetLabels gets a list of the labels used in the users inbox.
func GetLabels(srv *gmail.Service) (*gmail.ListLabelsResponse, error) {
	return srv.Users.Labels.List("me").Do()
}

// HasLabel takes a label and an email and checks if that email has that label
func HasLabel(label string, msg *gmail.Message) bool {
	for _, v := range msg.LabelIds {
		if v == strings.ToUpper(label) {
			return true
		}
	}
	return false
}

// PartialMetadata stores email metadata
type PartialMetadata struct {
	Sender, From, To, CC, Subject, MailingList, DeliveredTo, ThreadTopic []string
}

// GetByDate gets and returns emails within the time frame specified.
func GetByDate(srv *gmail.Service /*start time.Time, end time.Time*/) []*gmail.Message {
	inbox, err := srv.Users.Messages.List("me").Q("in:inbox after:2017/01/01 before:2017/01/30").Do()
	if err != nil {
		fmt.Println(err)
	}
	return getById(srv, inbox)
}

// GetPartialMetadata gets some of the useful metadata from the headers.
func GetPartialMetadata(msg *gmail.Message) *PartialMetadata {
	info := &PartialMetadata{}
	fmt.Println("========================================================")
	fmt.Println(msg.Snippet)
	for _, v := range msg.Payload.Headers {
		switch v.Name {
		case "Sender":
			info.Sender = append(info.Sender, v.Value)
		case "From":
			info.From = append(info.From, v.Value)
		case "To":
			info.To = append(info.To, v.Value)
		case "CC":
			info.CC = append(info.CC, v.Value)
		case "Subject":
			info.Subject = append(info.Subject, v.Value)
		case "Mailing-list":
			info.MailingList = append(info.MailingList, v.Value)
		case "Delivered-To":
			info.DeliveredTo = append(info.DeliveredTo, v.Value)
		case "Thread-Topic":
			info.ThreadTopic = append(info.ThreadTopic, v.Value)
		}
	}
	return info
}

// decodeEmailBody is used to decode the email body by converting from
// URLEncoded base64 to a string.
func decodeEmailBody(data string) (string, error) {
	decoded, err := base64.URLEncoding.DecodeString(data)
	if err != nil {
		fmt.Println("decode error:", err)
		return "", err
	}
	return string(decoded), nil
}

// ReceivedTime converts parses and converts a unix time stamp into a human
// readable format ().
func ReceivedTime(datetime int64) time.Time {
	conv := strconv.FormatInt(datetime, 10)
	// Remove trailing zeros.
	conv = conv[:len(conv)-3]
	tc, err := strconv.ParseInt(conv, 10, 64)
	if err != nil {
		fmt.Println(err)
	}
	return time.Unix(tc, 0)
}

func getById(srv *gmail.Service, msgs *gmail.ListMessagesResponse) []*gmail.Message {
	var msgSlice []*gmail.Message
	for _, v := range msgs.Messages {
		msg, _ := srv.Users.Messages.Get("me", v.Id).Do()
		msgSlice = append(msgSlice, msg)
	}
	return msgSlice
}

// GetMessages gets and returns gmail messages
func GetMessages(srv *gmail.Service, howMany uint) ([]*gmail.Message, error) {
	var msgSlice []*gmail.Message

	// Get the messages
	msgs, err := srv.Users.Messages.List("me").MaxResults(int64(howMany)).Do()
	if err != nil {
		return msgSlice, err
	}

	return getById(srv, msgs), nil
}

// func watchInbox() {
// 	req := &gmail.WatchRequest{
// 		LabelFilterAction: "include",
// 		LabelIds:          []string{"UNREAD"},
// 		TopicName:         "gmailmsg",
// 	}

// 	wr, _ := srv.Users.Watch("me", req).Do()
// 	fmt.Println(wr.ForceSendFields)
// }
<commit_msg>changing types and adding comments to PartialMetadata struct<commit_after>// Copyright (c) 2017 J. Hartsfield

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// Package inboxer is a Go library for checking email using the google Gmail
// API.
package inboxer

// SCOPE:
// TODO:
// channels and go routines
// Mark as read/unread/important/spam
// Get emails by label
// Get emails by query
//
// tests
// Watch inbox
// DOCS
// README.md
// how-to: add client credentials (for readme)
// Get Previews/snippet (put in docs)
//
// WORKS:
// Get emails by date
// Get emails by sender
// Get emails by recipient
// Get emails by subject
// Get emails by mailing-list
// Get emails by thread-topic
// Get labels
// Check for unread messages
// Convert date to human readable format
// Get Body
//
// DONE:
// LICENSE

import (
	"encoding/base64"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	gmail "google.golang.org/api/gmail/v1"
)

// GetBody gets, decodes, and returns the body of the email. It returns an
// error if decoding goes wrong. mimeType is used to indicate whether you want
// the plain text or html encoding ("text/html", "text/plain").
func GetBody(msg *gmail.Message, mimeType string) (string, error) {
	for _, v := range msg.Payload.Parts {
		if v.MimeType == "multipart/alternative" {
			for _, l := range v.Parts {
				if l.MimeType == mimeType && l.Body.Size >= 1 {
					dec, err := decodeEmailBody(l.Body.Data)
					if err != nil {
						return "", err
					}
					return dec, nil
				}
			}
		}
		if v.MimeType == mimeType && v.Body.Size >= 1 {
			dec, err := decodeEmailBody(v.Body.Data)
			if err != nil {
				return "", err
			}
			return dec, nil
		}
	}
	return "", errors.New("Couldn't Read Body")
}

// CheckForUnreadByLabel checks for unread mail maching the speciified label.
// NOTE: When checking your inbox for unread messages, it's not uncommon for
// it return thousands of unread messages that you don't know about. To see them
// in gmail, search your mail for "label:unread". For CheckForUnreadByLabel to
// work properly you need to mark all mail as read either through gmail or
// through the MarkAllAsRead() function found in this library.
func CheckForUnreadByLabel(srv *gmail.Service, label string) (int64, error) {
	inbox, err := srv.Users.Labels.Get("me", label).Do()
	if err != nil {
		return -1, err
	}

	// fmt.Println(label.MessagesUnread)
	if inbox.MessagesUnread == 0 && inbox.ThreadsUnread == 0 {
		return 0, nil
	}

	return inbox.MessagesUnread + inbox.ThreadsUnread, nil
}

// CheckForUnread checks for mail labeled "UNREAD".
// NOTE: When checking your inbox for unread messages, it's not uncommon for
// it return thousands of unread messages that you don't know about. To see them
// in gmail, search your mail for "label:unread". For CheckForUnread to
// work properly you need to mark all mail as read either through gmail or
// through the MarkAllAsRead() function found in this library.
func CheckForUnread(srv *gmail.Service) (int64, error) {
	inbox, err := srv.Users.Labels.Get("me", "UNREAD").Do()
	if err != nil {
		return -1, err
	}

	// fmt.Println(label.MessagesUnread)
	if inbox.MessagesUnread == 0 && inbox.ThreadsUnread == 0 {
		return 0, nil
	}

	return inbox.MessagesUnread + inbox.ThreadsUnread, nil
}

// GetLabels gets a list of the labels used in the users inbox.
func GetLabels(srv *gmail.Service) (*gmail.ListLabelsResponse, error) {
	return srv.Users.Labels.List("me").Do()
}

// HasLabel takes a label and an email and checks if that email has that label
func HasLabel(label string, msg *gmail.Message) bool {
	for _, v := range msg.LabelIds {
		if v == strings.ToUpper(label) {
			return true
		}
	}
	return false
}

// Query queries the inbox for a string following the search style of the gmail
// online mailbox.
// example:
// "in:sent after:2017/01/01 before:2017/01/30"
func Query(srv *gmail.Service, query string) []*gmail.Message {
	inbox, err := srv.Users.Messages.List("me").Q(query).Do()
	if err != nil {
		fmt.Println(err)
	}
	return getById(srv, inbox)
}

// PartialMetadata stores email metadata. Some fields may sound redundant, but
// infact have different contexts. Some are slices of string because the ones
// that have multiple values are still being sorted from those that don't.
type PartialMetadata struct {
	// Sender is the entity that originally created and sent the message
	Sender string
	// From is the entitiy that sent the message to you (e.g. googlegroups). Most
	// of the time this information is only relevant to mailing lists.
	From string
	// Subject is the email subject
	Subject string
	// Mailing list contains the name of the mailing list that the email was
	// posted to, if any.
	MailingList string
	// CC is the "carbon copy" list of addresses
	CC []string
	// To is the recipient of the email.
	To []string
	// ThreadTopic contains the topic of the thread (e.g. google groups threads)
	ThreadTopic []string
	// DeliveredTo is who the email was sent to. This can contain multiple
	// addresses if the email was forwarded.
	DeliveredTo []string
}

// GetPartialMetadata gets some of the useful metadata from the headers.
func GetPartialMetadata(msg *gmail.Message) *PartialMetadata {
	info := &PartialMetadata{}
	fmt.Println("========================================================")
	fmt.Println(msg.Snippet)
	for _, v := range msg.Payload.Headers {
		switch v.Name {
		case "Sender":
			info.Sender = append(info.Sender, v.Value)
		case "From":
			info.From = append(info.From, v.Value)
		case "To":
			info.To = append(info.To, v.Value)
		case "CC":
			info.CC = append(info.CC, v.Value)
		case "Subject":
			info.Subject = append(info.Subject, v.Value)
		case "Mailing-list":
			info.MailingList = append(info.MailingList, v.Value)
		case "Delivered-To":
			info.DeliveredTo = append(info.DeliveredTo, v.Value)
		case "Thread-Topic":
			info.ThreadTopic = append(info.ThreadTopic, v.Value)
		}
	}
	return info
}

// decodeEmailBody is used to decode the email body by converting from
// URLEncoded base64 to a string.
func decodeEmailBody(data string) (string, error) {
	decoded, err := base64.URLEncoding.DecodeString(data)
	if err != nil {
		fmt.Println("decode error:", err)
		return "", err
	}
	return string(decoded), nil
}

// ReceivedTime converts parses and converts a unix time stamp into a human
// readable format ().
func ReceivedTime(datetime int64) time.Time {
	conv := strconv.FormatInt(datetime, 10)
	// Remove trailing zeros.
	conv = conv[:len(conv)-3]
	tc, err := strconv.ParseInt(conv, 10, 64)
	if err != nil {
		fmt.Println(err)
	}
	return time.Unix(tc, 0)
}

func getById(srv *gmail.Service, msgs *gmail.ListMessagesResponse) []*gmail.Message {
	var msgSlice []*gmail.Message
	for _, v := range msgs.Messages {
		msg, _ := srv.Users.Messages.Get("me", v.Id).Do()
		msgSlice = append(msgSlice, msg)
	}
	return msgSlice
}

// GetMessages gets and returns gmail messages
func GetMessages(srv *gmail.Service, howMany uint) ([]*gmail.Message, error) {
	var msgSlice []*gmail.Message

	// Get the messages
	msgs, err := srv.Users.Messages.List("me").MaxResults(int64(howMany)).Do()
	if err != nil {
		return msgSlice, err
	}

	return getById(srv, msgs), nil
}

// func watchInbox() {
// 	req := &gmail.WatchRequest{
// 		LabelFilterAction: "include",
// 		LabelIds:          []string{"UNREAD"},
// 		TopicName:         "gmailmsg",
// 	}

// 	wr, _ := srv.Users.Watch("me", req).Do()
// 	fmt.Println(wr.ForceSendFields)
// }
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package repository

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	"fmt"
	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
	testlib "github.com/uber/zanzibar/test/lib"
)

const endpointUpdateRequestDir = "../../examples/example-gateway/endpoints"

func TestUpdateEndpoint(t *testing.T) {
	tempDir, err := copyExample("")
	t.Logf("Temp dir is created at %s\n", tempDir)
	if !assert.NoError(t, err, "Failed to copy example.") {
		return
	}
	requestFiles := []string{
		"googlenow/add_credentials.json",
		"googlenow/check_credentials.json",
		"bar/arg_not_struct.json",
		"baz/call.json",
		"baz/compare.json",
		"bar/normal.json",
	}
	for _, file := range requestFiles {
		t.Logf("Test request in %q\n", file)
		testUpdateEndpointConfig(t, tempDir, filepath.Join(endpointUpdateRequestDir, file))
	}
}

func testUpdateEndpointConfig(t *testing.T, tempDir string, requestFile string) {
	req := &EndpointConfig{}
	err := readJSONFile(requestFile, req)
	assert.NoError(t, err, "Failed to unmarshal endpoint config.")
	r := &Repository{
		localDir: tempDir,
	}
	endpointCfgDir := "endpoints"
	err = r.WriteEndpointConfig(endpointCfgDir, req, "{{placeholder}}")
	if !assert.NoError(t, err, "Failed to write endpoint config.") {
		return
	}
	jsonFile := codegen.CamelToSnake(req.HandleID) + ".json"
	endpointConfigActFile := filepath.Join(tempDir, endpointCfgDir, req.ID, jsonFile)
	actualendpointCfg, err := ioutil.ReadFile(endpointConfigActFile)
	if !assert.NoError(t, err, "Failed to read endpoint config file.") {
		return
	}
	endpointConfigExpFile := filepath.Join(exampleGateway, endpointCfgDir, req.ID, jsonFile)
	testlib.CompareGoldenFile(t, endpointConfigExpFile, actualendpointCfg)

	endpointGroupCfg, err := ioutil.ReadFile(filepath.Join(tempDir, endpointCfgDir, req.ID, endpointConfigFileName))
	if !assert.NoError(t, err, "Failed to read endpoint module config file.") {
		return
	}
	endpointGroupExpFile := filepath.Join(exampleGateway, endpointCfgDir, req.ID, endpointConfigFileName)
	testlib.CompareGoldenFile(t, endpointGroupExpFile, endpointGroupCfg)

	serviceGroupCfg, err := ioutil.ReadFile(filepath.Join(exampleGateway, endpointCfgDir, "../services", "example-gateway", serviceConfigFileName))
	if !assert.NoError(t, err, "Failed to read endpoint module config file.") {
		return
	}
	serviceGroupExpFile := filepath.Join(exampleGateway, endpointCfgDir, "../services", "example-gateway", serviceConfigFileName)
	testlib.CompareGoldenFile(t, serviceGroupExpFile, serviceGroupCfg)
}

func TestUpdateEndpointBadMiddlewareConfig(t *testing.T) {
	tempDir, err := copyExample("")
	t.Logf("Temp dir is created at %s\n", tempDir)
	if !assert.NoError(t, err, "Failed to copy example.") {
		return
	}
	requestFiles := []string{
		"bar/normal.json",
	}
	for _, file := range requestFiles {
		t.Logf("Test request in %q\n", file)
		testUpdateEndpointConfig(t, tempDir, filepath.Join(endpointUpdateRequestDir, file))
	}
}
<commit_msg>update service config<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package repository

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/uber/zanzibar/codegen"
	testlib "github.com/uber/zanzibar/test/lib"
)

const endpointUpdateRequestDir = "../../examples/example-gateway/endpoints"

func TestUpdateEndpoint(t *testing.T) {
	tempDir, err := copyExample("")
	t.Logf("Temp dir is created at %s\n", tempDir)
	if !assert.NoError(t, err, "Failed to copy example.") {
		return
	}
	requestFiles := []string{
		"googlenow/add_credentials.json",
		"googlenow/check_credentials.json",
		"bar/arg_not_struct.json",
		"baz/call.json",
		"baz/compare.json",
		"bar/normal.json",
	}
	for _, file := range requestFiles {
		t.Logf("Test request in %q\n", file)
		testUpdateEndpointConfig(t, tempDir, filepath.Join(endpointUpdateRequestDir, file))
	}
}

func testUpdateEndpointConfig(t *testing.T, tempDir string, requestFile string) {
	req := &EndpointConfig{}
	err := readJSONFile(requestFile, req)
	assert.NoError(t, err, "Failed to unmarshal endpoint config.")
	r := &Repository{
		localDir: tempDir,
	}
	endpointCfgDir := "endpoints"
	err = r.WriteEndpointConfig(endpointCfgDir, req, "{{placeholder}}")
	if !assert.NoError(t, err, "Failed to write endpoint config.") {
		return
	}
	jsonFile := codegen.CamelToSnake(req.HandleID) + ".json"
	endpointConfigActFile := filepath.Join(tempDir, endpointCfgDir, req.ID, jsonFile)
	actualendpointCfg, err := ioutil.ReadFile(endpointConfigActFile)
	if !assert.NoError(t, err, "Failed to read endpoint config file.") {
		return
	}
	endpointConfigExpFile := filepath.Join(exampleGateway, endpointCfgDir, req.ID, jsonFile)
	testlib.CompareGoldenFile(t, endpointConfigExpFile, actualendpointCfg)

	endpointGroupCfg, err := ioutil.ReadFile(filepath.Join(tempDir, endpointCfgDir, req.ID, endpointConfigFileName))
	if !assert.NoError(t, err, "Failed to read endpoint module config file.") {
		return
	}
	endpointGroupExpFile := filepath.Join(exampleGateway, endpointCfgDir, req.ID, endpointConfigFileName)
	testlib.CompareGoldenFile(t, endpointGroupExpFile, endpointGroupCfg)

	serviceGroupCfg, err := ioutil.ReadFile(filepath.Join(exampleGateway, endpointCfgDir, "../services", "example-gateway", serviceConfigFileName))
	if !assert.NoError(t, err, "Failed to read endpoint module config file.") {
		return
	}
	serviceGroupExpFile := filepath.Join(exampleGateway, endpointCfgDir, "../services", "example-gateway", serviceConfigFileName)
	testlib.CompareGoldenFile(t, serviceGroupExpFile, serviceGroupCfg)
}

func TestUpdateEndpointBadMiddlewareConfig(t *testing.T) {
	tempDir, err := copyExample("")
	t.Logf("Temp dir is created at %s\n", tempDir)
	if !assert.NoError(t, err, "Failed to copy example.") {
		return
	}
	requestFiles := []string{
		"bar/normal.json",
	}
	for _, file := range requestFiles {
		t.Logf("Test request in %q\n", file)
		testUpdateEndpointConfig(t, tempDir, filepath.Join(endpointUpdateRequestDir, file))
	}
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/monitor"
	"github.com/livepeer/lpms/ffmpeg"
	"github.com/livepeer/lpms/transcoder"
)

type Transcoder interface {
	Transcode(fname string, profiles []ffmpeg.VideoProfile) ([][]byte, error)
}

type LocalTranscoder struct {
	workDir string
}

func (lt *LocalTranscoder) Transcode(fname string, profiles []ffmpeg.VideoProfile) ([][]byte, error) {
	tr := transcoder.NewFFMpegSegmentTranscoder(profiles, lt.workDir)
	mid, seqNo, parseErr := parseURI(fname)
	start := time.Now()
	if monitor.Enabled && parseErr == nil {
		monitor.LogSegmentTranscodeStarting(seqNo, mid)
	}
	data, err := tr.Transcode(fname)
	if monitor.Enabled && parseErr == nil {
		monitor.LogSegmentTranscodeEnded(seqNo, mid, time.Since(start), common.ProfilesNames(profiles))
	}
	return data, err
}

func NewLocalTranscoder(workDir string) Transcoder {
	return &LocalTranscoder{workDir: workDir}
}

func parseURI(uri string) (string, uint64, error) {
	var mid string
	var seqNo uint64
	parts := strings.Split(uri, "/")
	if len(parts) < 3 {
		return mid, seqNo, fmt.Errorf("BadURI")
	}
	mid = parts[len(parts)-2]
	parts = strings.Split(parts[len(parts)-1], ".")
	seqNo, err := strconv.ParseUint(parts[0], 10, 64)
	return mid, seqNo, err
}
<commit_msg>On local transcode, create unique dir for output segments<commit_after>package core

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/livepeer/go-livepeer/common"
	"github.com/livepeer/go-livepeer/monitor"
	"github.com/livepeer/lpms/ffmpeg"
	"github.com/livepeer/lpms/transcoder"
)

type Transcoder interface {
	Transcode(fname string, profiles []ffmpeg.VideoProfile) ([][]byte, error)
}

type LocalTranscoder struct {
	workDir string
}

func (lt *LocalTranscoder) Transcode(fname string, profiles []ffmpeg.VideoProfile) ([][]byte, error) {
	dirName := randName()
	fullDirName := filepath.Join(lt.workDir, dirName)
	err := os.MkdirAll(fullDirName, 0755)
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(fullDirName)
	tr := transcoder.NewFFMpegSegmentTranscoder(profiles, fullDirName)
	mid, seqNo, parseErr := parseURI(fname)
	start := time.Now()
	if monitor.Enabled && parseErr == nil {
		monitor.LogSegmentTranscodeStarting(seqNo, mid)
	}
	data, err := tr.Transcode(fname)
	if monitor.Enabled && parseErr == nil {
		monitor.LogSegmentTranscodeEnded(seqNo, mid, time.Since(start), common.ProfilesNames(profiles))
	}
	return data, err
}

func NewLocalTranscoder(workDir string) Transcoder {
	return &LocalTranscoder{workDir: workDir}
}

func parseURI(uri string) (string, uint64, error) {
	var mid string
	var seqNo uint64
	parts := strings.Split(uri, "/")
	if len(parts) < 3 {
		return mid, seqNo, fmt.Errorf("BadURI")
	}
	mid = parts[len(parts)-2]
	parts = strings.Split(parts[len(parts)-1], ".")
	seqNo, err := strconv.ParseUint(parts[0], 10, 64)
	return mid, seqNo, err
}
<|endoftext|>


<commit_before>package stripe

// TerminalConnectionTokenParams is the set of parameters that can be used when creating a terminal connection token.
type TerminalConnectionTokenParams struct {
	Params          `form:"*"`
	OperatorAccount *string `form:"operator_account"`
}

// TerminalConnectionToken is the resource representing a Stripe terminal connection token.
type TerminalConnectionToken struct {
	Object string `json:"object"`
	Secret string `json:"connection_token"`
}
<commit_msg>ConnectionToken json -> secret<commit_after>package stripe

// TerminalConnectionTokenParams is the set of parameters that can be used when creating a terminal connection token.
type TerminalConnectionTokenParams struct {
	Params          `form:"*"`
	OperatorAccount *string `form:"operator_account"`
}

// TerminalConnectionToken is the resource representing a Stripe terminal connection token.
type TerminalConnectionToken struct {
	Object string `json:"object"`
	Secret string `json:"secret"`
}
<|endoftext|>


<commit_before>package rardecode

import (
	"bytes"
	"encoding/binary"
	"hash/crc32"
	"io"
)

const (
	fileSize = 0x1000000

	vmGlobalAddr      = 0x3C000
	vmGlobalSize      = 0x02000
	vmFixedGlobalSize = 0x40
)

// v3Filter is the interface type for RAR V3 filters.
// v3Filter performs the same function as the filter type, except that it also takes
// the initial register values r, and global data as input for the RAR V3 VM.
type v3Filter func(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error)

var (
	// standardV3Filters is a list of known filters. We can replace the use of a vm
	// filter with a custom filter function.
	standardV3Filters = []struct {
		crc uint32   // crc of code byte slice for filter
		len int      // length of code byte slice for filter
		f   v3Filter // replacement filter function
	}{
		{0xad576887, 53, e8FilterV3},
		{0x3cd7e57e, 57, e8e9FilterV3},
		{0x0e06077d, 29, deltaFilterV3},
	}
)

func filterE8(c byte, v5 bool, buf []byte, offset int64) ([]byte, error) {
	off := int32(offset)
	for b := buf; len(b) >= 5; {
		ch := b[0]
		b = b[1:]
		off++
		if ch != 0xe8 && ch != c {
			continue
		}
		if v5 {
			off %= fileSize
		}
		addr := int32(binary.LittleEndian.Uint32(b))
		if addr < 0 {
			if addr+off >= 0 {
				binary.LittleEndian.PutUint32(b, uint32(addr+fileSize))
			}
		} else if addr < fileSize {
			binary.LittleEndian.PutUint32(b, uint32(addr-off))
		}
		off += 4
		b = b[4:]
	}
	return buf, nil
}

func e8FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterE8(0xe8, false, buf, offset)
}

func e8e9FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterE8(0xe9, false, buf, offset)
}

func filterDelta(n int, buf []byte) ([]byte, error) {
	var res []byte
	l := len(buf)
	if cap(buf) >= 2*l {
		res = buf[l : 2*l] // use unused capacity
	} else {
		res = make([]byte, l, 2*l)
	}

	i := 0
	for j := 0; j < n; j++ {
		var c byte
		for k := j; k < len(res); k += n {
			c -= buf[i]
			i++
			res[k] = c
		}
	}
	return res, nil
}

func deltaFilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterDelta(int(r[0]), buf)
}

func filterArm(buf []byte, offset int64) ([]byte, error) {
	for i := 0; len(buf)-i > 3; i += 4 {
		if buf[i+3] == 0xeb {
			n := uint(buf[i])
			n += uint(buf[i+1]) * 0x100
			n += uint(buf[i+2]) * 0x10000
			n -= (n + uint(i)) / 4
			buf[i] = byte(n)
			buf[i+1] = byte(n >> 8)
			buf[i+2] = byte(n >> 16)
		}
	}
	return buf, nil
}

type vmFilter struct {
	execCount uint32
	global    []byte
	static    []byte
	code      []command
}

// execute implements v3filter type for VM based RAR 3 filters.
func (f *vmFilter) execute(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	if len(buf) > vmGlobalAddr {
		return buf, errInvalidFilter
	}
	v := newVM(buf)

	// register setup
	v.r[3] = vmGlobalAddr
	v.r[4] = uint32(len(buf))
	v.r[5] = f.execCount
	for i, n := range r {
		v.r[i] = n
	}

	// vm global data memory block
	vg := v.m[vmGlobalAddr : vmGlobalAddr+vmGlobalSize]

	// initialize fixed global memory
	for i, n := range v.r[:vmRegs-1] {
		binary.LittleEndian.PutUint32(vg[i*4:], n)
	}
	binary.LittleEndian.PutUint32(vg[0x1c:], uint32(len(buf)))
	binary.LittleEndian.PutUint64(vg[0x24:], uint64(offset))
	binary.LittleEndian.PutUint32(vg[0x2c:], f.execCount)

	// registers
	v.r[6] = uint32(offset)

	// copy program global memory
	var n int
	if len(f.global) > 0 {
		n = copy(vg[vmFixedGlobalSize:], f.global) // use saved global instead
	} else {
		n = copy(vg[vmFixedGlobalSize:], global)
	}
	copy(vg[vmFixedGlobalSize+n:], f.static)

	v.execute(f.code)

	f.execCount++

	// keep largest global buffer
	if cap(global) > cap(f.global) {
		f.global = global[:0]
	} else if len(f.global) > 0 {
		f.global = f.global[:0]
	}

	// check for global data to be saved for next program execution
	globalSize := binary.LittleEndian.Uint32(vg[0x30:])
	if globalSize > 0 {
		if globalSize > vmGlobalSize-vmFixedGlobalSize {
			globalSize = vmGlobalSize - vmFixedGlobalSize
		}
		if cap(f.global) < int(globalSize) {
			f.global = make([]byte, globalSize)
		} else {
			f.global = f.global[:globalSize]
		}
		copy(f.global, vg[vmFixedGlobalSize:])
	}

	// find program output
	length := binary.LittleEndian.Uint32(vg[0x1c:]) & vmMask
	start := binary.LittleEndian.Uint32(vg[0x20:]) & vmMask
	if start+length > vmSize {
		// TODO: error
		start = 0
		length = 0
	}
	if start != 0 && cap(v.m) > cap(buf) {
		// Initial buffer was to small for vm.
		// Copy output to beginning of vm memory so that decodeReader
		// will re-use the newly allocated vm memory and we will not
		// have to reallocate again next time.
		copy(v.m, v.m[start:start+length])
		start = 0
	}
	return v.m[start : start+length], nil
}

// getV3Filter returns a V3 filter function from a code byte slice.
func getV3Filter(code []byte) (v3Filter, error) {
	// check if filter is a known standard filter
	c := crc32.ChecksumIEEE(code)
	for _, f := range standardV3Filters {
		if f.crc == c && f.len == len(code) {
			return f.f, nil
		}
	}

	// create new vm filter
	f := new(vmFilter)
	r := newRarBitReader(bytes.NewReader(code[1:])) // skip first xor byte check

	// read static data
	n, err := r.readBits(1)
	if err != nil {
		return nil, err
	}
	if n > 0 {
		m, err := r.readUint32()
		if err != nil {
			return nil, err
		}
		f.static = make([]byte, m+1)
		err = r.readFull(f.static)
		if err != nil {
			return nil, err
		}
	}

	f.code, err = readCommands(r)
	if err == io.EOF {
		err = nil
	}

	return f.execute, err
}
<commit_msg>Fix V5 Arm filter calculation.<commit_after>package rardecode

import (
	"bytes"
	"encoding/binary"
	"hash/crc32"
	"io"
)

const (
	fileSize = 0x1000000

	vmGlobalAddr      = 0x3C000
	vmGlobalSize      = 0x02000
	vmFixedGlobalSize = 0x40
)

// v3Filter is the interface type for RAR V3 filters.
// v3Filter performs the same function as the filter type, except that it also takes
// the initial register values r, and global data as input for the RAR V3 VM.
type v3Filter func(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error)

var (
	// standardV3Filters is a list of known filters. We can replace the use of a vm
	// filter with a custom filter function.
	standardV3Filters = []struct {
		crc uint32   // crc of code byte slice for filter
		len int      // length of code byte slice for filter
		f   v3Filter // replacement filter function
	}{
		{0xad576887, 53, e8FilterV3},
		{0x3cd7e57e, 57, e8e9FilterV3},
		{0x0e06077d, 29, deltaFilterV3},
	}
)

func filterE8(c byte, v5 bool, buf []byte, offset int64) ([]byte, error) {
	off := int32(offset)
	for b := buf; len(b) >= 5; {
		ch := b[0]
		b = b[1:]
		off++
		if ch != 0xe8 && ch != c {
			continue
		}
		if v5 {
			off %= fileSize
		}
		addr := int32(binary.LittleEndian.Uint32(b))
		if addr < 0 {
			if addr+off >= 0 {
				binary.LittleEndian.PutUint32(b, uint32(addr+fileSize))
			}
		} else if addr < fileSize {
			binary.LittleEndian.PutUint32(b, uint32(addr-off))
		}
		off += 4
		b = b[4:]
	}
	return buf, nil
}

func e8FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterE8(0xe8, false, buf, offset)
}

func e8e9FilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterE8(0xe9, false, buf, offset)
}

func filterDelta(n int, buf []byte) ([]byte, error) {
	var res []byte
	l := len(buf)
	if cap(buf) >= 2*l {
		res = buf[l : 2*l] // use unused capacity
	} else {
		res = make([]byte, l, 2*l)
	}

	i := 0
	for j := 0; j < n; j++ {
		var c byte
		for k := j; k < len(res); k += n {
			c -= buf[i]
			i++
			res[k] = c
		}
	}
	return res, nil
}

func deltaFilterV3(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	return filterDelta(int(r[0]), buf)
}

func filterArm(buf []byte, offset int64) ([]byte, error) {
	for i := 0; len(buf)-i > 3; i += 4 {
		if buf[i+3] == 0xeb {
			n := uint(buf[i])
			n += uint(buf[i+1]) * 0x100
			n += uint(buf[i+2]) * 0x10000
			n -= (uint(offset) + uint(i)) / 4
			buf[i] = byte(n)
			buf[i+1] = byte(n >> 8)
			buf[i+2] = byte(n >> 16)
		}
	}
	return buf, nil
}

type vmFilter struct {
	execCount uint32
	global    []byte
	static    []byte
	code      []command
}

// execute implements v3filter type for VM based RAR 3 filters.
func (f *vmFilter) execute(r map[int]uint32, global, buf []byte, offset int64) ([]byte, error) {
	if len(buf) > vmGlobalAddr {
		return buf, errInvalidFilter
	}
	v := newVM(buf)

	// register setup
	v.r[3] = vmGlobalAddr
	v.r[4] = uint32(len(buf))
	v.r[5] = f.execCount
	for i, n := range r {
		v.r[i] = n
	}

	// vm global data memory block
	vg := v.m[vmGlobalAddr : vmGlobalAddr+vmGlobalSize]

	// initialize fixed global memory
	for i, n := range v.r[:vmRegs-1] {
		binary.LittleEndian.PutUint32(vg[i*4:], n)
	}
	binary.LittleEndian.PutUint32(vg[0x1c:], uint32(len(buf)))
	binary.LittleEndian.PutUint64(vg[0x24:], uint64(offset))
	binary.LittleEndian.PutUint32(vg[0x2c:], f.execCount)

	// registers
	v.r[6] = uint32(offset)

	// copy program global memory
	var n int
	if len(f.global) > 0 {
		n = copy(vg[vmFixedGlobalSize:], f.global) // use saved global instead
	} else {
		n = copy(vg[vmFixedGlobalSize:], global)
	}
	copy(vg[vmFixedGlobalSize+n:], f.static)

	v.execute(f.code)

	f.execCount++

	// keep largest global buffer
	if cap(global) > cap(f.global) {
		f.global = global[:0]
	} else if len(f.global) > 0 {
		f.global = f.global[:0]
	}

	// check for global data to be saved for next program execution
	globalSize := binary.LittleEndian.Uint32(vg[0x30:])
	if globalSize > 0 {
		if globalSize > vmGlobalSize-vmFixedGlobalSize {
			globalSize = vmGlobalSize - vmFixedGlobalSize
		}
		if cap(f.global) < int(globalSize) {
			f.global = make([]byte, globalSize)
		} else {
			f.global = f.global[:globalSize]
		}
		copy(f.global, vg[vmFixedGlobalSize:])
	}

	// find program output
	length := binary.LittleEndian.Uint32(vg[0x1c:]) & vmMask
	start := binary.LittleEndian.Uint32(vg[0x20:]) & vmMask
	if start+length > vmSize {
		// TODO: error
		start = 0
		length = 0
	}
	if start != 0 && cap(v.m) > cap(buf) {
		// Initial buffer was to small for vm.
		// Copy output to beginning of vm memory so that decodeReader
		// will re-use the newly allocated vm memory and we will not
		// have to reallocate again next time.
		copy(v.m, v.m[start:start+length])
		start = 0
	}
	return v.m[start : start+length], nil
}

// getV3Filter returns a V3 filter function from a code byte slice.
func getV3Filter(code []byte) (v3Filter, error) {
	// check if filter is a known standard filter
	c := crc32.ChecksumIEEE(code)
	for _, f := range standardV3Filters {
		if f.crc == c && f.len == len(code) {
			return f.f, nil
		}
	}

	// create new vm filter
	f := new(vmFilter)
	r := newRarBitReader(bytes.NewReader(code[1:])) // skip first xor byte check

	// read static data
	n, err := r.readBits(1)
	if err != nil {
		return nil, err
	}
	if n > 0 {
		m, err := r.readUint32()
		if err != nil {
			return nil, err
		}
		f.static = make([]byte, m+1)
		err = r.readFull(f.static)
		if err != nil {
			return nil, err
		}
	}

	f.code, err = readCommands(r)
	if err == io.EOF {
		err = nil
	}

	return f.execute, err
}
<|endoftext|>


<commit_before>package crawler

import (
	"errors"
	"fmt"
	machinery "github.com/RichardKnop/machinery/v1"
	signatures "github.com/RichardKnop/machinery/v1/signatures"
	"github.com/dokterbob/ipfs-search/indexer"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"net/http"
)

type Crawler struct {
	sh  *shell.Shell
	id  *indexer.Indexer
	mac *machinery.Server
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, mac *machinery.Server) *Crawler {
	c := new(Crawler)
	c.sh = sh
	c.id = id
	c.mac = mac
	return c
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Given a particular hash, start crawling
func (c Crawler) CrawlHash(hash string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		fmt.Printf("Already indexed '%s', skipping\n", hash)
		return nil
	}

	fmt.Printf("Crawling hash '%s'\n", hash)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		task := signatures.TaskSignature{
			Name: "crawl_file",
			Args: []signatures.TaskArg{
				signatures.TaskArg{
					Type:  "string",
					Value: hash,
				},
				signatures.TaskArg{
					Type:  "string",
					Value: nil,
				},
			},
		}
		_, err := c.mac.SendTask(&task)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for items
		properties := map[string]interface{}{
			"links": list.Links,
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			switch link.Type {
			case "File":
				// Add file to crawl queue
				task := signatures.TaskSignature{
					Name: "crawl_file",
					Args: []signatures.TaskArg{
						signatures.TaskArg{
							Type:  "string",
							Value: link.Hash,
						},
					},
				}
				_, err := c.mac.SendTask(&task)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				task := signatures.TaskSignature{
					Name: "crawl_hash",
					Args: []signatures.TaskArg{
						signatures.TaskArg{
							Type:  "string",
							Value: link.Hash,
						},
					},
				}
				_, err := c.mac.SendTask(&task)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				fmt.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		fmt.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	return nil
}

func (c Crawler) getMimeType(hash string) (string, error) {
	url := hashUrl(hash)
	response, err := c.sh.Cat(url)
	if err != nil {
		return "", err
	}

	var data []byte
	data = make([]byte, 512)
	numread, err := response.Read(data)
	if err != nil && err.Error() != "EOF" {
		return "", err
	}

	if numread == 0 {
		return "", errors.New("0 characters read, mime type detection failed")
	}

	err = response.Close()
	if err != nil {
		return "", err
	}

	// Sniffing only uses at most the first 512 bytes
	return http.DetectContentType(data), nil
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string) error {
	fmt.Printf("Crawling file %s\n", hash)

	mimetype, err := c.getMimeType(hash)
	if err != nil {
		return err
	}

	properties := map[string]interface{}{
		"mimetype": mimetype,
	}

	c.id.IndexItem("file", hash, properties)

	return nil
}
<commit_msg>Proper defer for Close().<commit_after>package crawler

import (
	"errors"
	"fmt"
	machinery "github.com/RichardKnop/machinery/v1"
	signatures "github.com/RichardKnop/machinery/v1/signatures"
	"github.com/dokterbob/ipfs-search/indexer"
	"gopkg.in/ipfs/go-ipfs-api.v1"
	"net/http"
)

type Crawler struct {
	sh  *shell.Shell
	id  *indexer.Indexer
	mac *machinery.Server
}

func NewCrawler(sh *shell.Shell, id *indexer.Indexer, mac *machinery.Server) *Crawler {
	c := new(Crawler)
	c.sh = sh
	c.id = id
	c.mac = mac
	return c
}

func hashUrl(hash string) string {
	return fmt.Sprintf("/ipfs/%s", hash)
}

// Given a particular hash, start crawling
func (c Crawler) CrawlHash(hash string) error {
	indexed, err := c.id.IsIndexed(hash)
	if err != nil {
		return err
	}

	if indexed {
		fmt.Printf("Already indexed '%s', skipping\n", hash)
		return nil
	}

	fmt.Printf("Crawling hash '%s'\n", hash)

	url := hashUrl(hash)

	list, err := c.sh.FileList(url)
	if err != nil {
		return err
	}

	switch list.Type {
	case "File":
		// Add to file crawl queue
		task := signatures.TaskSignature{
			Name: "crawl_file",
			Args: []signatures.TaskArg{
				signatures.TaskArg{
					Type:  "string",
					Value: hash,
				},
				signatures.TaskArg{
					Type:  "string",
					Value: nil,
				},
			},
		}
		_, err := c.mac.SendTask(&task)
		if err != nil {
			// failed to send the task
			return err
		}
	case "Directory":
		// Index name and size for items
		properties := map[string]interface{}{
			"links": list.Links,
		}

		c.id.IndexItem("directory", hash, properties)

		for _, link := range list.Links {
			switch link.Type {
			case "File":
				// Add file to crawl queue
				task := signatures.TaskSignature{
					Name: "crawl_file",
					Args: []signatures.TaskArg{
						signatures.TaskArg{
							Type:  "string",
							Value: link.Hash,
						},
					},
				}
				_, err := c.mac.SendTask(&task)
				if err != nil {
					// failed to send the task
					return err
				}

			case "Directory":
				// Add directory to crawl queue
				task := signatures.TaskSignature{
					Name: "crawl_hash",
					Args: []signatures.TaskArg{
						signatures.TaskArg{
							Type:  "string",
							Value: link.Hash,
						},
					},
				}
				_, err := c.mac.SendTask(&task)
				if err != nil {
					// failed to send the task
					return err
				}
			default:
				fmt.Printf("Type '%s' skipped for '%s'", list.Type, hash)
			}
		}
	default:
		fmt.Printf("Type '%s' skipped for '%s'", list.Type, hash)
	}

	return nil
}

func (c Crawler) getMimeType(hash string) (string, error) {
	url := hashUrl(hash)
	response, err := c.sh.Cat(url)
	if err != nil {
		return "", err
	}

	defer response.Close()

	var data []byte
	data = make([]byte, 512)
	numread, err := response.Read(data)
	if err != nil && err.Error() != "EOF" {
		return "", err
	}

	if numread == 0 {
		return "", errors.New("0 characters read, mime type detection failed")
	}

	// Sniffing only uses at most the first 512 bytes
	return http.DetectContentType(data), nil
}

// Crawl a single object, known to be a file
func (c Crawler) CrawlFile(hash string) error {
	fmt.Printf("Crawling file %s\n", hash)

	mimetype, err := c.getMimeType(hash)
	if err != nil {
		return err
	}

	properties := map[string]interface{}{
		"mimetype": mimetype,
	}

	c.id.IndexItem("file", hash, properties)

	return nil
}
<|endoftext|>


<commit_before>// run

// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"testing"
)

type T struct { int }

var globl *T

func F() {
	t := &T{}
	globl = t
}

func G() {
	t := &T{}
	_ = t
}

func main() {
	nf := testing.AllocsPerRun(100, F)
	ng := testing.AllocsPerRun(100, G)
	if int(nf) != 1 {
		fmt.Printf("AllocsPerRun(100, F) = %v, want 1\n", nf)
		os.Exit(1)
	}
	if int(ng) != 0 {
		fmt.Printf("AllocsPerRun(100, G) = %v, want 0\n", ng)
		os.Exit(1)
	}
}
<commit_msg>test: adjust issue4618 for gccgo allocation behaviour<commit_after>// run

// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"runtime"
	"testing"
)

type T struct { int }

var globl *T

func F() {
	t := &T{}
	globl = t
}

func G() {
	t := &T{}
	_ = t
}

func main() {
	nf := testing.AllocsPerRun(100, F)
	ng := testing.AllocsPerRun(100, G)
	if int(nf) != 1 {
		fmt.Printf("AllocsPerRun(100, F) = %v, want 1\n", nf)
		os.Exit(1)
	}
	if int(ng) != 0 && (runtime.Compiler != "gccgo" || int(ng) != 1) {
		fmt.Printf("AllocsPerRun(100, G) = %v, want 0\n", ng)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

/*
usage: fn init <name>

If there's a Dockerfile found, this will generate the basic file with just the image name. exit
It will then try to decipher the runtime based on the files in the current directory, if it can't figure it out, it will ask.
It will then take a best guess for what the entrypoint will be based on the language, it it can't guess, it will ask.

*/

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"strings"

	"github.com/iron-io/functions/fn/langs"
	"github.com/urfave/cli"
)

var (
	fileExtToRuntime = map[string]string{
		".go": "go",
		".js": "node",
		".rb": "ruby",
		".py": "python",
	}

	fnInitRuntimes []string
)

func init() {
	for rt := range fileExtToRuntime {
		fnInitRuntimes = append(fnInitRuntimes, rt)
	}
}

type initFnCmd struct {
	name       string
	force      bool
	runtime    *string
	entrypoint *string
}

func initFn() cli.Command {
	a := initFnCmd{}

	return cli.Command{
		Name:        "init",
		Usage:       "create a local func.yaml file",
		Description: "Creates a func.yaml file in the current directory.  ",
		ArgsUsage:   "<DOCKERHUB_USERNAME/FUNCTION_NAME>",
		Action:      a.init,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:        "force, f",
				Usage:       "overwrite existing func.yaml",
				Destination: &a.force,
			},
			cli.StringFlag{
				Name:        "runtime",
				Usage:       "choose an existing runtime - " + strings.Join(fnInitRuntimes, ", "),
				Destination: a.runtime,
			},
			cli.StringFlag{
				Name:        "entrypoint",
				Usage:       "entrypoint is the command to run to start this function - equivalent to Dockerfile ENTRYPOINT.",
				Destination: a.entrypoint,
			},
		},
	}
}

func (a *initFnCmd) init(c *cli.Context) error {
	if !a.force {
		ff, err := findFuncfile()
		if _, ok := err.(*notFoundError); !ok && err != nil {
			return err
		}
		if ff != nil {
			return errors.New("function file already exists")
		}
	}

	err := a.buildFuncFile(c)
	if err != nil {
		return err
	}

	ff := &funcfile{
		Name:       a.name,
		Runtime:    a.runtime,
		Version:    initialVersion,
		Entrypoint: a.entrypoint,
	}

	if err := encodeFuncfileYAML("func.yaml", ff); err != nil {
		return err
	}

	fmt.Println("func.yaml created.")
	return nil
}

func (a *initFnCmd) buildFuncFile(c *cli.Context) error {
	pwd, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("error detecting current working directory: %s\n", err)
	}

	a.name = c.Args().First()
	if a.name == "" || strings.Contains(a.name, ":") {
		return errors.New("Please specify a name for your function in the following format <DOCKERHUB_USERNAME>/<FUNCTION_NAME>")
	}

	if exists("Dockerfile") {
		fmt.Println("Dockerfile found, will use that to build.")
		return nil
	}

	var rt string
	if a.runtime == nil || *a.runtime == "" {
		rt, err = detectRuntime(pwd)
		if err != nil {
			return err
		}
		a.runtime = &rt
		fmt.Printf("assuming %v runtime\n", rt)
	}
	if _, ok := acceptableFnRuntimes[*a.runtime]; !ok {
		return fmt.Errorf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", *a.runtime)
	}

	if a.entrypoint == nil || *a.entrypoint == "" {
		ep, err := detectEntrypoint(*a.runtime)
		if err != nil {
			return fmt.Errorf("could not detect entrypoint for %v, use --entrypoint to add it explicitly. %v", *a.runtime, err)
		}
		a.entrypoint = &ep
	}

	return nil
}

func detectRuntime(path string) (runtime string, err error) {
	for ext, runtime := range fileExtToRuntime {
		fn := filepath.Join(path, fmt.Sprintf("func%s", ext))
		if exists(fn) {
			return runtime, nil
		}
	}

	return "", fmt.Errorf("no supported files found to guess runtime, please set runtime explicitly with --runtime flag")
}

func detectEntrypoint(runtime string) (string, error) {
	helper, err := langs.GetLangHelper(runtime)
	if err != nil {
		return "", err
	}

	return helper.Entrypoint(), nil
}
<commit_msg>fn: fix broken flags (#336)<commit_after>package main

/*
usage: fn init <name>

If there's a Dockerfile found, this will generate the basic file with just the image name. exit
It will then try to decipher the runtime based on the files in the current directory, if it can't figure it out, it will ask.
It will then take a best guess for what the entrypoint will be based on the language, it it can't guess, it will ask.

*/

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"strings"

	"github.com/iron-io/functions/fn/langs"
	"github.com/urfave/cli"
)

var (
	fileExtToRuntime = map[string]string{
		".go": "go",
		".js": "node",
		".rb": "ruby",
		".py": "python",
	}

	fnInitRuntimes []string
)

func init() {
	for rt := range fileExtToRuntime {
		fnInitRuntimes = append(fnInitRuntimes, rt)
	}
}

type initFnCmd struct {
	name       string
	force      bool
	runtime    string
	entrypoint string
}

func initFn() cli.Command {
	a := initFnCmd{}

	return cli.Command{
		Name:        "init",
		Usage:       "create a local func.yaml file",
		Description: "Creates a func.yaml file in the current directory.  ",
		ArgsUsage:   "<DOCKERHUB_USERNAME/FUNCTION_NAME>",
		Action:      a.init,
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:        "force, f",
				Usage:       "overwrite existing func.yaml",
				Destination: &a.force,
			},
			cli.StringFlag{
				Name:        "runtime",
				Usage:       "choose an existing runtime - " + strings.Join(fnInitRuntimes, ", "),
				Destination: &a.runtime,
			},
			cli.StringFlag{
				Name:        "entrypoint",
				Usage:       "entrypoint is the command to run to start this function - equivalent to Dockerfile ENTRYPOINT.",
				Destination: &a.entrypoint,
			},
		},
	}
}

func (a *initFnCmd) init(c *cli.Context) error {
	if !a.force {
		ff, err := findFuncfile()
		if _, ok := err.(*notFoundError); !ok && err != nil {
			return err
		}
		if ff != nil {
			return errors.New("function file already exists")
		}
	}

	err := a.buildFuncFile(c)
	if err != nil {
		return err
	}

	ff := &funcfile{
		Name:       a.name,
		Runtime:    &a.runtime,
		Version:    initialVersion,
		Entrypoint: &a.entrypoint,
	}

	if err := encodeFuncfileYAML("func.yaml", ff); err != nil {
		return err
	}

	fmt.Println("func.yaml created.")
	return nil
}

func (a *initFnCmd) buildFuncFile(c *cli.Context) error {
	pwd, err := os.Getwd()
	if err != nil {
		return fmt.Errorf("error detecting current working directory: %s\n", err)
	}

	a.name = c.Args().First()
	if a.name == "" || strings.Contains(a.name, ":") {
		return errors.New("Please specify a name for your function in the following format <DOCKERHUB_USERNAME>/<FUNCTION_NAME>")
	}

	if exists("Dockerfile") {
		fmt.Println("Dockerfile found, will use that to build.")
		return nil
	}

	var rt string
	if a.runtime == "" {
		rt, err = detectRuntime(pwd)
		if err != nil {
			return err
		}
		a.runtime = rt
		fmt.Printf("assuming %v runtime\n", rt)
	}
	if _, ok := acceptableFnRuntimes[a.runtime]; !ok {
		return fmt.Errorf("init does not support the %s runtime, you'll have to create your own Dockerfile for this function", a.runtime)
	}

	if a.entrypoint == "" {
		ep, err := detectEntrypoint(a.runtime)
		if err != nil {
			return fmt.Errorf("could not detect entrypoint for %v, use --entrypoint to add it explicitly. %v", a.runtime, err)
		}
		a.entrypoint = ep
	}

	return nil
}

func detectRuntime(path string) (runtime string, err error) {
	for ext, runtime := range fileExtToRuntime {
		fn := filepath.Join(path, fmt.Sprintf("func%s", ext))
		if exists(fn) {
			return runtime, nil
		}
	}

	return "", fmt.Errorf("no supported files found to guess runtime, please set runtime explicitly with --runtime flag")
}

func detectEntrypoint(runtime string) (string, error) {
	helper, err := langs.GetLangHelper(runtime)
	if err != nil {
		return "", err
	}

	return helper.Entrypoint(), nil
}
<|endoftext|>


<commit_before>/*Package fs provides tools for creating temporary files, and testing the
contents and structure of a directory.
*/
package fs // import "gotest.tools/fs"

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"gotest.tools/assert"
	"gotest.tools/x/subtest"
)

// Path objects return their filesystem path. Path may be implemented by a
// real filesystem object (such as File and Dir) or by a type which updates
// entries in a Manifest.
type Path interface {
	Path() string
	Remove()
}

var (
	_ Path = &Dir{}
	_ Path = &File{}
)

// File is a temporary file on the filesystem
type File struct {
	path string
}

type helperT interface {
	Helper()
}

// NewFile creates a new file in a temporary directory using prefix as part of
// the filename. The PathOps are applied to the before returning the File.
func NewFile(t assert.TestingT, prefix string, ops ...PathOp) *File {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	tempfile, err := ioutil.TempFile("", cleanPrefix(prefix)+"-")
	assert.NilError(t, err)
	file := &File{path: tempfile.Name()}
	assert.NilError(t, tempfile.Close())
	file.Update(t, ops...)
	if tc, ok := t.(subtest.TestContext); ok {
		tc.AddCleanup(file.Remove)
	}
	return file
}

func cleanPrefix(prefix string) string {
	// windows requires both / and \ are replaced
	if runtime.GOOS == "windows" {
		prefix = strings.Replace(prefix, string(os.PathSeparator), "-", -1)
	}
	return strings.Replace(prefix, "/", "-", -1)
}

// Update applies the specified PathOps to the File.
func (f *File) Update(t assert.TestingT, ops ...PathOp) {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	for _, op := range ops {
		assert.NilError(t, op(f))
	}
}

// Path returns the full path to the file
func (f *File) Path() string {
	return f.path
}

// Remove the file
func (f *File) Remove() {
	// nolint: errcheck
	os.Remove(f.path)
}

// Dir is a temporary directory
type Dir struct {
	path string
}

// NewDir returns a new temporary directory using prefix as part of the directory
// name. The PathOps are applied before returning the Dir.
func NewDir(t assert.TestingT, prefix string, ops ...PathOp) *Dir {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	path, err := ioutil.TempDir("", cleanPrefix(prefix)+"-")
	assert.NilError(t, err)
	dir := &Dir{path: path}
	dir.Update(t, ops...)
	if tc, ok := t.(subtest.TestContext); ok {
		tc.AddCleanup(dir.Remove)
	}
	return dir
}

// Update applies the specified PathOps to the Dir.
func (d *Dir) Update(t assert.TestingT, ops ...PathOp) {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	for _, op := range ops {
		assert.NilError(t, op(d))
	}
}

// Path returns the full path to the directory
func (d *Dir) Path() string {
	return d.path
}

// Remove the directory
func (d *Dir) Remove() {
	// nolint: errcheck
	os.RemoveAll(d.path)
}

// Join returns a new path with this directory as the base of the path
func (d *Dir) Join(parts ...string) string {
	return filepath.Join(append([]string{d.Path()}, parts...)...)
}
<commit_msg>reuse existing applyPathOps function<commit_after>/*Package fs provides tools for creating temporary files, and testing the
contents and structure of a directory.
*/
package fs // import "gotest.tools/fs"

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"gotest.tools/assert"
	"gotest.tools/x/subtest"
)

// Path objects return their filesystem path. Path may be implemented by a
// real filesystem object (such as File and Dir) or by a type which updates
// entries in a Manifest.
type Path interface {
	Path() string
	Remove()
}

var (
	_ Path = &Dir{}
	_ Path = &File{}
)

// File is a temporary file on the filesystem
type File struct {
	path string
}

type helperT interface {
	Helper()
}

// NewFile creates a new file in a temporary directory using prefix as part of
// the filename. The PathOps are applied to the before returning the File.
func NewFile(t assert.TestingT, prefix string, ops ...PathOp) *File {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	tempfile, err := ioutil.TempFile("", cleanPrefix(prefix)+"-")
	assert.NilError(t, err)
	file := &File{path: tempfile.Name()}
	assert.NilError(t, tempfile.Close())
	assert.NilError(t, applyPathOps(file, ops))
	if tc, ok := t.(subtest.TestContext); ok {
		tc.AddCleanup(file.Remove)
	}
	return file
}

func cleanPrefix(prefix string) string {
	// windows requires both / and \ are replaced
	if runtime.GOOS == "windows" {
		prefix = strings.Replace(prefix, string(os.PathSeparator), "-", -1)
	}
	return strings.Replace(prefix, "/", "-", -1)
}

// Update applies the specified PathOps to the File.
func (f *File) Update(t assert.TestingT, ops ...PathOp) {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	assert.NilError(t, applyPathOps(f, ops))
}

// Path returns the full path to the file
func (f *File) Path() string {
	return f.path
}

// Remove the file
func (f *File) Remove() {
	// nolint: errcheck
	os.Remove(f.path)
}

// Dir is a temporary directory
type Dir struct {
	path string
}

// NewDir returns a new temporary directory using prefix as part of the directory
// name. The PathOps are applied before returning the Dir.
func NewDir(t assert.TestingT, prefix string, ops ...PathOp) *Dir {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	path, err := ioutil.TempDir("", cleanPrefix(prefix)+"-")
	assert.NilError(t, err)
	dir := &Dir{path: path}
	assert.NilError(t, applyPathOps(dir, ops))
	if tc, ok := t.(subtest.TestContext); ok {
		tc.AddCleanup(dir.Remove)
	}
	return dir
}

// Update applies the specified PathOps to the Dir.
func (d *Dir) Update(t assert.TestingT, ops ...PathOp) {
	if ht, ok := t.(helperT); ok {
		ht.Helper()
	}
	assert.NilError(t, applyPathOps(d, ops))
}

// Path returns the full path to the directory
func (d *Dir) Path() string {
	return d.path
}

// Remove the directory
func (d *Dir) Remove() {
	// nolint: errcheck
	os.RemoveAll(d.path)
}

// Join returns a new path with this directory as the base of the path
func (d *Dir) Join(parts ...string) string {
	return filepath.Join(append([]string{d.Path()}, parts...)...)
}
<|endoftext|>


<commit_before>package azure

import (
	"context"
	"fmt"

	armStorage "github.com/Azure/azure-sdk-for-go/arm/storage"
	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/helper/schema"
)

// New creates a new backend for S3 remote state.
func New() backend.Backend {
	s := &schema.Backend{
		Schema: map[string]*schema.Schema{
			"storage_account_name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				Description: "The name of the storage account.",
			},

			"container_name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				Description: "The container name.",
			},

			"key": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				Description: "The blob key.",
			},

			"environment": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Azure cloud environment.",
				Default:     "",
			},

			"access_key": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The access key.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_ACCESS_KEY", ""),
			},

			"resource_group_name": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The resource group name.",
			},

			"arm_subscription_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Subscription ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_SUBSCRIPTION_ID", ""),
			},

			"arm_client_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Client ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_ID", ""),
			},

			"arm_client_secret": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Client Secret.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_SECRET", ""),
			},

			"arm_tenant_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Tenant ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_TENANT_ID", ""),
			},
		},
	}

	result := &Backend{Backend: s}
	result.Backend.ConfigureFunc = result.configure
	return result
}

type Backend struct {
	*schema.Backend

	// The fields below are set from configure
	blobClient storage.BlobStorageClient

	containerName string
	keyName       string
	leaseID       string
}

func (b *Backend) configure(ctx context.Context) error {
	if b.containerName != "" {
		return nil
	}

	// Grab the resource data
	data := schema.FromContextBackendConfig(ctx)

	b.containerName = data.Get("container_name").(string)
	b.keyName = data.Get("key").(string)

	blobClient, err := getBlobClient(data)
	if err != nil {
		return err
	}
	b.blobClient = blobClient

	return nil
}

func getBlobClient(d *schema.ResourceData) (storage.BlobStorageClient, error) {
	var client storage.BlobStorageClient

	env, err := getAzureEnvironment(d.Get("environment").(string))
	if err != nil {
		return client, err
	}

	storageAccountName := d.Get("storage_account_name").(string)

	accessKey, err := getAccessKey(d, storageAccountName, env)
	if err != nil {
		return client, err
	}

	storageClient, err := storage.NewClient(storageAccountName, accessKey, env.StorageEndpointSuffix,
		storage.DefaultAPIVersion, true)
	if err != nil {
		return client, fmt.Errorf("Error creating storage client for storage account %q: %s", storageAccountName, err)
	}

	client = storageClient.GetBlobService()
	return client, nil
}

func getAccessKey(d *schema.ResourceData, storageAccountName string, env azure.Environment) (string, error) {
	if key, ok := d.GetOk("access_key"); ok {
		return key.(string), nil
	}

	resourceGroupName, rgOk := d.GetOk("resource_group_name")
	subscriptionID, subOk := d.GetOk("arm_subscription_id")
	clientID, clientIDOk := d.GetOk("arm_client_id")
	clientSecret, clientSecretOK := d.GetOk("arm_client_secret")
	tenantID, tenantIDOk := d.GetOk("arm_tenant_id")
	if !rgOk || !subOk || !clientIDOk || !clientSecretOK || !tenantIDOk {
		return "", fmt.Errorf("resource_group_name and credentials must be provided when access_key is absent")
	}

	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, tenantID.(string))
	if err != nil {
		return "", err
	}

	spt, err := adal.NewServicePrincipalToken(*oauthConfig, clientID.(string), clientSecret.(string), env.ResourceManagerEndpoint)
	if err != nil {
		return "", err
	}

	accountsClient := armStorage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID.(string))
	accountsClient.Authorizer = autorest.NewBearerAuthorizer(spt)

	keys, err := accountsClient.ListKeys(resourceGroupName.(string), storageAccountName)
	if err != nil {
		return "", fmt.Errorf("Error retrieving keys for storage account %q: %s", storageAccountName, err)
	}

	if keys.Keys == nil {
		return "", fmt.Errorf("Nil key returned for storage account %q", storageAccountName)
	}

	accessKeys := *keys.Keys
	return *accessKeys[0].Value, nil
}

func getAzureEnvironment(environment string) (azure.Environment, error) {
	if environment == "" {
		return azure.PublicCloud, nil
	}

	env, err := azure.EnvironmentFromName(environment)
	if err != nil {
		// try again with wrapped value to support readable values like german instead of AZUREGERMANCLOUD
		var innerErr error
		env, innerErr = azure.EnvironmentFromName(fmt.Sprintf("AZURE%sCLOUD", environment))
		if innerErr != nil {
			return env, fmt.Errorf("invalid 'environment' configuration: %s", err)
		}
	}

	return env, nil
}
<commit_msg>Removing dead code<commit_after>package azure

import (
	"context"
	"fmt"

	armStorage "github.com/Azure/azure-sdk-for-go/arm/storage"
	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/Azure/go-autorest/autorest"
	"github.com/Azure/go-autorest/autorest/adal"
	"github.com/Azure/go-autorest/autorest/azure"
	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/helper/schema"
)

// New creates a new backend for S3 remote state.
func New() backend.Backend {
	s := &schema.Backend{
		Schema: map[string]*schema.Schema{
			"storage_account_name": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The name of the storage account.",
			},

			"container_name": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The container name.",
			},

			"key": {
				Type:        schema.TypeString,
				Required:    true,
				Description: "The blob key.",
			},

			"environment": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Azure cloud environment.",
				Default:     "",
			},

			"access_key": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The access key.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_ACCESS_KEY", ""),
			},

			"resource_group_name": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The resource group name.",
			},

			"arm_subscription_id": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Subscription ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_SUBSCRIPTION_ID", ""),
			},

			"arm_client_id": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Client ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_ID", ""),
			},

			"arm_client_secret": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Client Secret.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_CLIENT_SECRET", ""),
			},

			"arm_tenant_id": {
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The Tenant ID.",
				DefaultFunc: schema.EnvDefaultFunc("ARM_TENANT_ID", ""),
			},
		},
	}

	result := &Backend{Backend: s}
	result.Backend.ConfigureFunc = result.configure
	return result
}

type Backend struct {
	*schema.Backend

	// The fields below are set from configure
	blobClient storage.BlobStorageClient

	containerName string
	keyName       string
	leaseID       string
}

func (b *Backend) configure(ctx context.Context) error {
	if b.containerName != "" {
		return nil
	}

	// Grab the resource data
	data := schema.FromContextBackendConfig(ctx)

	b.containerName = data.Get("container_name").(string)
	b.keyName = data.Get("key").(string)

	blobClient, err := getBlobClient(data)
	if err != nil {
		return err
	}
	b.blobClient = blobClient

	return nil
}

func getBlobClient(d *schema.ResourceData) (storage.BlobStorageClient, error) {
	var client storage.BlobStorageClient

	env, err := getAzureEnvironment(d.Get("environment").(string))
	if err != nil {
		return client, err
	}

	storageAccountName := d.Get("storage_account_name").(string)

	accessKey, err := getAccessKey(d, storageAccountName, env)
	if err != nil {
		return client, err
	}

	storageClient, err := storage.NewClient(storageAccountName, accessKey, env.StorageEndpointSuffix,
		storage.DefaultAPIVersion, true)
	if err != nil {
		return client, fmt.Errorf("Error creating storage client for storage account %q: %s", storageAccountName, err)
	}

	client = storageClient.GetBlobService()
	return client, nil
}

func getAccessKey(d *schema.ResourceData, storageAccountName string, env azure.Environment) (string, error) {
	if key, ok := d.GetOk("access_key"); ok {
		return key.(string), nil
	}

	resourceGroupName, rgOk := d.GetOk("resource_group_name")
	subscriptionID, subOk := d.GetOk("arm_subscription_id")
	clientID, clientIDOk := d.GetOk("arm_client_id")
	clientSecret, clientSecretOK := d.GetOk("arm_client_secret")
	tenantID, tenantIDOk := d.GetOk("arm_tenant_id")
	if !rgOk || !subOk || !clientIDOk || !clientSecretOK || !tenantIDOk {
		return "", fmt.Errorf("resource_group_name and credentials must be provided when access_key is absent")
	}

	oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, tenantID.(string))
	if err != nil {
		return "", err
	}

	spt, err := adal.NewServicePrincipalToken(*oauthConfig, clientID.(string), clientSecret.(string), env.ResourceManagerEndpoint)
	if err != nil {
		return "", err
	}

	accountsClient := armStorage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID.(string))
	accountsClient.Authorizer = autorest.NewBearerAuthorizer(spt)

	keys, err := accountsClient.ListKeys(resourceGroupName.(string), storageAccountName)
	if err != nil {
		return "", fmt.Errorf("Error retrieving keys for storage account %q: %s", storageAccountName, err)
	}

	if keys.Keys == nil {
		return "", fmt.Errorf("Nil key returned for storage account %q", storageAccountName)
	}

	accessKeys := *keys.Keys
	return *accessKeys[0].Value, nil
}

func getAzureEnvironment(environment string) (azure.Environment, error) {
	if environment == "" {
		return azure.PublicCloud, nil
	}

	env, err := azure.EnvironmentFromName(environment)
	if err != nil {
		// try again with wrapped value to support readable values like german instead of AZUREGERMANCLOUD
		var innerErr error
		env, innerErr = azure.EnvironmentFromName(fmt.Sprintf("AZURE%sCLOUD", environment))
		if innerErr != nil {
			return env, fmt.Errorf("invalid 'environment' configuration: %s", err)
		}
	}

	return env, nil
}
<|endoftext|>


<commit_before>package cribbage

import (
	"fmt"
	"github.com/fatih/color"
)

const (
	SPADE   = "♠"
	HEART   = "♥"
	DIAMOND = "♦"
	CLUB    = "♣"
)

func RenderCard(card *Card) string {
	var printer func(...interface{}) string
	if isRed(card) {
		printer = color.New(color.BgWhite, color.FgRed).SprintFunc()
	} else {
		printer = color.New(color.BgWhite, color.FgBlack).SprintFunc()
	}
	return printer(renderRank(card), renderSuit(card))
}

func RenderHand(hand *Hand) string {
	out := ""
	for i, card := range hand.cards {
		if card == nil {
			card = &Card{}
		}
		out += fmt.Sprintf("%d:", i) + RenderCard(card)
		if i < len(hand.cards) {
			out += " "
		}
	}
	return out
}

func isRed(card *Card) bool {
	return card.Suit == "Hearts" || card.Suit == "Diamonds"
}

func renderSuit(card *Card) string {
	switch card.Suit {
	case "Hearts":
		return HEART
	case "Spades":
		return SPADE
	case "Clubs":
		return CLUB
	case "Diamonds":
		return DIAMOND
	default:
		return "?"
	}
}

func renderRank(card *Card) string {
	switch card.Rank {
	case "Two":
		return "2"
	case "Three":
		return "3"
	case "Four":
		return "4"
	case "Five":
		return "5"
	case "Six":
		return "6"
	case "Seven":
		return "7"
	case "Eight":
		return "8"
	case "Nine":
		return "9"
	case "Ten":
		return "10"
	case "Jack":
		return "J"
	case "Queen":
		return "Q"
	case "King":
		return "K"
	case "Ace":
		return "A"
	default:
		return "?"
	}
}
<commit_msg>Change card back rendering<commit_after>package cribbage

import (
	"fmt"
	"github.com/fatih/color"
)

const (
	SPADE   = "♠"
	HEART   = "♥"
	DIAMOND = "♦"
	CLUB    = "♣"
)

func RenderCard(card *Card) string {
	var printer func(...interface{}) string
	if isUnknown(card) {
		printer = color.New(color.BgBlue, color.FgWhite).SprintFunc()
	} else if isRed(card) {
		printer = color.New(color.BgWhite, color.FgRed).SprintFunc()
	} else {
		printer = color.New(color.BgWhite, color.FgBlack).SprintFunc()
	}
	return printer(renderRank(card), renderSuit(card))
}

func RenderHand(hand *Hand) string {
	out := ""
	for i, card := range hand.cards {
		if card == nil {
			card = &Card{}
		}
		out += fmt.Sprintf("%d:", i) + RenderCard(card)
		if i < len(hand.cards) {
			out += " "
		}
	}
	return out
}

func isRed(card *Card) bool {
	return card.Suit == "Hearts" || card.Suit == "Diamonds"
}

func isUnknown(card *Card) bool {
	return card.Suit == "" || card.Suit == ""
}

func renderSuit(card *Card) string {
	switch card.Suit {
	case "Hearts":
		return HEART
	case "Spades":
		return SPADE
	case "Clubs":
		return CLUB
	case "Diamonds":
		return DIAMOND
	default:
		return "▹"
	}
}

func renderRank(card *Card) string {
	switch card.Rank {
	case "Two":
		return "2"
	case "Three":
		return "3"
	case "Four":
		return "4"
	case "Five":
		return "5"
	case "Six":
		return "6"
	case "Seven":
		return "7"
	case "Eight":
		return "8"
	case "Nine":
		return "9"
	case "Ten":
		return "10"
	case "Jack":
		return "J"
	case "Queen":
		return "Q"
	case "King":
		return "K"
	case "Ace":
		return "A"
	default:
		return "◃"
	}
}
<|endoftext|>


<commit_before>package gowfs

import "os"
import "bytes"
import "fmt"
import "io"
import "path"
import "io/ioutil"

const MAX_UP_CHUNK int64 = 1 * (1024 * 1024) * 1024 // 1 GB.
const MAX_DOWN_CHUNK int64 = 500 * (1024 * 1024)    // 500 MB

type FsShell struct {
	FileSystem  *FileSystem
	WorkingPath string
}

// Appends the specified list of local files to the HDFS path.
func (shell FsShell) AppendToFile(filePaths []string, hdfsPath string) (bool, error) {

	for _, path := range filePaths {
		file, err := os.Open(path)

		if err != nil {
			return false, err
		}
		defer file.Close()

		data, _, err := slirpLocalFile(*file, 0)
		if err != nil {
			return false, err
		}

		_, err = shell.FileSystem.Append(bytes.NewBuffer(data), Path{Name: hdfsPath}, 0)
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

// Returns a writer with the content of the specified files.
func (shell FsShell) Cat(hdfsPaths []string, writr io.Writer) error {
	for _, path := range hdfsPaths {
		stat, err := shell.FileSystem.GetFileStatus(Path{Name: path})
		if err != nil {
			return err
		}
		//TODO add code to chunk super large files.
		if stat.Length < MAX_DOWN_CHUNK {
			readr, err := shell.FileSystem.Open(Path{Name: path}, 0, stat.Length, 4096)
			if err != nil {
				return err
			}
			io.Copy(writr, readr)
		}
	}
	return nil
}

// Changes the group association of the given hdfs paths.
func (shell FsShell) Chgrp(hdfsPaths []string, grpName string) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetOwner(Path{Name: path}, "", grpName)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Changes the owner of the specified hdfs paths.
func (shell FsShell) Chown(hdfsPaths []string, owner string) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetOwner(Path{Name: path}, owner, "")
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Changes the filemode of the provided hdfs paths.
func (shell FsShell) Chmod(hdfsPaths []string, perm os.FileMode) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetPermission(Path{Name: path}, perm)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Tests the existence of a remote HDFS file/directory.
func (shell FsShell) Exists(hdfsPath string) (bool, error) {
	_, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})
	if err != nil {
		if remoteErr, ok := err.(RemoteException); ok && remoteErr.JavaClassName == "java.io.FileNotFoundException" {
			return false, nil
		} else {
			return false, err /* a different err */
		}
	}
	return true, nil
}

// Copies one specified local file to the remote HDFS server.
// Uses default permission, blocksize, and replication.
func (shell FsShell) Put(localFile string, hdfsPath string, overwrite bool) (bool, error) {
	if _, err := os.Stat(localFile); os.IsNotExist(err) {
		return false, fmt.Errorf("File %v not found.", localFile)
	}

	file, err := os.Open(localFile)
	if err != nil {
		return false, err
	}
	defer file.Close()

	// put as a new remote file
	_, err = shell.FileSystem.Create(
		file,
		Path{Name: hdfsPath + "/" + µ(path.Split(localFile))[1].(string)},
		overwrite,
		134217728,
		3,
		0644,
		4096)

	if err != nil {
		return false, err
	}

	return true, nil
}

// Copies sepcified local files to remote HDFS server.
// The hdfsPath must be a directory (created if it does not exist).
// Uses default permission, blocksize, and replication.
func (shell FsShell) PutMany(files []string, hdfsPath string, overwrite bool) (bool, error) {
	// if multiple files, put in remote directory
	if len(files) > 1 {
		stat, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})

		// if remote dir missing, crete it.
		if remoteErr := err.(RemoteException); remoteErr.JavaClassName == "java.io.FileNotFoundException" {
			if _, err := shell.FileSystem.MkDirs(Path{Name: hdfsPath}, 0700); err != nil {
				return false, err
			}
		}
		if stat.Type == "FILE" {
			return false, fmt.Errorf("HDFS resource %s must be a directory in this context.", hdfsPath)
		}
	}
	for _, file := range files {
		shell.Put(file, hdfsPath+"/"+µ(path.Split(file))[1].(string), overwrite)
	}
	return true, nil
}

// Retrieves a remote HDFS file and saves as the specified local file.
func (shell FsShell) Get(hdfsPath, localFile string) (bool, error) {
	file, err := os.Create(localFile)
	if err != nil {
		return false, err
	}
	defer file.Close()

	reader, err := shell.FileSystem.Open(Path{Name: hdfsPath}, 0, 0, 0)
	if err != nil {
		return false, err
	}
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return false, err
	}
	defer reader.Close()

	_, err = file.Write(data)
	if err != nil {
		return false, err
	}
	file.Sync()
	return true, nil
}

// Merges content of remote HDFS path into a single local file.
// func (shell FsShell) GetMerge(hdfsPath, localFile string)(bool, error){
// 	return false, fmt.Errorf("Function is unimplemented.")
// }

// Copies local file to remote destination, then local file is removed.
func (shell FsShell) MoveFromLocal(localFile, hdfsPath string, overwrite bool) (bool, error) {
	ok, err := shell.Put(localFile, hdfsPath, overwrite)
	// validate operation, then remove local
	if ok && err != nil {
		hdfStat, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})
		if err != nil {
			return false, fmt.Errorf("Unable to verify remote file. ", err.Error())
		}

		file, err := os.Open(localFile)
		if err != nil {
			return false, fmt.Errorf("Unable to validate operation. ", err.Error())
		}
		if hdfStat.Length != µ(file.Stat())[0].(os.FileInfo).Size() {
			return false, fmt.Errorf("Remote and local file size mismatch.")
		}
		file.Close()               // close now.
		err = os.Remove(localFile) // remove it.
		if err != nil {
			return false, err
		}
	} else {
		return false, err
	}
	return true, nil
}

// Copies remote HDFS file locally.  The remote file is then removed.
func (shell FsShell) MoveToLocal(hdfsPath, localFile string) (bool, error) {
	hdfStat, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})
	_, err = shell.Get(hdfsPath, localFile)
	if err != nil {
		return false, err
	}

	file, err := os.Open(localFile)
	if err != nil {
		return false, fmt.Errorf("Unable to access local file %s: %s", localFile, err.Error())
	}
	defer file.Close()

	fileStat, err := file.Stat()
	if err != nil {
		return false, fmt.Errorf("Unable to access local file %s: %s", localFile, err.Error())
	}

	// ensure file was copied all the way
	if hdfStat.Length != fileStat.Size() {
		return false, fmt.Errorf("Local file size does not match remote file size. Aborting.")
	}

	// remove remote File
	ok, err := shell.FileSystem.Delete(Path{Name: hdfsPath}, false)
	if err != nil {
		return false, fmt.Errorf("Unable to remove remote %s file: %s", hdfsPath, err.Error())
	}

	return ok, nil
}

// Removes the specified HDFS source.
func (shell FsShell) Rm(hdfsPath string) (bool, error) {
	return false, fmt.Errorf("Function is unimplemented.")
}

// TODO: slirp file in x Gbyte chunks when file.Stat() >> X.
//       this is to avoid blow up memory on large files.
func slirpLocalFile(file os.File, offset int64) ([]byte, int64, error) {
	stat, err := file.Stat()
	if err != nil {
		return nil, 0, err
	}

	if stat.Size() < MAX_UP_CHUNK {
		data, err := ioutil.ReadFile(file.Name())
		if err != nil {
			return nil, 0, err
		}
		return data, 0, nil
	} // else chunck it

	return nil, 0, nil
}

//TODO: slirp file in X GBytes chucks from server to avoid blowing up network.
// func slirpRemoteFile (hdfsPath string, offset int64, totalSize int64)([]byte, int64, error) {

// }
<commit_msg>for bug, MoveFromLocal function didn't remove local file (#18)<commit_after>package gowfs

import "os"
import "bytes"
import "fmt"
import "io"
import "path"
import "io/ioutil"

const MAX_UP_CHUNK int64 = 1 * (1024 * 1024) * 1024 // 1 GB.
const MAX_DOWN_CHUNK int64 = 500 * (1024 * 1024)    // 500 MB

type FsShell struct {
	FileSystem  *FileSystem
	WorkingPath string
}

// Appends the specified list of local files to the HDFS path.
func (shell FsShell) AppendToFile(filePaths []string, hdfsPath string) (bool, error) {

	for _, path := range filePaths {
		file, err := os.Open(path)

		if err != nil {
			return false, err
		}
		defer file.Close()

		data, _, err := slirpLocalFile(*file, 0)
		if err != nil {
			return false, err
		}

		_, err = shell.FileSystem.Append(bytes.NewBuffer(data), Path{Name: hdfsPath}, 0)
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

// Returns a writer with the content of the specified files.
func (shell FsShell) Cat(hdfsPaths []string, writr io.Writer) error {
	for _, path := range hdfsPaths {
		stat, err := shell.FileSystem.GetFileStatus(Path{Name: path})
		if err != nil {
			return err
		}
		//TODO add code to chunk super large files.
		if stat.Length < MAX_DOWN_CHUNK {
			readr, err := shell.FileSystem.Open(Path{Name: path}, 0, stat.Length, 4096)
			if err != nil {
				return err
			}
			io.Copy(writr, readr)
		}
	}
	return nil
}

// Changes the group association of the given hdfs paths.
func (shell FsShell) Chgrp(hdfsPaths []string, grpName string) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetOwner(Path{Name: path}, "", grpName)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Changes the owner of the specified hdfs paths.
func (shell FsShell) Chown(hdfsPaths []string, owner string) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetOwner(Path{Name: path}, owner, "")
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Changes the filemode of the provided hdfs paths.
func (shell FsShell) Chmod(hdfsPaths []string, perm os.FileMode) (bool, error) {
	for _, path := range hdfsPaths {
		_, err := shell.FileSystem.SetPermission(Path{Name: path}, perm)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

// Tests the existence of a remote HDFS file/directory.
func (shell FsShell) Exists(hdfsPath string) (bool, error) {
	_, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})
	if err != nil {
		if remoteErr, ok := err.(RemoteException); ok && remoteErr.JavaClassName == "java.io.FileNotFoundException" {
			return false, nil
		} else {
			return false, err /* a different err */
		}
	}
	return true, nil
}

// Copies one specified local file to the remote HDFS server.
// Uses default permission, blocksize, and replication.
func (shell FsShell) Put(localFile string, hdfsPath string, overwrite bool) (bool, error) {
	if _, err := os.Stat(localFile); os.IsNotExist(err) {
		return false, fmt.Errorf("File %v not found.", localFile)
	}

	file, err := os.Open(localFile)
	if err != nil {
		return false, err
	}
	defer file.Close()

	// put as a new remote file
	_, err = shell.FileSystem.Create(
		file,
		Path{Name: hdfsPath + "/" + µ(path.Split(localFile))[1].(string)},
		overwrite,
		134217728,
		3,
		0644,
		4096)

	if err != nil {
		return false, err
	}

	return true, nil
}

// Copies sepcified local files to remote HDFS server.
// The hdfsPath must be a directory (created if it does not exist).
// Uses default permission, blocksize, and replication.
func (shell FsShell) PutMany(files []string, hdfsPath string, overwrite bool) (bool, error) {
	// if multiple files, put in remote directory
	if len(files) > 1 {
		stat, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})

		// if remote dir missing, crete it.
		if remoteErr := err.(RemoteException); remoteErr.JavaClassName == "java.io.FileNotFoundException" {
			if _, err := shell.FileSystem.MkDirs(Path{Name: hdfsPath}, 0700); err != nil {
				return false, err
			}
		}
		if stat.Type == "FILE" {
			return false, fmt.Errorf("HDFS resource %s must be a directory in this context.", hdfsPath)
		}
	}
	for _, file := range files {
		shell.Put(file, hdfsPath+"/"+µ(path.Split(file))[1].(string), overwrite)
	}
	return true, nil
}

// Retrieves a remote HDFS file and saves as the specified local file.
func (shell FsShell) Get(hdfsPath, localFile string) (bool, error) {
	file, err := os.Create(localFile)
	if err != nil {
		return false, err
	}
	defer file.Close()

	reader, err := shell.FileSystem.Open(Path{Name: hdfsPath}, 0, 0, 0)
	if err != nil {
		return false, err
	}
	data, err := ioutil.ReadAll(reader)
	if err != nil {
		return false, err
	}
	defer reader.Close()

	_, err = file.Write(data)
	if err != nil {
		return false, err
	}
	file.Sync()
	return true, nil
}

// Merges content of remote HDFS path into a single local file.
// func (shell FsShell) GetMerge(hdfsPath, localFile string)(bool, error){
// 	return false, fmt.Errorf("Function is unimplemented.")
// }

// Copies local file to remote destination, then local file is removed.
func (shell FsShell) MoveFromLocal(localFile, hdfsPath string, overwrite bool) (bool, error) {
	ok, err := shell.Put(localFile, hdfsPath, overwrite)
	// validate operation, then remove local
	if ok && err == nil {
		hdfStat, err := shell.FileSystem.GetFileStatus(Path{Name: path.Join(hdfsPath, path.Base(localFile))})
		if err != nil {
			return false, fmt.Errorf("Unable to verify remote file. err is %v", err.Error())
		}

		file, err := os.Open(localFile)
		if err != nil {
			return false, fmt.Errorf("Unable to validate operation. err is %v", err.Error())
		}
		if hdfStat.Length != µ(file.Stat())[0].(os.FileInfo).Size() {
			return false, fmt.Errorf("Remote and local file size mismatch.")
		}
		file.Close()               // close now.
		err = os.Remove(localFile) // remove it.
		if err != nil {
			return false, err
		}
	} else {
		return false, err
	}
	return true, nil
}

// Copies remote HDFS file locally.  The remote file is then removed.
func (shell FsShell) MoveToLocal(hdfsPath, localFile string) (bool, error) {
	hdfStat, err := shell.FileSystem.GetFileStatus(Path{Name: hdfsPath})
	_, err = shell.Get(hdfsPath, localFile)
	if err != nil {
		return false, err
	}

	file, err := os.Open(localFile)
	if err != nil {
		return false, fmt.Errorf("Unable to access local file %s: %s", localFile, err.Error())
	}
	defer file.Close()

	fileStat, err := file.Stat()
	if err != nil {
		return false, fmt.Errorf("Unable to access local file %s: %s", localFile, err.Error())
	}

	// ensure file was copied all the way
	if hdfStat.Length != fileStat.Size() {
		return false, fmt.Errorf("Local file size does not match remote file size. Aborting.")
	}

	// remove remote File
	ok, err := shell.FileSystem.Delete(Path{Name: hdfsPath}, false)
	if err != nil {
		return false, fmt.Errorf("Unable to remove remote %s file: %s", hdfsPath, err.Error())
	}

	return ok, nil
}

// Removes the specified HDFS source.
func (shell FsShell) Rm(hdfsPath string) (bool, error) {
	return false, fmt.Errorf("Function is unimplemented.")
}

// TODO: slirp file in x Gbyte chunks when file.Stat() >> X.
//       this is to avoid blow up memory on large files.
func slirpLocalFile(file os.File, offset int64) ([]byte, int64, error) {
	stat, err := file.Stat()
	if err != nil {
		return nil, 0, err
	}

	if stat.Size() < MAX_UP_CHUNK {
		data, err := ioutil.ReadFile(file.Name())
		if err != nil {
			return nil, 0, err
		}
		return data, 0, nil
	} // else chunck it

	return nil, 0, nil
}

//TODO: slirp file in X GBytes chucks from server to avoid blowing up network.
// func slirpRemoteFile (hdfsPath string, offset int64, totalSize int64)([]byte, int64, error) {

// }
<|endoftext|>