<commit_before>package command

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
	"os"
	"strings"
	"time"
)

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)

	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return nil, err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
				client = newClient
			} else {
				return err
			}
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}
	return eachEntryFunc, nil
}

func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
	var writeErr error
	err = util.Retry("writeFile", func() error {
		reader := filer.NewFileReader(filerSource, newEntry)
		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
		if writeErr != nil {
			return writeErr
		}
		return nil
	})
	return
}

func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
		if err != nil {
			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
			return time.Now()
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-timeAgo)
	}
	return lastOffsetTs
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<commit_msg>add logs for writing to remote file<commit_after>package command

import (
	"context"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"
)

func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {

	// read filer remote storage mount mappings
	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
	if detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)

	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
	client, err := remote_storage.GetRemoteStorage(remoteStorage)
	if err != nil {
		return nil, err
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			if remoteLoc, found := mappings.Mappings[mountedDir]; found {
				if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
					glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
				}
			} else {
				glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
				os.Exit(0)
			}
		}
		if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			remoteStorage = conf
			if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
				client = newClient
			} else {
				return err
			}
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping updating: %+v", resp)
				return nil
			}
			if message.NewEntry.IsDirectory {
				return client.WriteDirectory(dest, message.NewEntry)
			}
			if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
				if filer.IsSameData(message.OldEntry, message.NewEntry) {
					glog.V(2).Infof("update meta: %+v", resp)
					return client.UpdateFileMetadata(dest, message.OldEntry, message.NewEntry)
				}
			}
			glog.V(2).Infof("update: %+v", resp)
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
			if err := client.DeleteFile(oldDest); err != nil {
				return err
			}
			remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewEntry, dest)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}

		return nil
	}
	return eachEntryFunc, nil
}

func retriedWriteFile(client remote_storage.RemoteStorageClient, filerSource *source.FilerSource, newEntry *filer_pb.Entry, dest *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
	var writeErr error
	err = util.Retry("writeFile", func() error {
		reader := filer.NewFileReader(filerSource, newEntry)
		glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
		remoteEntry, writeErr = client.WriteFile(dest, newEntry, reader)
		if writeErr != nil {
			return writeErr
		}
		return nil
	})
	glog.Errorf("write to %s: %v", dest, err)
	return
}

func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
	// 1. specified by timeAgo
	// 2. last offset timestamp for this directory
	// 3. directory creation time
	var lastOffsetTs time.Time
	if timeAgo == 0 {
		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
		if err != nil {
			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
			return time.Now()
		}

		lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
		if mountedDirEntry != nil {
			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
				glog.V(0).Infof("resume from %v", lastOffsetTs)
			} else {
				lastOffsetTs = time.Unix(mountedDirEntry.Attributes.Crtime, 0)
			}
		} else {
			lastOffsetTs = time.Now()
		}
	} else {
		lastOffsetTs = time.Now().Add(-timeAgo)
	}
	return lastOffsetTs
}

func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
	source := string(sourcePath[len(mountDir):])
	dest := util.FullPath(remoteMountLocation.Path).Child(source)
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteMountLocation.Name,
		Bucket: remoteMountLocation.Bucket,
		Path:   string(dest),
	}
}

func shouldSendToRemote(entry *filer_pb.Entry) bool {
	if entry.RemoteEntry == nil {
		return true
	}
	if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
		return true
	}
	return false
}

func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
	remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
	entry.RemoteEntry = remoteEntry
	return filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
		_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
			Directory: dir,
			Entry:     entry,
		})
		return err
	})
}
<|endoftext|>


<commit_before>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package sleuth

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	"github.com/ursiform/logger"
)

func init() {
	// Tests should run using a different group than production.
	group = "SLEUTH-vT"
}

// testCodes compares the error codes in an error with a list of wanted codes.
func testCodes(t *testing.T, err error, want []int) {
	codes := err.(*Error).Codes
	if len(codes) != len(want) {
		t.Errorf("expected codes length %d to be %d", len(codes), len(want))
		return
	}
	for i, code := range codes {
		if code != want[i] {
			t.Errorf("expected code [%d] to be [%d]", code, want[i])
			return
		}
	}
}

// Test client.go units.

func TestClientAddBadMember(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.add(group, "foo", "bar", "", "")
	if err == nil {
		t.Errorf("expected client dispatch to fail on bad action")
	}
	testCodes(t, err, []int{errAdd})
}

func TestClientDispatchBadAction(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.dispatch([]byte(group + "FAIL"))
	if err == nil {
		t.Errorf("expected client dispatch to fail on bad action")
	}
	testCodes(t, err, []int{errDispatchAction})
}

func TestClientDispatchEmpty(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.dispatch([]byte{})
	if err == nil {
		t.Errorf("expected client dispatch to fail on empty payload")
	}
	testCodes(t, err, []int{errDispatchHeader})
}

func TestClientDoUnknownScheme(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	req, _ := http.NewRequest("POST", "foo://bar/baz", nil)
	_, err := c.Do(req)
	if err == nil {
		t.Errorf("expected client Do to fail on unknown scheme")
	}
	testCodes(t, err, []int{errScheme})
}

func TestClientDoUnknownService(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	req, _ := http.NewRequest("POST", "sleuth://foo/bar", nil)
	_, err := c.Do(req)
	if err == nil {
		t.Errorf("expected client Do to fail on unknown service")
	}
	testCodes(t, err, []int{errUnknownService})
}

func TestClientReceiveBadHandle(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	res := &response{Handle: "foo"}
	err := c.receive(marshalResponse(res)[len(group)+len(recv):])
	if err == nil {
		t.Errorf("expected client receive to fail on bad handle")
	}
	testCodes(t, err, []int{errRECV})
}

func TestClientReceiveBadPayload(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.receive([]byte(""))
	if err == nil {
		t.Errorf("expected client receive to fail on bad payload")
	}
	testCodes(t, err, []int{errUnzip, errResUnmarshal, errRECV})
}

func TestClientReplyBadPayload(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.reply([]byte(""))
	if err == nil {
		t.Errorf("expected client reply to fail on bad payload")
	}
	testCodes(t, err, []int{errUnzip, errReqUnmarshal, errREPL})
}

// Test error.go units.

func TestError(t *testing.T) {
	code := 1
	message := "test"
	want := "sleuth: test [1]"
	err := newError(code, message)
	if err.Error() != want {
		t.Errorf("expected error to be formatted as: %s", want)
	}
}

// Test request.go units.

func TestUnmarshalRequestBadJSON(t *testing.T) {
	payload := zip([]byte("{bad json}"))
	_, _, err := unmarshalRequest(payload)
	if err == nil {
		t.Errorf("expected unmarshalRequest to fail on bad json")
	}
	testCodes(t, err, []int{errReqUnmarshalJSON})
}

// Test response.go units.

func TestUnmarshalResponseBadJSON(t *testing.T) {
	payload := zip([]byte("{bad json}"))
	_, _, err := unmarshalResponse(payload)
	if err == nil {
		t.Errorf("expected unmarshalResponse to fail on bad json")
	}
	testCodes(t, err, []int{errResUnmarshalJSON})
}

// Test sleuth.go units.

func TestNewBadInterface(t *testing.T) {
	_, err := New(&Config{Interface: "foo"})
	if err == nil {
		t.Errorf("expected New to fail on start with bad interface")
	}
	testCodes(t, err, []int{errStart, errCreate, errNew})
}

func TestNewBadLogLevel(t *testing.T) {
	c, _ := New(&Config{LogLevel: "foo"})
	if c.log.Level() != logger.Debug {
		t.Errorf("expected log level 'foo' to be coerced to 'debug'")
		return
	}
}

func TestNewBadPort(t *testing.T) {
	_, err := New(&Config{Port: 1})
	if err == nil {
		t.Errorf("expected New to fail on start with bad port")
	}
	testCodes(t, err, []int{errStart, errCreate, errNew})
}

func TestNewBadService(t *testing.T) {
	_, err := New(&Config{Handler: http.FileServer(http.Dir("."))})
	if err == nil {
		t.Errorf("expected New to fail with bad service")
	}
	testCodes(t, err, []int{errService})
}

// Test workers.go units.

func TestWorkersAddDuplicate(t *testing.T) {
	w := newWorkers()
	p := &peer{name: "foo", node: "bar", service: "baz"}
	if n := w.add(p); n != w.add(p) {
		t.Error("expected duplicate addition to be ignored")
	}
}

func TestWorkersAvailable(t *testing.T) {
	w := newWorkers()
	p := &peer{name: "foo", node: "bar", service: "baz"}
	if w.add(p); !w.available() {
		t.Error("expected workers to be available")
	}
}

func TestWorkersNext(t *testing.T) {
	w := newWorkers()
	w.add(&peer{name: "foo", node: "bar", service: "baz"})
	w.add(&peer{name: "qux", node: "quux", service: "corge"})
	if w.next().name != "foo" {
		t.Error("expected next to return first added item")
	}
	if w.next().name != "qux" {
		t.Error("expected next to return second added item")
	}
	if w.next().name != "foo" {
		t.Error("expected next to loop back to first added item")
	}
}

func TestWorkersNextNonexistent(t *testing.T) {
	w := newWorkers()
	if p := w.next(); p != nil {
		t.Error("expected nonexistent worker to be nil")
	}
}

func TestWorkersRemove(t *testing.T) {
	w := newWorkers()
	w.add(&peer{name: "foo", node: "bar", service: "baz"})
	if _, p := w.remove("foo"); p == nil || p.name != "foo" || w.available() {
		t.Error("expected worker to be removed")
	}
}

func TestWorkersRemoveNonexistent(t *testing.T) {
	w := newWorkers()
	if _, p := w.remove("foo"); p != nil {
		t.Error("expected nonexistent worker removal to return nil")
	}
}

// Test writer.go units.

type goodWhisperer struct{}

func (g *goodWhisperer) Whisper(addr string, payload []byte) error {
	return nil
}

type badWhisperer struct{}

func (b *badWhisperer) Whisper(addr string, payload []byte) error {
	return errors.New("bad whisperer error")
}

func TestWriterWrite(t *testing.T) {
	data := []byte("foo bar baz")
	w := newWriter(new(goodWhisperer), &destination{node: "qux", handle: "quux"})
	if n, err := w.Write(data); err != nil {
		t.Errorf("expected write to succeed: %s", err.Error())
	} else if n <= 0 {
		t.Errorf("expected (%d) to be greater than 0", n)
	}
}

func TestWriterErrWhisper(t *testing.T) {
	data := []byte("foo bar baz")
	w := newWriter(new(badWhisperer), &destination{node: "qux", handle: "quux"})
	if _, err := w.Write(data); err == nil {
		t.Errorf("expected writer to fail")
	}
}

// Test zip.go units.

func TestZipUnzip(t *testing.T) {
	in := []byte("a value that should be zipped")
	zipped := zip(in)
	if out, err := unzip(zipped); err != nil {
		t.Errorf("unzip failed: ", err.Error())
	} else if string(out) != string(in) {
		t.Errorf("zip failed")
	}
}

func TestUnzipErr(t *testing.T) {
	in := []byte("a value that cannot be unzipped")
	_, err := unzip(in)
	testCodes(t, err, []int{errUnzip})
}

// Test integrated package.

type echoHandler struct{}

func (h *echoHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	body, _ := ioutil.ReadAll(req.Body)
	res.Write(body)
}

type silentHandler struct{}

func (*silentHandler) ServeHTTP(http.ResponseWriter, *http.Request) {
	// Silent handler does nothing.
}

func TestRequestResponseCycle(t *testing.T) {
	// Create client.
	client, err := New(nil)
	if err != nil {
		t.Errorf("client instantiation failed: %s", err.Error())
		return
	}
	defer func(client *Client, t *testing.T) {
		if err := client.Close(); err != nil {
			t.Errorf("client close failed: %s", err.Error())
		}
	}(client, t)
	// Create server.
	addr := "sleuth-test-server-one"
	server, err := New(&Config{
		Handler: new(echoHandler), Service: addr})
	if err != nil {
		t.Errorf("server instantiation failed: %s", err.Error())
		return
	}
	defer func(server *Client, t *testing.T) {
		if err := server.Close(); err != nil {
			t.Errorf("server close failed: %s", err.Error())
		}
	}(server, t)
	// Wait until the server has been added to the client pool.
	client.WaitFor(addr)
	// Set timeout to 10 seconds to accommodate slow test spin-up.
	client.Timeout = time.Second * 10
	if client.block(addr) {
		t.Errorf("call to block should have returned immediately")
	}
	body := "foo bar baz"
	buffer := bytes.NewBuffer([]byte(body))
	request, err := http.NewRequest("GET", scheme+"://"+addr+"/", buffer)
	if err != nil {
		t.Errorf("request instantiation failed: %s", err.Error())
		return
	}
	response, err := client.Do(request)
	if err != nil {
		t.Errorf("client.Do failed: %s", err.Error())
		return
	}
	if response.StatusCode != http.StatusOK {
		t.Errorf("client.Do expected %d got %d", http.StatusOK, response.StatusCode)
		return
	}
	output, _ := ioutil.ReadAll(response.Body)
	// It's not really necessary to close a sleuth response.
	response.Body.Close()
	if string(output) != body {
		t.Errorf("client.Do expected %s to equal %s", string(output), body)
		return
	}
}

func TestTimeout(t *testing.T) {
	// Create server.
	addr := "sleuth-test-server-two"
	server, err := New(&Config{Handler: new(silentHandler), Service: addr})
	if err != nil {
		t.Errorf("server instantiation failed: %s", err.Error())
		return
	}
	defer func(server *Client, t *testing.T) {
		if err := server.Close(); err != nil {
			t.Errorf("server close failed: %s", err.Error())
		}
	}(server, t)
	// Create client.
	client, err := New(nil)
	if err != nil {
		t.Errorf("client instantiation failed: %s", err.Error())
		return
	}
	defer func(client *Client, t *testing.T) {
		if err := client.Close(); err != nil {
			t.Errorf("client close failed: %s", err.Error())
		}
	}(client, t)
	// Wait long enough that the server should be ready (but not guaranteed).
	// If the server is ready, then WaitFor will not block.
	<-time.After(2000 * time.Millisecond)
	// Wait until the server has been added to the client pool.
	client.WaitFor(addr)
	request, err := http.NewRequest("GET", scheme+"://"+addr+"/", nil)
	if err != nil {
		t.Errorf("request instantiation failed: %s", err.Error())
		return
	}
	_, err = client.Do(request)
	if err == nil {
		t.Errorf("client request should have timed out")
		return
	}
}
<commit_msg>Bugfix: add directive to Errorf call<commit_after>// Copyright 2016 Afshin Darian. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package sleuth

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"testing"
	"time"

	"github.com/ursiform/logger"
)

func init() {
	// Tests should run using a different group than production.
	group = "SLEUTH-vT"
}

// testCodes compares the error codes in an error with a list of wanted codes.
func testCodes(t *testing.T, err error, want []int) {
	codes := err.(*Error).Codes
	if len(codes) != len(want) {
		t.Errorf("expected codes length %d to be %d", len(codes), len(want))
		return
	}
	for i, code := range codes {
		if code != want[i] {
			t.Errorf("expected code [%d] to be [%d]", code, want[i])
			return
		}
	}
}

// Test client.go units.

func TestClientAddBadMember(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.add(group, "foo", "bar", "", "")
	if err == nil {
		t.Errorf("expected client dispatch to fail on bad action")
	}
	testCodes(t, err, []int{errAdd})
}

func TestClientDispatchBadAction(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.dispatch([]byte(group + "FAIL"))
	if err == nil {
		t.Errorf("expected client dispatch to fail on bad action")
	}
	testCodes(t, err, []int{errDispatchAction})
}

func TestClientDispatchEmpty(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.dispatch([]byte{})
	if err == nil {
		t.Errorf("expected client dispatch to fail on empty payload")
	}
	testCodes(t, err, []int{errDispatchHeader})
}

func TestClientDoUnknownScheme(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	req, _ := http.NewRequest("POST", "foo://bar/baz", nil)
	_, err := c.Do(req)
	if err == nil {
		t.Errorf("expected client Do to fail on unknown scheme")
	}
	testCodes(t, err, []int{errScheme})
}

func TestClientDoUnknownService(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	req, _ := http.NewRequest("POST", "sleuth://foo/bar", nil)
	_, err := c.Do(req)
	if err == nil {
		t.Errorf("expected client Do to fail on unknown service")
	}
	testCodes(t, err, []int{errUnknownService})
}

func TestClientReceiveBadHandle(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	res := &response{Handle: "foo"}
	err := c.receive(marshalResponse(res)[len(group)+len(recv):])
	if err == nil {
		t.Errorf("expected client receive to fail on bad handle")
	}
	testCodes(t, err, []int{errRECV})
}

func TestClientReceiveBadPayload(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.receive([]byte(""))
	if err == nil {
		t.Errorf("expected client receive to fail on bad payload")
	}
	testCodes(t, err, []int{errUnzip, errResUnmarshal, errRECV})
}

func TestClientReplyBadPayload(t *testing.T) {
	log, _ := logger.New(logger.Silent)
	c := newClient(nil, log)
	err := c.reply([]byte(""))
	if err == nil {
		t.Errorf("expected client reply to fail on bad payload")
	}
	testCodes(t, err, []int{errUnzip, errReqUnmarshal, errREPL})
}

// Test error.go units.

func TestError(t *testing.T) {
	code := 1
	message := "test"
	want := "sleuth: test [1]"
	err := newError(code, message)
	if err.Error() != want {
		t.Errorf("expected error to be formatted as: %s", want)
	}
}

// Test request.go units.

func TestUnmarshalRequestBadJSON(t *testing.T) {
	payload := zip([]byte("{bad json}"))
	_, _, err := unmarshalRequest(payload)
	if err == nil {
		t.Errorf("expected unmarshalRequest to fail on bad json")
	}
	testCodes(t, err, []int{errReqUnmarshalJSON})
}

// Test response.go units.

func TestUnmarshalResponseBadJSON(t *testing.T) {
	payload := zip([]byte("{bad json}"))
	_, _, err := unmarshalResponse(payload)
	if err == nil {
		t.Errorf("expected unmarshalResponse to fail on bad json")
	}
	testCodes(t, err, []int{errResUnmarshalJSON})
}

// Test sleuth.go units.

func TestNewBadInterface(t *testing.T) {
	_, err := New(&Config{Interface: "foo"})
	if err == nil {
		t.Errorf("expected New to fail on start with bad interface")
	}
	testCodes(t, err, []int{errStart, errCreate, errNew})
}

func TestNewBadLogLevel(t *testing.T) {
	c, _ := New(&Config{LogLevel: "foo"})
	if c.log.Level() != logger.Debug {
		t.Errorf("expected log level 'foo' to be coerced to 'debug'")
		return
	}
}

func TestNewBadPort(t *testing.T) {
	_, err := New(&Config{Port: 1})
	if err == nil {
		t.Errorf("expected New to fail on start with bad port")
	}
	testCodes(t, err, []int{errStart, errCreate, errNew})
}

func TestNewBadService(t *testing.T) {
	_, err := New(&Config{Handler: http.FileServer(http.Dir("."))})
	if err == nil {
		t.Errorf("expected New to fail with bad service")
	}
	testCodes(t, err, []int{errService})
}

// Test workers.go units.

func TestWorkersAddDuplicate(t *testing.T) {
	w := newWorkers()
	p := &peer{name: "foo", node: "bar", service: "baz"}
	if n := w.add(p); n != w.add(p) {
		t.Error("expected duplicate addition to be ignored")
	}
}

func TestWorkersAvailable(t *testing.T) {
	w := newWorkers()
	p := &peer{name: "foo", node: "bar", service: "baz"}
	if w.add(p); !w.available() {
		t.Error("expected workers to be available")
	}
}

func TestWorkersNext(t *testing.T) {
	w := newWorkers()
	w.add(&peer{name: "foo", node: "bar", service: "baz"})
	w.add(&peer{name: "qux", node: "quux", service: "corge"})
	if w.next().name != "foo" {
		t.Error("expected next to return first added item")
	}
	if w.next().name != "qux" {
		t.Error("expected next to return second added item")
	}
	if w.next().name != "foo" {
		t.Error("expected next to loop back to first added item")
	}
}

func TestWorkersNextNonexistent(t *testing.T) {
	w := newWorkers()
	if p := w.next(); p != nil {
		t.Error("expected nonexistent worker to be nil")
	}
}

func TestWorkersRemove(t *testing.T) {
	w := newWorkers()
	w.add(&peer{name: "foo", node: "bar", service: "baz"})
	if _, p := w.remove("foo"); p == nil || p.name != "foo" || w.available() {
		t.Error("expected worker to be removed")
	}
}

func TestWorkersRemoveNonexistent(t *testing.T) {
	w := newWorkers()
	if _, p := w.remove("foo"); p != nil {
		t.Error("expected nonexistent worker removal to return nil")
	}
}

// Test writer.go units.

type goodWhisperer struct{}

func (g *goodWhisperer) Whisper(addr string, payload []byte) error {
	return nil
}

type badWhisperer struct{}

func (b *badWhisperer) Whisper(addr string, payload []byte) error {
	return errors.New("bad whisperer error")
}

func TestWriterWrite(t *testing.T) {
	data := []byte("foo bar baz")
	w := newWriter(new(goodWhisperer), &destination{node: "qux", handle: "quux"})
	if n, err := w.Write(data); err != nil {
		t.Errorf("expected write to succeed: %s", err.Error())
	} else if n <= 0 {
		t.Errorf("expected (%d) to be greater than 0", n)
	}
}

func TestWriterErrWhisper(t *testing.T) {
	data := []byte("foo bar baz")
	w := newWriter(new(badWhisperer), &destination{node: "qux", handle: "quux"})
	if _, err := w.Write(data); err == nil {
		t.Errorf("expected writer to fail")
	}
}

// Test zip.go units.

func TestZipUnzip(t *testing.T) {
	in := []byte("a value that should be zipped")
	zipped := zip(in)
	if out, err := unzip(zipped); err != nil {
		t.Errorf("unzip failed: %s", err.Error())
	} else if string(out) != string(in) {
		t.Errorf("zip failed")
	}
}

func TestUnzipErr(t *testing.T) {
	in := []byte("a value that cannot be unzipped")
	_, err := unzip(in)
	testCodes(t, err, []int{errUnzip})
}

// Test integrated package.

type echoHandler struct{}

func (h *echoHandler) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	body, _ := ioutil.ReadAll(req.Body)
	res.Write(body)
}

type silentHandler struct{}

func (*silentHandler) ServeHTTP(http.ResponseWriter, *http.Request) {
	// Silent handler does nothing.
}

func TestRequestResponseCycle(t *testing.T) {
	// Create client.
	client, err := New(nil)
	if err != nil {
		t.Errorf("client instantiation failed: %s", err.Error())
		return
	}
	defer func(client *Client, t *testing.T) {
		if err := client.Close(); err != nil {
			t.Errorf("client close failed: %s", err.Error())
		}
	}(client, t)
	// Create server.
	addr := "sleuth-test-server-one"
	server, err := New(&Config{
		Handler: new(echoHandler), Service: addr})
	if err != nil {
		t.Errorf("server instantiation failed: %s", err.Error())
		return
	}
	defer func(server *Client, t *testing.T) {
		if err := server.Close(); err != nil {
			t.Errorf("server close failed: %s", err.Error())
		}
	}(server, t)
	// Wait until the server has been added to the client pool.
	client.WaitFor(addr)
	// Set timeout to 10 seconds to accommodate slow test spin-up.
	client.Timeout = time.Second * 10
	if client.block(addr) {
		t.Errorf("call to block should have returned immediately")
	}
	body := "foo bar baz"
	buffer := bytes.NewBuffer([]byte(body))
	request, err := http.NewRequest("GET", scheme+"://"+addr+"/", buffer)
	if err != nil {
		t.Errorf("request instantiation failed: %s", err.Error())
		return
	}
	response, err := client.Do(request)
	if err != nil {
		t.Errorf("client.Do failed: %s", err.Error())
		return
	}
	if response.StatusCode != http.StatusOK {
		t.Errorf("client.Do expected %d got %d", http.StatusOK, response.StatusCode)
		return
	}
	output, _ := ioutil.ReadAll(response.Body)
	// It's not really necessary to close a sleuth response.
	response.Body.Close()
	if string(output) != body {
		t.Errorf("client.Do expected %s to equal %s", string(output), body)
		return
	}
}

func TestTimeout(t *testing.T) {
	// Create server.
	addr := "sleuth-test-server-two"
	server, err := New(&Config{Handler: new(silentHandler), Service: addr})
	if err != nil {
		t.Errorf("server instantiation failed: %s", err.Error())
		return
	}
	defer func(server *Client, t *testing.T) {
		if err := server.Close(); err != nil {
			t.Errorf("server close failed: %s", err.Error())
		}
	}(server, t)
	// Create client.
	client, err := New(nil)
	if err != nil {
		t.Errorf("client instantiation failed: %s", err.Error())
		return
	}
	defer func(client *Client, t *testing.T) {
		if err := client.Close(); err != nil {
			t.Errorf("client close failed: %s", err.Error())
		}
	}(client, t)
	// Wait long enough that the server should be ready (but not guaranteed).
	// If the server is ready, then WaitFor will not block.
	<-time.After(2000 * time.Millisecond)
	// Wait until the server has been added to the client pool.
	client.WaitFor(addr)
	request, err := http.NewRequest("GET", scheme+"://"+addr+"/", nil)
	if err != nil {
		t.Errorf("request instantiation failed: %s", err.Error())
		return
	}
	_, err = client.Do(request)
	if err == nil {
		t.Errorf("client request should have timed out")
		return
	}
}
<|endoftext|>


<commit_before>package spi

import (
	"reflect"
	"rtos"
	"sync/atomic"
	"sync/fence"
	"unsafe"

	"stm32/hal/dma"
)

type DriverError byte

const ErrTimeout DriverError = 1

func (e DriverError) Error() string {
	switch e {
	case ErrTimeout:
		return "timeout"
	default:
		return ""
	}
}

type Driver struct {
	deadline int64
	p        *Periph
	rxDMA    *dma.Channel
	txDMA    *dma.Channel
	dmacnt   int
	done     rtos.EventFlag
	err      uint32
}

// NewDriver provides convenient way to create heap allocated Driver struct.
func NewDriver(p *Periph, txdma, rxdma *dma.Channel) *Driver {
	d := new(Driver)
	d.p = p
	d.rxDMA = rxdma
	d.txDMA = txdma
	return d
}

func (d *Driver) Periph() *Periph {
	return d.p
}

func (d *Driver) TxDMA() *dma.Channel {
	return d.txDMA
}

func (d *Driver) RxDMA() *dma.Channel {
	return d.rxDMA
}

func (d *Driver) DMAISR(ch *dma.Channel) {
	ev, err := ch.Status()
	if err&^dma.ErrFIFO != 0 {
		goto done
	}
	if ev&dma.Complete != 0 {
		ch.Clear(dma.Complete, 0)
		if atomic.AddInt(&d.dmacnt, -1) == 0 {
			goto done
		}
	}
	return
done:
	ch.DisableIRQ(dma.EvAll, dma.ErrAll)
	d.done.Signal(1)
}

func (d *Driver) ISR() {
	d.p.DisableIRQ(RxNotEmpty | Err)
	d.done.Signal(1)
}

func (d *Driver) SetDeadline(deadline int64) {
	d.deadline = deadline
}

// WriteReadByte writes and reads byte.
func (d *Driver) WriteReadByte(b byte) byte {
	if d.err != 0 {
		return 0
	}
	p := d.p
	p.SetDuplex(Full)
	d.done.Reset(0)
	p.EnableIRQ(RxNotEmpty | Err)
	fence.W() // This orders writes to normal and I/O memory.
	p.StoreByte(b)
	if !d.done.Wait(1, d.deadline) {
		d.err = uint32(ErrTimeout) << 16
		return 0
	}
	b = p.LoadByte()
	if _, e := p.Status(); e != 0 {
		d.err = uint32(e) << 8
		return 0
	}
	return b
}

// WriteReadWord16 writes and reads 16-bit word.
func (d *Driver) WriteReadWord16(w uint16) uint16 {
	if d.err != 0 {
		return 0
	}
	p := d.p
	p.SetDuplex(Full)
	d.done.Reset(0)
	p.EnableIRQ(RxNotEmpty | Err)
	fence.W() // This orders writes to normal and I/O memory.
	p.StoreWord16(w)
	if !d.done.Wait(1, d.deadline) {
		d.err = uint32(ErrTimeout) << 16
		return 0
	}
	w = p.LoadWord16()
	if _, e := p.Status(); e != 0 {
		d.err = uint32(e) << 8
		return 0
	}
	return w
}

func (d *Driver) setupDMA(ch *dma.Channel, mode dma.Mode, wordSize uintptr) {
	ch.Setup(mode)
	ch.SetWordSize(wordSize, wordSize)
	ch.SetAddrP(unsafe.Pointer(d.p.raw.DR.Addr()))
}

func startDMA(ch *dma.Channel, addr uintptr, n int) {
	ch.SetAddrM(unsafe.Pointer(addr))
	ch.SetLen(n)
	ch.Clear(dma.EvAll, dma.ErrAll)
	ch.EnableIRQ(dma.Complete, dma.ErrAll&^dma.ErrFIFO)
	fence.W() // This orders writes to normal and I/O memory.
	ch.Enable()
}

func (d *Driver) writeReadDMA(out, in uintptr, olen, ilen int, wsize uintptr) (n int) {
	txdmacfg := dma.MTP | dma.FT4
	if olen > 1 {
		txdmacfg |= dma.IncM
	}
	d.setupDMA(d.txDMA, txdmacfg, 1)
	d.setupDMA(d.rxDMA, dma.PTM|dma.IncM|dma.FT4, wsize)
	p := d.p
	p.SetDuplex(Full)
	p.EnableDMA(RxNotEmpty | TxEmpty)
	p.EnableIRQ(Err)
	for {
		m := ilen - n
		if m == 0 {
			break
		}
		if m > 0xffff {
			m = 0xffff
		}
		d.dmacnt = 2
		d.done.Reset(0)
		startDMA(d.rxDMA, in, m)
		startDMA(d.txDMA, out, m)
		if olen > 1 {
			out += uintptr(m)
		}
		in += uintptr(m)
		n += m
		done := d.done.Wait(1, d.deadline)
		d.txDMA.Disable() // Required by STM32F1 to allow setup next transfer.
		d.rxDMA.Disable() // Required by STM32F1 to allow setup next transfer.
		if !done {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.rxDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(ErrTimeout) << 16
			n -= d.rxDMA.Len()
			break
		}
		if _, e := p.Status(); e != 0 {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.rxDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(e) << 8
			n -= d.rxDMA.Len()
			break
		}
		_, rxe := d.rxDMA.Status()
		_, txe := d.txDMA.Status()
		if e := (rxe | txe) &^ dma.ErrFIFO; e != 0 {
			d.err = uint32(e)
			n -= d.rxDMA.Len()
			break
		}
	}
	p.DisableDMA(RxNotEmpty | TxEmpty)
	p.DisableIRQ(Err)
	return
}

func (d *Driver) writeDMA(out uintptr, n int, wsize uintptr, incm dma.Mode) {
	d.setupDMA(d.txDMA, dma.MTP|incm|dma.FT4, wsize)
	p := d.p
	p.SetDuplex(HalfOut) // Avoid ErrOverflow.
	p.EnableDMA(TxEmpty)
	p.EnableIRQ(Err)
	for n > 0 {
		m := n
		if m > 0xffff {
			m = 0xffff
		}
		d.dmacnt = 1
		d.done.Reset(0)
		startDMA(d.txDMA, out, m)
		n -= m
		if incm != 0 {
			out += uintptr(m)
		}
		done := d.done.Wait(1, d.deadline)
		d.txDMA.Disable() // Required by STM32F1 to allow setup next transfer
		if !done {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(ErrTimeout) << 16
			break
		}
		if _, e := p.Status(); e != 0 {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(e) << 8
			break
		}
		_, txe := d.txDMA.Status()
		if e := txe &^ dma.ErrFIFO; e != 0 {
			d.err = uint32(e)
			break
		}
	}
	p.DisableDMA(TxEmpty)
	p.DisableIRQ(Err)
	// Now DMA finished but SPI can still send buffered data. Wait for end.
	for {
		if ev, _ := p.Status(); ev&Busy == 0 {
			break
		}
		rtos.SchedYield()
	}
}

// Err returns value of internal error variable and clears it if clear is true.
func (d *Driver) Err(clear bool) error {
	e := d.err
	if e == 0 {
		return nil
	}
	if clear {
		d.err = 0
	}
	if err := DriverError(e >> 16); err != 0 {
		return err
	}
	if err := Error(e >> 8); err != 0 {
		if err&ErrOverrun != 0 && clear {
			d.p.LoadByte()
			d.p.Status()
		}
		return err
	}
	return dma.Error(e)
}

func (d *Driver) writeRead(oaddr, iaddr uintptr, olen, ilen int, wsize uintptr) int {
	if olen > ilen {
		var n int
		if ilen > 0 {
			n = d.writeReadDMA(oaddr, iaddr, ilen, ilen, wsize)
			if d.err != 0 {
				return n
			}
			olen -= ilen
			oaddr += uintptr(ilen)
		}
		d.writeDMA(oaddr, olen, wsize, dma.IncM)
		return n
	}
	if ilen > olen {
		var n int
		ffff := uint16(0xffff)
		if olen > 0 {
			n = d.writeReadDMA(oaddr, iaddr, olen, olen, wsize)
			if d.err != 0 {
				return n
			}
			ilen -= olen
			iaddr += uintptr(olen)
			oaddr += uintptr(olen - 1)
		} else {
			oaddr = uintptr(unsafe.Pointer(&ffff))
		}
		return n + d.writeReadDMA(oaddr, iaddr, 1, ilen, wsize)
	}
	return d.writeReadDMA(oaddr, iaddr, ilen, ilen, wsize)
}

func (d *Driver) WriteStringRead(out string, in []byte) int {
	olen := len(out)
	ilen := len(in)
	if d.err != 0 || olen == 0 && ilen == 0 {
		return 0
	}
	if olen <= 1 && ilen <= 1 {
		// Avoid DMA for one byte transfers.
		b := byte(0xff)
		if olen != 0 {
			b = out[0]
		}
		b = d.WriteReadByte(b)
		if ilen != 0 {
			in[0] = b
			return 1
		}
		return 0
	}
	oaddr := (*reflect.StringHeader)(unsafe.Pointer(&out)).Data
	iaddr := (*reflect.SliceHeader)(unsafe.Pointer(&in)).Data
	return d.writeRead(oaddr, iaddr, olen, ilen, 1)
}

func (d *Driver) WriteRead(out, in []byte) int {
	return d.WriteStringRead(*(*string)(unsafe.Pointer(&out)), in)
}

func (d *Driver) WriteReadMany(oi ...[]byte) int {
	var n int
	for k := 0; k < len(oi); k += 2 {
		var in []byte
		if k+1 < len(oi) {
			in = oi[k+1]
		}
		out := oi[k]
		n += d.WriteRead(out, in)
	}
	return n
}

func (d *Driver) RepeatByte(b byte, n int) {
	if d.err != 0 {
		return
	}
	switch {
	case n > 1:
		d.writeDMA(uintptr(unsafe.Pointer(&b)), n, 1, 0)
	case n == 1:
		// Avoid DMA for one byte transfers.
		d.WriteReadByte(b)
	}
}

func (d *Driver) WriteRead16(out, in []uint16) int {
	olen := len(out)
	ilen := len(in)
	if d.err != 0 || olen == 0 && ilen == 0 {
		return 0
	}
	if olen <= 1 && ilen <= 1 {
		// Avoid DMA for one word transfers.
		w := uint16(0xffff)
		if olen != 0 {
			w = out[0]
		}
		w = d.WriteReadWord16(w)
		if ilen != 0 {
			in[0] = w
			return 1
		}
		return 0
	}
	oaddr := (*reflect.SliceHeader)(unsafe.Pointer(&out)).Data
	iaddr := (*reflect.SliceHeader)(unsafe.Pointer(&in)).Data
	return d.writeRead(oaddr, iaddr, olen, ilen, 2)
}

func (d *Driver) WriteReadMany16(oi ...[]uint16) int {
	var n int
	for k := 0; k < len(oi); k += 2 {
		var in []uint16
		if k+1 < len(oi) {
			in = oi[k+1]
		}
		out := oi[k]
		n += d.WriteRead16(out, in)
	}
	return n
}

func (d *Driver) RepeatWord16(w uint16, n int) {
	if d.err != 0 {
		return
	}
	switch {
	case n > 1:
		d.writeDMA(uintptr(unsafe.Pointer(&w)), n, 2, 0)
	case n == 1:
		// Avoid DMA for one word transfers.
		d.WriteReadWord16(w)
	}
}
<commit_msg>stm32/hal/spi: New MakeDriver method.<commit_after>package spi

import (
	"reflect"
	"rtos"
	"sync/atomic"
	"sync/fence"
	"unsafe"

	"stm32/hal/dma"
)

type DriverError byte

const ErrTimeout DriverError = 1

func (e DriverError) Error() string {
	switch e {
	case ErrTimeout:
		return "timeout"
	default:
		return ""
	}
}

type Driver struct {
	deadline int64
	p        *Periph
	rxDMA    *dma.Channel
	txDMA    *dma.Channel
	dmacnt   int
	done     rtos.EventFlag
	err      uint32
}

// MakeDriver returns initialized SPI driver that uses provided SPI peripheral
// and DMA channels.
func MakeDriver(p *Periph, txdma, rxdma *dma.Channel) Driver {
	return Driver{p: p, rxDMA: rxdma, txDMA: txdma}
}

// NewDriver provides convenient way to create heap allocated Driver struct.
func NewDriver(p *Periph, txdma, rxdma *dma.Channel) *Driver {
	d := new(Driver)
	*d = MakeDriver(p, txdma, rxdma)
	return d
}

func (d *Driver) Periph() *Periph {
	return d.p
}

func (d *Driver) TxDMA() *dma.Channel {
	return d.txDMA
}

func (d *Driver) RxDMA() *dma.Channel {
	return d.rxDMA
}

func (d *Driver) DMAISR(ch *dma.Channel) {
	ev, err := ch.Status()
	if err&^dma.ErrFIFO != 0 {
		goto done
	}
	if ev&dma.Complete != 0 {
		ch.Clear(dma.Complete, 0)
		if atomic.AddInt(&d.dmacnt, -1) == 0 {
			goto done
		}
	}
	return
done:
	ch.DisableIRQ(dma.EvAll, dma.ErrAll)
	d.done.Signal(1)
}

func (d *Driver) ISR() {
	d.p.DisableIRQ(RxNotEmpty | Err)
	d.done.Signal(1)
}

func (d *Driver) SetDeadline(deadline int64) {
	d.deadline = deadline
}

// WriteReadByte writes and reads byte.
func (d *Driver) WriteReadByte(b byte) byte {
	if d.err != 0 {
		return 0
	}
	p := d.p
	p.SetDuplex(Full)
	d.done.Reset(0)
	p.EnableIRQ(RxNotEmpty | Err)
	fence.W() // This orders writes to normal and I/O memory.
	p.StoreByte(b)
	if !d.done.Wait(1, d.deadline) {
		d.err = uint32(ErrTimeout) << 16
		return 0
	}
	b = p.LoadByte()
	if _, e := p.Status(); e != 0 {
		d.err = uint32(e) << 8
		return 0
	}
	return b
}

// WriteReadWord16 writes and reads 16-bit word.
func (d *Driver) WriteReadWord16(w uint16) uint16 {
	if d.err != 0 {
		return 0
	}
	p := d.p
	p.SetDuplex(Full)
	d.done.Reset(0)
	p.EnableIRQ(RxNotEmpty | Err)
	fence.W() // This orders writes to normal and I/O memory.
	p.StoreWord16(w)
	if !d.done.Wait(1, d.deadline) {
		d.err = uint32(ErrTimeout) << 16
		return 0
	}
	w = p.LoadWord16()
	if _, e := p.Status(); e != 0 {
		d.err = uint32(e) << 8
		return 0
	}
	return w
}

func (d *Driver) setupDMA(ch *dma.Channel, mode dma.Mode, wordSize uintptr) {
	ch.Setup(mode)
	ch.SetWordSize(wordSize, wordSize)
	ch.SetAddrP(unsafe.Pointer(d.p.raw.DR.Addr()))
}

func startDMA(ch *dma.Channel, addr uintptr, n int) {
	ch.SetAddrM(unsafe.Pointer(addr))
	ch.SetLen(n)
	ch.Clear(dma.EvAll, dma.ErrAll)
	ch.EnableIRQ(dma.Complete, dma.ErrAll&^dma.ErrFIFO)
	fence.W() // This orders writes to normal and I/O memory.
	ch.Enable()
}

func (d *Driver) writeReadDMA(out, in uintptr, olen, ilen int, wsize uintptr) (n int) {
	txdmacfg := dma.MTP | dma.FT4
	if olen > 1 {
		txdmacfg |= dma.IncM
	}
	d.setupDMA(d.txDMA, txdmacfg, 1)
	d.setupDMA(d.rxDMA, dma.PTM|dma.IncM|dma.FT4, wsize)
	p := d.p
	p.SetDuplex(Full)
	p.EnableDMA(RxNotEmpty | TxEmpty)
	p.EnableIRQ(Err)
	for {
		m := ilen - n
		if m == 0 {
			break
		}
		if m > 0xffff {
			m = 0xffff
		}
		d.dmacnt = 2
		d.done.Reset(0)
		startDMA(d.rxDMA, in, m)
		startDMA(d.txDMA, out, m)
		if olen > 1 {
			out += uintptr(m)
		}
		in += uintptr(m)
		n += m
		done := d.done.Wait(1, d.deadline)
		d.txDMA.Disable() // Required by STM32F1 to allow setup next transfer.
		d.rxDMA.Disable() // Required by STM32F1 to allow setup next transfer.
		if !done {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.rxDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(ErrTimeout) << 16
			n -= d.rxDMA.Len()
			break
		}
		if _, e := p.Status(); e != 0 {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.rxDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(e) << 8
			n -= d.rxDMA.Len()
			break
		}
		_, rxe := d.rxDMA.Status()
		_, txe := d.txDMA.Status()
		if e := (rxe | txe) &^ dma.ErrFIFO; e != 0 {
			d.err = uint32(e)
			n -= d.rxDMA.Len()
			break
		}
	}
	p.DisableDMA(RxNotEmpty | TxEmpty)
	p.DisableIRQ(Err)
	return
}

func (d *Driver) writeDMA(out uintptr, n int, wsize uintptr, incm dma.Mode) {
	d.setupDMA(d.txDMA, dma.MTP|incm|dma.FT4, wsize)
	p := d.p
	p.SetDuplex(HalfOut) // Avoid ErrOverflow.
	p.EnableDMA(TxEmpty)
	p.EnableIRQ(Err)
	for n > 0 {
		m := n
		if m > 0xffff {
			m = 0xffff
		}
		d.dmacnt = 1
		d.done.Reset(0)
		startDMA(d.txDMA, out, m)
		n -= m
		if incm != 0 {
			out += uintptr(m)
		}
		done := d.done.Wait(1, d.deadline)
		d.txDMA.Disable() // Required by STM32F1 to allow setup next transfer
		if !done {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(ErrTimeout) << 16
			break
		}
		if _, e := p.Status(); e != 0 {
			d.txDMA.DisableIRQ(dma.EvAll, dma.ErrAll)
			d.err = uint32(e) << 8
			break
		}
		_, txe := d.txDMA.Status()
		if e := txe &^ dma.ErrFIFO; e != 0 {
			d.err = uint32(e)
			break
		}
	}
	p.DisableDMA(TxEmpty)
	p.DisableIRQ(Err)
	// Now DMA finished but SPI can still send buffered data. Wait for end.
	for {
		if ev, _ := p.Status(); ev&Busy == 0 {
			break
		}
		rtos.SchedYield()
	}
}

// Err returns value of internal error variable and clears it if clear is true.
func (d *Driver) Err(clear bool) error {
	e := d.err
	if e == 0 {
		return nil
	}
	if clear {
		d.err = 0
	}
	if err := DriverError(e >> 16); err != 0 {
		return err
	}
	if err := Error(e >> 8); err != 0 {
		if err&ErrOverrun != 0 && clear {
			d.p.LoadByte()
			d.p.Status()
		}
		return err
	}
	return dma.Error(e)
}

func (d *Driver) writeRead(oaddr, iaddr uintptr, olen, ilen int, wsize uintptr) int {
	if olen > ilen {
		var n int
		if ilen > 0 {
			n = d.writeReadDMA(oaddr, iaddr, ilen, ilen, wsize)
			if d.err != 0 {
				return n
			}
			olen -= ilen
			oaddr += uintptr(ilen)
		}
		d.writeDMA(oaddr, olen, wsize, dma.IncM)
		return n
	}
	if ilen > olen {
		var n int
		ffff := uint16(0xffff)
		if olen > 0 {
			n = d.writeReadDMA(oaddr, iaddr, olen, olen, wsize)
			if d.err != 0 {
				return n
			}
			ilen -= olen
			iaddr += uintptr(olen)
			oaddr += uintptr(olen - 1)
		} else {
			oaddr = uintptr(unsafe.Pointer(&ffff))
		}
		return n + d.writeReadDMA(oaddr, iaddr, 1, ilen, wsize)
	}
	return d.writeReadDMA(oaddr, iaddr, ilen, ilen, wsize)
}

func (d *Driver) WriteStringRead(out string, in []byte) int {
	olen := len(out)
	ilen := len(in)
	if d.err != 0 || olen == 0 && ilen == 0 {
		return 0
	}
	if olen <= 1 && ilen <= 1 {
		// Avoid DMA for one byte transfers.
		b := byte(0xff)
		if olen != 0 {
			b = out[0]
		}
		b = d.WriteReadByte(b)
		if ilen != 0 {
			in[0] = b
			return 1
		}
		return 0
	}
	oaddr := (*reflect.StringHeader)(unsafe.Pointer(&out)).Data
	iaddr := (*reflect.SliceHeader)(unsafe.Pointer(&in)).Data
	return d.writeRead(oaddr, iaddr, olen, ilen, 1)
}

func (d *Driver) WriteRead(out, in []byte) int {
	return d.WriteStringRead(*(*string)(unsafe.Pointer(&out)), in)
}

func (d *Driver) WriteReadMany(oi ...[]byte) int {
	var n int
	for k := 0; k < len(oi); k += 2 {
		var in []byte
		if k+1 < len(oi) {
			in = oi[k+1]
		}
		out := oi[k]
		n += d.WriteRead(out, in)
	}
	return n
}

func (d *Driver) RepeatByte(b byte, n int) {
	if d.err != 0 {
		return
	}
	switch {
	case n > 1:
		d.writeDMA(uintptr(unsafe.Pointer(&b)), n, 1, 0)
	case n == 1:
		// Avoid DMA for one byte transfers.
		d.WriteReadByte(b)
	}
}

func (d *Driver) WriteRead16(out, in []uint16) int {
	olen := len(out)
	ilen := len(in)
	if d.err != 0 || olen == 0 && ilen == 0 {
		return 0
	}
	if olen <= 1 && ilen <= 1 {
		// Avoid DMA for one word transfers.
		w := uint16(0xffff)
		if olen != 0 {
			w = out[0]
		}
		w = d.WriteReadWord16(w)
		if ilen != 0 {
			in[0] = w
			return 1
		}
		return 0
	}
	oaddr := (*reflect.SliceHeader)(unsafe.Pointer(&out)).Data
	iaddr := (*reflect.SliceHeader)(unsafe.Pointer(&in)).Data
	return d.writeRead(oaddr, iaddr, olen, ilen, 2)
}

func (d *Driver) WriteReadMany16(oi ...[]uint16) int {
	var n int
	for k := 0; k < len(oi); k += 2 {
		var in []uint16
		if k+1 < len(oi) {
			in = oi[k+1]
		}
		out := oi[k]
		n += d.WriteRead16(out, in)
	}
	return n
}

func (d *Driver) RepeatWord16(w uint16, n int) {
	if d.err != 0 {
		return
	}
	switch {
	case n > 1:
		d.writeDMA(uintptr(unsafe.Pointer(&w)), n, 2, 0)
	case n == 1:
		// Avoid DMA for one word transfers.
		d.WriteReadWord16(w)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/iam"

	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIamRolePolicy() *schema.Resource {
	return &schema.Resource{
		// PutRolePolicy API is idempotent, so these can be the same.
		Create: resourceAwsIamRolePolicyPut,
		Update: resourceAwsIamRolePolicyPut,

		Read:   resourceAwsIamRolePolicyRead,
		Delete: resourceAwsIamRolePolicyDelete,

		Schema: map[string]*schema.Schema{
			"policy": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsIamRolePolicyPut(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	request := &iam.PutRolePolicyInput{
		RoleName:       aws.String(d.Get("role").(string)),
		PolicyName:     aws.String(d.Get("name").(string)),
		PolicyDocument: aws.String(d.Get("policy").(string)),
	}

	if _, err := iamconn.PutRolePolicy(request); err != nil {
		return fmt.Errorf("Error putting IAM role policy %s: %s", *request.PolicyName, err)
	}

	d.SetId(fmt.Sprintf("%s:%s", *request.RoleName, *request.PolicyName))
	return nil
}

func resourceAwsIamRolePolicyRead(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	role, name := resourceAwsIamRolePolicyParseId(d.Id())

	request := &iam.GetRolePolicyInput{
		PolicyName: aws.String(name),
		RoleName:   aws.String(role),
	}

	var err error
	getResp, err := iamconn.GetRolePolicy(request)
	if err != nil {
		if iamerr, ok := err.(awserr.Error); ok && iamerr.Code() == "NoSuchEntity" { // XXX test me
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading IAM policy %s from role %s: %s", name, role, err)
	}

	if getResp.PolicyDocument == nil {
		return fmt.Errorf("GetRolePolicy returned a nil policy document")
	}

	policy, err := url.QueryUnescape(*getResp.PolicyDocument)
	if err != nil {
		return err
	}
	return d.Set("policy", policy)
}

func resourceAwsIamRolePolicyDelete(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	role, name := resourceAwsIamRolePolicyParseId(d.Id())

	request := &iam.DeleteRolePolicyInput{
		PolicyName: aws.String(name),
		RoleName:   aws.String(role),
	}

	if _, err := iamconn.DeleteRolePolicy(request); err != nil {
		return fmt.Errorf("Error deleting IAM role policy %s: %s", d.Id(), err)
	}
	return nil
}

func resourceAwsIamRolePolicyParseId(id string) (roleName, policyName string) {
	parts := strings.SplitN(id, ":", 2)
	roleName = parts[0]
	policyName = parts[1]
	return
}
<commit_msg>provider/aws: Add validation for aws_iam_role_policy.name<commit_after>package aws

import (
	"fmt"
	"net/url"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/iam"

	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsIamRolePolicy() *schema.Resource {
	return &schema.Resource{
		// PutRolePolicy API is idempotent, so these can be the same.
		Create: resourceAwsIamRolePolicyPut,
		Update: resourceAwsIamRolePolicyPut,

		Read:   resourceAwsIamRolePolicyRead,
		Delete: resourceAwsIamRolePolicyDelete,

		Schema: map[string]*schema.Schema{
			"policy": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					// https://github.com/boto/botocore/blob/2485f5c/botocore/data/iam/2010-05-08/service-2.json#L8291-L8296
					value := v.(string)
					if len(value) > 128 {
						errors = append(errors, fmt.Errorf(
							"%q cannot be longer than 128 characters", k))
					}
					if !regexp.MustCompile("^[\\w+=,.@-]+$").MatchString(value) {
						errors = append(errors, fmt.Errorf(
							"%q must match [\\w+=,.@-]", k))
					}
					return
				},
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsIamRolePolicyPut(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	request := &iam.PutRolePolicyInput{
		RoleName:       aws.String(d.Get("role").(string)),
		PolicyName:     aws.String(d.Get("name").(string)),
		PolicyDocument: aws.String(d.Get("policy").(string)),
	}

	if _, err := iamconn.PutRolePolicy(request); err != nil {
		return fmt.Errorf("Error putting IAM role policy %s: %s", *request.PolicyName, err)
	}

	d.SetId(fmt.Sprintf("%s:%s", *request.RoleName, *request.PolicyName))
	return nil
}

func resourceAwsIamRolePolicyRead(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	role, name := resourceAwsIamRolePolicyParseId(d.Id())

	request := &iam.GetRolePolicyInput{
		PolicyName: aws.String(name),
		RoleName:   aws.String(role),
	}

	var err error
	getResp, err := iamconn.GetRolePolicy(request)
	if err != nil {
		if iamerr, ok := err.(awserr.Error); ok && iamerr.Code() == "NoSuchEntity" { // XXX test me
			d.SetId("")
			return nil
		}
		return fmt.Errorf("Error reading IAM policy %s from role %s: %s", name, role, err)
	}

	if getResp.PolicyDocument == nil {
		return fmt.Errorf("GetRolePolicy returned a nil policy document")
	}

	policy, err := url.QueryUnescape(*getResp.PolicyDocument)
	if err != nil {
		return err
	}
	return d.Set("policy", policy)
}

func resourceAwsIamRolePolicyDelete(d *schema.ResourceData, meta interface{}) error {
	iamconn := meta.(*AWSClient).iamconn

	role, name := resourceAwsIamRolePolicyParseId(d.Id())

	request := &iam.DeleteRolePolicyInput{
		PolicyName: aws.String(name),
		RoleName:   aws.String(role),
	}

	if _, err := iamconn.DeleteRolePolicy(request); err != nil {
		return fmt.Errorf("Error deleting IAM role policy %s: %s", d.Id(), err)
	}
	return nil
}

func resourceAwsIamRolePolicyParseId(id string) (roleName, policyName string) {
	parts := strings.SplitN(id, ":", 2)
	roleName = parts[0]
	policyName = parts[1]
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	//"crypto/tls"
	"errors"
	//"flag"
	//"fmt"
	//"github.com/prometheus/client_golang/prometheus"
	//"gopkg.in/ldap.v2"
	"gopkg.in/yaml.v2"
	//"io"
	"io/ioutil"
	"log"
	//"net"
	//"net/http"
	"os"
	"os/exec"
	"regexp"
	//"strconv"
	"strings"
	//"sync"
	//"time"
)

// describes the network config and the mechanism for user auth.
// While the contents of the certificaes are public, we want to
// restrict generation to authenticated users
type baseConfig struct {
	HttpAddress     string
	TLSCertFilename string
	TLSKeyFilename  string
	UserAuth        string
	SSH_CA_Filename string
}

type LdapConfig struct {
	BindPattern   string
	LDAPTargetURL string
}

type AppConfigFile struct {
	Base baseConfig
	//	Ldap   LdapConfig
}

func getUserPubKey(username string) (string, error) {
	cmd := exec.Command("/usr/bin/sss_ssh_authorizedkeys", username)
	cmd.Stdin = strings.NewReader("some input")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	log.Printf("Pub key: %s\n", out.String())
	return out.String(), nil
}

// gen_user_cert a username and key, returns a short lived cert for that user
func gen_cert_internal(username string, userPubKey string, users_ca_filename string, host_identity string) (string, error) {

	//Convert userKey into temp file
	content := []byte(userPubKey)
	tmpfile, err := ioutil.TempFile("/tmp/", "userkey")
	if err != nil {
		log.Fatal(err)
	}
	defer tmpfile.Close()
	defer os.Remove(tmpfile.Name()) // clean up

	if _, err := tmpfile.Write(content); err != nil {
		log.Fatal(err)
	}

	keyIdentity := host_identity + "_" + username

	cmd := exec.Command("ssh-keygen", "-s", users_ca_filename, "-I", keyIdentity, "-n", username, "-V", "+1d", tmpfile.Name())
	cmd.Stdin = strings.NewReader("\n")
	var out bytes.Buffer
	cmd.Stdout = &out

	var cmderr bytes.Buffer
	cmd.Stderr = &cmderr
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("stdout: %q\n", out.String())
	log.Printf("stderr: %q\n", cmderr.String())

	//Signed user key /tmp/userkey322296953-cert.pub: id "foo" serial 0 for bar valid from 2016-12-05T21:38:00 to 2016-12-06T19:39:45
	re := regexp.MustCompile("^Signed user key ([^:]+):")
	match := re.FindStringSubmatch(cmderr.String())
	if len(match) != 2 {
		log.Printf("badmatch; %v\n", match)
		err := errors.New("cannot find signed key name, re find failure")
		return "", err
	}
	outFilename := match[1]
	log.Printf("outfilename: %v\n", outFilename)
	defer os.Remove(outFilename)

	fileBytes, err := ioutil.ReadFile(outFilename)
	if err != nil {
		return "", err
	}

	return string(fileBytes[:]), nil
}

func genUserCert(userName string, users_ca_filename string) (string, error) {

	userPubKey, err := getUserPubKey(userName)
	if err != nil {
		log.Fatal(err)
	}
	hostIdentity := "mon-sre-dev"
	cert, err := gen_cert_internal(userName, userPubKey, users_ca_filename, hostIdentity)
	if err != nil {
		log.Fatal(err)
	}
	return cert, err
}

func loadVerifyConfigFile(configFilename string) (AppConfigFile, error) {
	if _, err := os.Stat(configFilename); os.IsNotExist(err) {
		log.Printf("Missing config file\n")
		panic(err)
	}
	var config AppConfigFile
	source, err := ioutil.ReadFile(configFilename)
	if err != nil {
		panic(err)
	}
	err = yaml.Unmarshal(source, &config)
	if err != nil {
		panic(err)
	}
	return config, nil
}

func main() {

	configFilename := "config.yml"
	/*
		if _, err := os.Stat(configFilename); os.IsNotExist(err) {
			log.Printf("Missing config file\n")
			panic(err)
		}
		var config AppConfigFile
		source, err := ioutil.ReadFile(configFilename)
		if err != nil {
			panic(err)
		}
		err = yaml.Unmarshal(source, &config)
		if err != nil {
			panic(err)
		}
	*/
	config, err := loadVerifyConfigFile(configFilename)
	if err != nil {
		panic(err)
	}
	cert, err := genUserCert("camilo_viecco1", config.Base.SSH_CA_Filename)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("cert=%s", cert)
}
<commit_msg>config file and flags<commit_after>package main

import (
	"bytes"
	//"crypto/tls"
	"errors"
	"flag"
	//"fmt"
	//"github.com/prometheus/client_golang/prometheus"
	//"gopkg.in/ldap.v2"
	"gopkg.in/yaml.v2"
	//"io"
	"io/ioutil"
	"log"
	//"net"
	//"net/http"
	"os"
	"os/exec"
	"regexp"
	//"strconv"
	"strings"
	//"sync"
	//"time"
)

// describes the network config and the mechanism for user auth.
// While the contents of the certificaes are public, we want to
// restrict generation to authenticated users
type baseConfig struct {
	HttpAddress     string
	TLSCertFilename string
	TLSKeyFilename  string
	UserAuth        string
	SSH_CA_Filename string
}

type LdapConfig struct {
	BindPattern   string
	LDAPTargetURL string
}

type AppConfigFile struct {
	Base baseConfig
	//	Ldap   LdapConfig
}

var (
	configFilename = flag.String("config", "config.yml", "The filename of the configuration")
	debug          = flag.Bool("debug", false, "Enable debug messages to console")
)

func getUserPubKey(username string) (string, error) {
	cmd := exec.Command("/usr/bin/sss_ssh_authorizedkeys", username)
	cmd.Stdin = strings.NewReader("some input")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	log.Printf("Pub key: %s\n", out.String())
	return out.String(), nil
}

// gen_user_cert a username and key, returns a short lived cert for that user
func gen_cert_internal(username string, userPubKey string, users_ca_filename string, host_identity string) (string, error) {

	//Convert userKey into temp file
	content := []byte(userPubKey)
	tmpfile, err := ioutil.TempFile("/tmp/", "userkey")
	if err != nil {
		log.Fatal(err)
	}
	defer tmpfile.Close()
	defer os.Remove(tmpfile.Name()) // clean up

	if _, err := tmpfile.Write(content); err != nil {
		log.Fatal(err)
	}

	keyIdentity := host_identity + "_" + username

	cmd := exec.Command("ssh-keygen", "-s", users_ca_filename, "-I", keyIdentity, "-n", username, "-V", "+1d", tmpfile.Name())
	cmd.Stdin = strings.NewReader("\n")
	var out bytes.Buffer
	cmd.Stdout = &out

	var cmderr bytes.Buffer
	cmd.Stderr = &cmderr
	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("stdout: %q\n", out.String())
	log.Printf("stderr: %q\n", cmderr.String())

	//Signed user key /tmp/userkey322296953-cert.pub: id "foo" serial 0 for bar valid from 2016-12-05T21:38:00 to 2016-12-06T19:39:45
	re := regexp.MustCompile("^Signed user key ([^:]+):")
	match := re.FindStringSubmatch(cmderr.String())
	if len(match) != 2 {
		log.Printf("badmatch; %v\n", match)
		err := errors.New("cannot find signed key name, re find failure")
		return "", err
	}
	outFilename := match[1]
	log.Printf("outfilename: %v\n", outFilename)
	defer os.Remove(outFilename)

	fileBytes, err := ioutil.ReadFile(outFilename)
	if err != nil {
		return "", err
	}

	return string(fileBytes[:]), nil
}

func genUserCert(userName string, users_ca_filename string) (string, error) {

	userPubKey, err := getUserPubKey(userName)
	if err != nil {
		log.Fatal(err)
	}
	hostIdentity := "mon-sre-dev"
	cert, err := gen_cert_internal(userName, userPubKey, users_ca_filename, hostIdentity)
	if err != nil {
		log.Fatal(err)
	}
	return cert, err
}

func loadVerifyConfigFile(configFilename string) (AppConfigFile, error) {
	var config AppConfigFile
	if _, err := os.Stat(configFilename); os.IsNotExist(err) {
		//log.Printf("Missing config file\n")
		err = errors.New("mising config file failure")
		return config, err
	}
	source, err := ioutil.ReadFile(configFilename)
	if err != nil {
		//panic(err)
		err = errors.New("cannot read config file")
		return config, err
	}
	err = yaml.Unmarshal(source, &config)
	if err != nil {
		err = errors.New("Cannot parse config file")
		return config, err
	}

	//verify config
	if _, err := os.Stat(config.Base.SSH_CA_Filename); os.IsNotExist(err) {
		//log.Printf("Missing config file\n")
		err = errors.New("mising ssh CA file")
		return config, err
	}

	return config, nil
}

func main() {
	flag.Parse()

	config, err := loadVerifyConfigFile(*configFilename)
	if err != nil {
		panic(err)
	}
	cert, err := genUserCert("camilo_viecco1", config.Base.SSH_CA_Filename)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("cert=%s", cert)
}
<|endoftext|>


<commit_before>package filesys

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"io"
	"sync"
	"time"
)

type MemoryChunkPages struct {
	fh             *FileHandle
	writeWaitGroup sync.WaitGroup
	chunkAddLock   sync.Mutex
	lastErr        error
	collection     string
	replication    string
	uploadPipeline *page_writer.UploadPipeline
	hasWrites      bool
}

var (
	_ = page_writer.DirtyPages(&MemoryChunkPages{})
)

func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *MemoryChunkPages {

	dirtyPages := &MemoryChunkPages{
		fh: fh,
	}

	swapFileDir := fh.f.wfs.option.getTempFilePageDir()

	dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.f.fullpath(),
		fh.f.wfs.concurrentWriters, chunkSize, dirtyPages.saveChunkedFileIntevalToStorage, swapFileDir)

	return dirtyPages
}

func (pages *MemoryChunkPages) AddPage(offset int64, data []byte) {
	pages.hasWrites = true

	glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.f.fullpath(), offset, offset+int64(len(data)))
	pages.uploadPipeline.SaveDataAt(data, offset)

	return
}

func (pages *MemoryChunkPages) FlushData() error {
	if !pages.hasWrites {
		return nil
	}
	pages.uploadPipeline.FlushAll()
	if pages.lastErr != nil {
		return fmt.Errorf("flush data: %v", pages.lastErr)
	}
	return nil
}

func (pages *MemoryChunkPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
	if !pages.hasWrites {
		return
	}
	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset)
}

func (pages *MemoryChunkPages) GetStorageOptions() (collection, replication string) {
	return pages.collection, pages.replication
}

func (pages *MemoryChunkPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {

	mtime := time.Now().UnixNano()
	defer cleanupFn()

	chunk, collection, replication, err := pages.fh.f.wfs.saveDataAsChunk(pages.fh.f.fullpath())(reader, pages.fh.f.Name, offset)
	if err != nil {
		glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.fh.f.fullpath(), offset, offset+size, err)
		pages.lastErr = err
		return
	}
	chunk.Mtime = mtime
	pages.collection, pages.replication = collection, replication
	pages.chunkAddLock.Lock()
	pages.fh.f.addChunks([]*filer_pb.FileChunk{chunk})
	pages.fh.entryViewCache = nil
	glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.fh.f.fullpath(), chunk.FileId, offset, offset+size)
	pages.chunkAddLock.Unlock()

}

func (pages MemoryChunkPages) Destroy() {
	pages.uploadPipeline.Shutdown()
}

func (pages *MemoryChunkPages) LockForRead(startOffset, stopOffset int64) {
	pages.uploadPipeline.LockForRead(startOffset, stopOffset)
}
func (pages *MemoryChunkPages) UnlockForRead(startOffset, stopOffset int64) {
	pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
}
<commit_msg>rename<commit_after>package filesys

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filesys/page_writer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"io"
	"sync"
	"time"
)

type ChunkedDirtyPages struct {
	fh             *FileHandle
	writeWaitGroup sync.WaitGroup
	chunkAddLock   sync.Mutex
	lastErr        error
	collection     string
	replication    string
	uploadPipeline *page_writer.UploadPipeline
	hasWrites      bool
}

var (
	_ = page_writer.DirtyPages(&ChunkedDirtyPages{})
)

func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {

	dirtyPages := &ChunkedDirtyPages{
		fh: fh,
	}

	swapFileDir := fh.f.wfs.option.getTempFilePageDir()

	dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.f.fullpath(),
		fh.f.wfs.concurrentWriters, chunkSize, dirtyPages.saveChunkedFileIntevalToStorage, swapFileDir)

	return dirtyPages
}

func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte) {
	pages.hasWrites = true

	glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.f.fullpath(), offset, offset+int64(len(data)))
	pages.uploadPipeline.SaveDataAt(data, offset)

	return
}

func (pages *ChunkedDirtyPages) FlushData() error {
	if !pages.hasWrites {
		return nil
	}
	pages.uploadPipeline.FlushAll()
	if pages.lastErr != nil {
		return fmt.Errorf("flush data: %v", pages.lastErr)
	}
	return nil
}

func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) {
	if !pages.hasWrites {
		return
	}
	return pages.uploadPipeline.MaybeReadDataAt(data, startOffset)
}

func (pages *ChunkedDirtyPages) GetStorageOptions() (collection, replication string) {
	return pages.collection, pages.replication
}

func (pages *ChunkedDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) {

	mtime := time.Now().UnixNano()
	defer cleanupFn()

	chunk, collection, replication, err := pages.fh.f.wfs.saveDataAsChunk(pages.fh.f.fullpath())(reader, pages.fh.f.Name, offset)
	if err != nil {
		glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.fh.f.fullpath(), offset, offset+size, err)
		pages.lastErr = err
		return
	}
	chunk.Mtime = mtime
	pages.collection, pages.replication = collection, replication
	pages.chunkAddLock.Lock()
	pages.fh.f.addChunks([]*filer_pb.FileChunk{chunk})
	pages.fh.entryViewCache = nil
	glog.V(3).Infof("%s saveToStorage %s [%d,%d)", pages.fh.f.fullpath(), chunk.FileId, offset, offset+size)
	pages.chunkAddLock.Unlock()

}

func (pages ChunkedDirtyPages) Destroy() {
	pages.uploadPipeline.Shutdown()
}

func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) {
	pages.uploadPipeline.LockForRead(startOffset, stopOffset)
}
func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) {
	pages.uploadPipeline.UnlockForRead(startOffset, stopOffset)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime/debug"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
)

var (
	listen           = flag.String("l", ":8888", "port to accept requests")
	targetProduction = flag.String("a", "http://localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget        = flag.String("b", "http://localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
)

type Hosts struct {
	Target      url.URL
	Alternative url.URL
}

var hosts Hosts
var client *http.Client
var proxy *httputil.ReverseProxy

type TimeoutTransport struct {
	http.Transport
}

func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.Transport.RoundTrip(req)
}

func clientCall(id string, req, req2 *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered in clientCall", r, debug.Stack())
		}
	}()

	resp, err := client.Do(req2)

	if err != nil {
		logMessage("[%v][%v][<B Error>][<%v>]\n", id, err)
	} else {
		r, e := httputil.DumpResponse(resp, true)

		if e != nil {
			logMessage("[%v][%v][<B Error Dump>][<%v>]\n", id, e)
		} else {
			logMessage("[%v][%v][<B Resp>][<%v>]\n", id, string(r))
		}

		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

func teeDirector(req *http.Request) {
	id := uuid.NewUUID().String()
	fmt.Printf("[%v][%v][<Request>][<%+v>]\n", time.Now().Format(time.RFC3339Nano), id, req)

	go clientCall(id, req, duplicateRequest(req))

	targetQuery := hosts.Target.RawQuery
	req.URL.Scheme = hosts.Target.Scheme
	req.URL.Host = hosts.Target.Host
	req.URL.Path = singleJoiningSlash(hosts.Target.Path, req.URL.Path)
	if targetQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = targetQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
	}
}

func logMessage(message string, id string, logObj interface{}) {
	fmt.Printf(message, time.Now().Format(time.RFC3339Nano), id, logObj)
}

func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}

func duplicateRequest(request *http.Request) (request1 *http.Request) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	request.Body = ioutil.NopCloser(bytes.NewReader(b2.Bytes()))
	bodyReader := ioutil.NopCloser(bytes.NewReader(b1.Bytes()))

	request2 := &http.Request{
		Method: request.Method,
		URL: &url.URL{
			Scheme:   hosts.Alternative.Scheme,
			Host:     hosts.Alternative.Host,
			Path:     singleJoiningSlash(hosts.Alternative.Path, request.URL.Path),
			RawQuery: request.URL.RawQuery,
		},
		Proto:         request.Proto,
		ProtoMajor:    request.ProtoMajor,
		ProtoMinor:    request.ProtoMinor,
		Header:        request.Header,
		Body:          bodyReader,
		ContentLength: request.ContentLength,
		Close:         false,
	}

	return request2
}

func handler(w http.ResponseWriter, r *http.Request) {
	dump, e := httputil.DumpRequest(r, true)
	if e != nil {
		logMessage("[%v][%v][<In Error Dump>][<%v>]\n", "", e)
	} else {
		logMessage("[%v][%v][<In Req>][<%v>]\n", "", string(dump))
	}
	proxy.ServeHTTP(w, r)
}

func prettyPrintRequest(req *http.Request) string {
	result, _ := json.MarshalIndent(req, "", "\t")
	return string(result)
}

func main() {
	flag.Parse()

	target, _ := url.Parse(*targetProduction)
	alt, _ := url.Parse(*altTarget)
	client = &http.Client{
		Timeout: time.Millisecond * 2000,
	}

	hosts = Hosts{
		Target:      *target,
		Alternative: *alt,
	}

	u, _ := url.Parse(*targetProduction)
	proxy = httputil.NewSingleHostReverseProxy(u)
	proxy.Transport = &TimeoutTransport{}
	proxy.Director = teeDirector

	http.HandleFunc("/", handler)
	http.ListenAndServe(*listen, nil)
}
<commit_msg>Print duplicated request on errors<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime/debug"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
)

var (
	listen           = flag.String("l", ":8888", "port to accept requests")
	targetProduction = flag.String("a", "http://localhost:8080", "where production traffic goes. http://localhost:8080/production")
	altTarget        = flag.String("b", "http://localhost:8081", "where testing traffic goes. response are skipped. http://localhost:8081/test")
)

type Hosts struct {
	Target      url.URL
	Alternative url.URL
}

var hosts Hosts
var client *http.Client
var proxy *httputil.ReverseProxy

type TimeoutTransport struct {
	http.Transport
}

func (t *TimeoutTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	return t.Transport.RoundTrip(req)
}

func clientCall(id string, req, req2 *http.Request) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered in clientCall", r, debug.Stack())
		}
	}()

	resp, err := client.Do(req2)
	if err != nil {
		fmt.Printf("[%v][%v][<B Error>][<%v>][Request: %+v]\n", time.Now().Format(time.RFC3339Nano), id, err, req2)
		return
	}

	r, e := httputil.DumpResponse(resp, true)
	if e != nil {
		logMessage("[%v][%v][<B Error Dump>][<%v>]\n", id, e)
	} else {
		logMessage("[%v][%v][<B Resp>][<%v>]\n", id, string(r))
	}

	io.Copy(ioutil.Discard, resp.Body)
	resp.Body.Close()
}

func teeDirector(req *http.Request) {
	id := uuid.NewUUID().String()
	fmt.Printf("[%v][%v][<Request>][<%+v>]\n", time.Now().Format(time.RFC3339Nano), id, req)

	go clientCall(id, req, duplicateRequest(req))

	targetQuery := hosts.Target.RawQuery
	req.URL.Scheme = hosts.Target.Scheme
	req.URL.Host = hosts.Target.Host
	req.URL.Path = singleJoiningSlash(hosts.Target.Path, req.URL.Path)
	if targetQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = targetQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
	}
}

func logMessage(message string, id string, logObj interface{}) {
	fmt.Printf(message, time.Now().Format(time.RFC3339Nano), id, logObj)
}

func singleJoiningSlash(a, b string) string {
	aslash := strings.HasSuffix(a, "/")
	bslash := strings.HasPrefix(b, "/")
	switch {
	case aslash && bslash:
		return a + b[1:]
	case !aslash && !bslash:
		return a + "/" + b
	}
	return a + b
}

func duplicateRequest(request *http.Request) (request1 *http.Request) {
	b1 := new(bytes.Buffer)
	b2 := new(bytes.Buffer)
	w := io.MultiWriter(b1, b2)
	io.Copy(w, request.Body)
	request.Body = ioutil.NopCloser(bytes.NewReader(b2.Bytes()))
	bodyReader := ioutil.NopCloser(bytes.NewReader(b1.Bytes()))

	request2 := &http.Request{
		Method: request.Method,
		URL: &url.URL{
			Scheme:   hosts.Alternative.Scheme,
			Host:     hosts.Alternative.Host,
			Path:     singleJoiningSlash(hosts.Alternative.Path, request.URL.Path),
			RawQuery: request.URL.RawQuery,
		},
		Proto:         request.Proto,
		ProtoMajor:    request.ProtoMajor,
		ProtoMinor:    request.ProtoMinor,
		Header:        request.Header,
		Body:          bodyReader,
		ContentLength: request.ContentLength,
		Close:         false,
	}

	return request2
}

func handler(w http.ResponseWriter, r *http.Request) {
	dump, e := httputil.DumpRequest(r, true)
	if e != nil {
		logMessage("[%v][%v][<In Error Dump>][<%v>]\n", "", e)
	} else {
		logMessage("[%v][%v][<In Req>][<%v>]\n", "", string(dump))
	}
	proxy.ServeHTTP(w, r)
}

func main() {
	flag.Parse()

	target, _ := url.Parse(*targetProduction)
	alt, _ := url.Parse(*altTarget)
	client = &http.Client{
		Timeout: time.Millisecond * 2000,
	}

	hosts = Hosts{
		Target:      *target,
		Alternative: *alt,
	}

	u, _ := url.Parse(*targetProduction)
	proxy = httputil.NewSingleHostReverseProxy(u)
	proxy.Transport = &TimeoutTransport{}
	proxy.Director = teeDirector

	http.HandleFunc("/", handler)
	http.ListenAndServe(*listen, nil)
}
<|endoftext|>


<commit_before>package types

import (
	"strconv"
	"time"
)

type EventType string

const (
	EventTypeCreate      EventType = "m.room.create"
	EventTypeName                  = "m.room.name"
	EventTypeTopic                 = "m.room.topic"
	EventTypeMember                = "m.room.member"
	EventTypeAliases               = "m.room.aliases"
	EventTypeJoinRules             = "m.room.join_rules"
	EventTypePowerLevels           = "m.room.power_levels"
)

type Content interface{}

type TimeStamp struct {
	time.Time
}

type Event struct {
	Id        EventId   `json:"event_id"`
	RoomId    RoomId    `json:"room_id"`
	EventType EventType `json:"type"`
	TimeStamp TimeStamp `json:"origin_server_ts"`
	Content   Content   `json:"content"`
}

type State struct {
	Event
	StateKey   string  `json:"state_key"`
	OldContent Content `json:"prev_content"`
}

func (ts TimeStamp) MarshalJSON() ([]byte, error) {
	ms := ts.UnixNano() / int64(time.Millisecond)
	return []byte(strconv.FormatInt(ms, 10)), nil
}

func (ts *TimeStamp) UnmarshalJSON(data []byte) error {
	ms, err := strconv.ParseInt(string(data), 10, 64)
	if err != nil {
		return err
	}

	ts.Time = time.Unix(0, ms*int64(time.Millisecond))
	return nil
}

type TestContent struct {
	Name string `json:"name"`
}

type MembershipContent struct {
	Membership  string `json:"membership"`
	DisplayName string `json:"displayname"`
	AvatarUrl   string `json:"avatar_url"`
}
<commit_msg>types/events: added typed content types<commit_after>package types

import (
	"encoding/json"
	"strconv"
	"time"
)

type EventType string

const (
	EventTypeCreate      EventType = "m.room.create"
	EventTypeName                  = "m.room.name"
	EventTypeTopic                 = "m.room.topic"
	EventTypeAliases               = "m.room.aliases"
	EventTypeJoinRules             = "m.room.join_rules"
	EventTypeMembership            = "m.room.member"
	EventTypePowerLevels           = "m.room.power_levels"
)

type TypedContent interface {
	EventType() EventType
}

type Content interface{}

type TimeStamp struct {
	time.Time
}

type Event struct {
	Id        EventId   `json:"event_id"`
	RoomId    RoomId    `json:"room_id"`
	UserId    UserId    `json:"user_od"`
	EventType EventType `json:"type"`
	TimeStamp TimeStamp `json:"origin_server_ts"`
	Content   Content   `json:"content"`
}

type State struct {
	Event
	StateKey   string  `json:"state_key"`
	OldContent Content `json:"prev_content"`
}

func (ts TimeStamp) MarshalJSON() ([]byte, error) {
	ms := ts.UnixNano() / int64(time.Millisecond)
	return []byte(strconv.FormatInt(ms, 10)), nil
}

func (ts *TimeStamp) UnmarshalJSON(data []byte) error {
	ms, err := strconv.ParseInt(string(data), 10, 64)
	if err != nil {
		return err
	}

	ts.Time = time.Unix(0, ms*int64(time.Millisecond))
	return nil
}

type GenericContent struct {
	Content   map[string]interface{}
	eventType EventType
}

func (c *GenericContent) EventType() EventType {
	return c.eventType
}

func (c *GenericContent) MarshalJSON() ([]byte, error) {
	return json.Marshal(c.Content)
}

type TestContent struct {
	Name string `json:"name"`
}

type MembershipEventContent struct {
	Membership  string `json:"membership"`
	DisplayName string `json:"displayname"`
	AvatarUrl   string `json:"avatar_url"`
}

func (c *MembershipEventContent) EventType() EventType {
	return EventTypeMembership
}

type CreateEventContent struct {
	Creator UserId `json:"creator"`
}

func (c *CreateEventContent) EventType() EventType {
	return EventTypeCreate
}

type NameEventContent struct {
	Name string `json:"name"`
}

func (c *NameEventContent) EventType() EventType {
	return EventTypeName
}

type TopicEventContent struct {
	Topic string `json:"topic"`
}

func (c *TopicEventContent) EventType() EventType {
	return EventTypeTopic
}

type AliasesEventContent struct {
	Aliases string `json:"aliases"`
}

func (c *AliasesEventContent) EventType() EventType {
	return EventTypeAliases
}

type PowerLevelsEventContent struct {
	Ban          int             `json:"ban"`
	Kick         int             `json:"kick"`
	Redact       int             `json:"redact"`
	UserDefault  int             `json:"users_default"`
	CreateState  int             `json:"state_default"`
	EventDefault int             `json:"events_default"`
	Users        map[UserId]int  `json:"users"`
	Events       map[EventId]int `json:"events"`
}

func (c *PowerLevelsEventContent) EventType() EventType {
	return EventTypePowerLevels
}

type JoinRulesEventContent struct {
	JoinRule JoinRule `json:"join_rule"`
}

func (c *JoinRulesEventContent) EventType() EventType {
	return EventTypeJoinRules
}
<|endoftext|>


<commit_before>package jsonrpc2

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/rpc"
)

const contentType = "application/json"

type httpServerConn struct {
	req     io.Reader
	res     io.Writer
	replied bool
}

func (conn *httpServerConn) Read(buf []byte) (int, error) {
	return conn.req.Read(buf)
}

func (conn *httpServerConn) Write(buf []byte) (int, error) {
	conn.replied = true
	return conn.res.Write(buf)
}

func (conn *httpServerConn) Close() error {
	return nil
}

type httpHandler struct {
	rpc *rpc.Server
}

// HTTPHandler returns handler for HTTP requests which will execute
// incoming RPC using srv. If srv is nil then use rpc.DefaultServer.
//
// Specification: http://www.simple-is-better.org/json-rpc/transport_http.html
//   - Pipelined Requests/Responses not supported.
//   - GET Request not supported.
func HTTPHandler(srv *rpc.Server) http.Handler {
	if srv == nil {
		srv = rpc.DefaultServer
	}
	return &httpHandler{srv}
}

func (h *httpHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", contentType)

	if req.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	if req.Header.Get("Content-Type") != contentType || req.Header.Get("Accept") != contentType {
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	}

	conn := &httpServerConn{req: req.Body, res: w}
	h.rpc.ServeRequest(NewServerCodec(conn, h.rpc))
	if !conn.replied {
		w.WriteHeader(http.StatusNoContent)
	}
}

type httpClientConn struct {
	url   string
	ready chan io.ReadCloser
	body  io.ReadCloser
}

func (conn *httpClientConn) Read(buf []byte) (int, error) {
	if conn.body == nil {
		conn.body = <-conn.ready
	}
	n, err := conn.body.Read(buf)
	if err == io.EOF {
		conn.body.Close()
		conn.body = nil
		err = nil
		if n == 0 {
			return conn.Read(buf)
		}
	}
	return n, err
}

func (conn *httpClientConn) Write(buf []byte) (int, error) {
	req, err := http.NewRequest("POST", conn.url, bytes.NewReader(buf))
	if err != nil {
		return 0, err
	}
	req.Header.Add("Content-Type", contentType)
	req.Header.Add("Accept", contentType)
	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return 0, err
	}
	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusAccepted {
		return 0, fmt.Errorf("bad HTTP Status: %s", resp.Status)
	}
	if resp.Header.Get("Content-Type") != contentType {
		return 0, fmt.Errorf("bad HTTP Content-Type: %s", resp.Header.Get("Content-Type"))
	}
	if resp.StatusCode == http.StatusOK {
		conn.ready <- resp.Body
	}
	return len(buf), nil
}

func (conn *httpClientConn) Close() error {
	return nil
}

// NewHTTPClient returns a new Client to handle requests to the
// set of services at the given url.
func NewHTTPClient(url string) *Client {
	return NewClient(&httpClientConn{url: url, ready: make(chan io.ReadCloser, 16)})
}
<commit_msg>HTTP Client: allow parallel requests<commit_after>package jsonrpc2

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/rpc"
)

const contentType = "application/json"

type httpServerConn struct {
	req     io.Reader
	res     io.Writer
	replied bool
}

func (conn *httpServerConn) Read(buf []byte) (int, error) {
	return conn.req.Read(buf)
}

func (conn *httpServerConn) Write(buf []byte) (int, error) {
	conn.replied = true
	return conn.res.Write(buf)
}

func (conn *httpServerConn) Close() error {
	return nil
}

type httpHandler struct {
	rpc *rpc.Server
}

// HTTPHandler returns handler for HTTP requests which will execute
// incoming RPC using srv. If srv is nil then use rpc.DefaultServer.
//
// Specification: http://www.simple-is-better.org/json-rpc/transport_http.html
//   - Pipelined Requests/Responses not supported.
//   - GET Request not supported.
func HTTPHandler(srv *rpc.Server) http.Handler {
	if srv == nil {
		srv = rpc.DefaultServer
	}
	return &httpHandler{srv}
}

func (h *httpHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", contentType)

	if req.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}
	if req.Header.Get("Content-Type") != contentType || req.Header.Get("Accept") != contentType {
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	}

	conn := &httpServerConn{req: req.Body, res: w}
	h.rpc.ServeRequest(NewServerCodec(conn, h.rpc))
	if !conn.replied {
		w.WriteHeader(http.StatusNoContent)
	}
}

type httpClientConn struct {
	url   string
	ready chan io.ReadCloser
	body  io.ReadCloser
}

func (conn *httpClientConn) Read(buf []byte) (int, error) {
	if conn.body == nil {
		conn.body = <-conn.ready
	}
	n, err := conn.body.Read(buf)
	if err == io.EOF {
		conn.body.Close()
		conn.body = nil
		err = nil
		if n == 0 {
			return conn.Read(buf)
		}
	}
	return n, err
}

func (conn *httpClientConn) Write(buf []byte) (int, error) {
	b := make([]byte, len(buf))
	copy(b, buf)
	go func() {
		req, err := http.NewRequest("POST", conn.url, bytes.NewReader(b))
		if err == nil {
			req.Header.Add("Content-Type", contentType)
			req.Header.Add("Accept", contentType)
			var resp *http.Response
			resp, err = (&http.Client{}).Do(req)
			if err != nil {
			} else if resp.Header.Get("Content-Type") != contentType {
				err = fmt.Errorf("bad HTTP Content-Type: %s", resp.Header.Get("Content-Type"))
			} else if resp.StatusCode == http.StatusOK {
				conn.ready <- resp.Body
				return
			} else if resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusAccepted {
				resp.Body.Close() // is it should be read to EOF first?
				return
			} else {
				err = fmt.Errorf("bad HTTP Status: %s", resp.Status)
			}
			resp.Body.Close() // is it should be read to EOF first?
		}
		var res clientResponse
		if json.Unmarshal(b, &res) == nil && res.ID == nil {
			return // ignore error from Notification
		}
		res.Error = NewError(errInternal.Code, err.Error())
		buf := &bytes.Buffer{}
		json.NewEncoder(buf).Encode(res)
		conn.ready <- ioutil.NopCloser(buf)
	}()
	return len(buf), nil
}

func (conn *httpClientConn) Close() error {
	return nil
}

// NewHTTPClient returns a new Client to handle requests to the
// set of services at the given url.
func NewHTTPClient(url string) *Client {
	return NewClient(&httpClientConn{url: url, ready: make(chan io.ReadCloser, 16)})
}
<|endoftext|>