<commit_before>

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
      s "strings"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

// Init create tables for tests
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// Create table one
	err := createTableOne(stub)
	if err != nil {
		return nil, fmt.Errorf("Error creating table one during init. %s", err)
	}

	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {

	switch function {

	case "insertRowTableOne":
		if len(args) < 3 {
			return nil, errors.New("insertTableOne failed. Must include 3 column values")
		}
     var str [] string

		for i := 0; i < len(args); i++ {

		str = s.Split(args[i],"-")

		col1Val := str[0]
		col2Int, err := strconv.ParseInt(str[1], 10, 32)
		if err != nil {
			return nil, errors.New("insertTableOne failed. arg[1] must be convertable to int32")
		}
		col2Val := int32(col2Int)
		col3Int, err := strconv.ParseInt(str[2], 10, 32)
		if err != nil {
			return nil, errors.New("insertTableOne failed. arg[2] must be convertable to int32")
		}
		col3Val := int32(col3Int)

		var columns []*shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
		col3 := shim.Column{Value: &shim.Column_Int32{Int32: col3Val}}
		columns = append(columns, &col1)
		columns = append(columns, &col2)
		columns = append(columns, &col3)

		row := shim.Row{Columns: columns}
		ok, err := stub.InsertRow("tableOne", row)
		if err != nil {
			return nil, fmt.Errorf("insertTableOne operation failed. %s", err)
		}
		if !ok {
			return nil, errors.New("insertTableOne operation failed. Row with given key already exists")
		}

	   }

	case "replaceRowTableOne":
		if len(args) < 3 {
			return nil, errors.New("replaceRowTableOne failed. Must include 3 column values")
		}

		col1Val := args[0]
		col2Int, err := strconv.ParseInt(args[1], 10, 32)
		if err != nil {
			return nil, errors.New("replaceRowTableOne failed. arg[1] must be convertable to int32")
		}
		col2Val := int32(col2Int)
		col3Int, err := strconv.ParseInt(args[2], 10, 32)
		if err != nil {
			return nil, errors.New("replaceRowTableOne failed. arg[2] must be convertable to int32")
		}
		col3Val := int32(col3Int)

		var columns []*shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
		col3 := shim.Column{Value: &shim.Column_Int32{Int32: col3Val}}
		columns = append(columns, &col1)
		columns = append(columns, &col2)
		columns = append(columns, &col3)

		row := shim.Row{Columns: columns}
		ok, err := stub.ReplaceRow("tableOne", row)
		if err != nil {
			return nil, fmt.Errorf("replaceRowTableOne operation failed. %s", err)
		}
		if !ok {
			return nil, errors.New("replaceRowTableOne operation failed. Row with given key does not exist")
		}

	case "deleteAndRecreateTableOne":

		err := stub.DeleteTable("tableOne")
		if err != nil {
			return nil, fmt.Errorf("deleteAndRecreateTableOne operation failed. Error deleting table. %s", err)
		}

		err = createTableOne(stub)
		if err != nil {
			return nil, fmt.Errorf("deleteAndRecreateTableOne operation failed. Error creating table. %s", err)
		}

		return nil, nil

	default:
		return nil, errors.New("Unsupported operation")
	}
	return nil, nil
}

// Query callback representing the query of a chaincode
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {

	case "getRowTableOne":
		if len(args) < 1 {
			return nil, errors.New("getRowTableOne failed. Must include 1 key value")
		}

		col1Val := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		columns = append(columns, col1)

		row, err := stub.GetRow("tableOne", columns)
		if err != nil {
			return nil, fmt.Errorf("getRowTableOne operation failed. %s", err)
		}

		rowString := fmt.Sprintf("%s", row)
		return []byte(rowString), nil

	case "getRowsTableOne":
	if len(args) < 1 {
			return nil, errors.New("getRowsTableOne failed. Must include at least key values")
		}

		var columns []shim.Column
		col1Val := args[0]
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		columns = append(columns, col1)
	if len(args) > 1 {
			col2Int, err := strconv.ParseInt(args[1], 10, 32)
			if err != nil {
				return nil, errors.New("getRowsTableTwo failed. arg[1] must be convertable to int32")
			}

     col2Val := int32(col2Int)
			col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
			columns = append(columns, col2)

         }

         rowChannel, err := stub.GetRows("tableTwo", columns)

        if err != nil {
			return nil, fmt.Errorf("getRowsTableTwo operation failed. %s", err)
		}

		var rows []shim.Row
		for {
			select {
			case row, ok := <-rowChannel:
				if !ok {
					rowChannel = nil
				} else {
					rows = append(rows, row)
				}
			}
			if rowChannel == nil {
				break
			}
		}
      jsonRows, err := json.Marshal(rows)
	  if err != nil {
			return nil, fmt.Errorf("getRowsTableTwo operation failed. Error marshaling JSON: %s", err)
		}

		return jsonRows, nil

	default:
		return nil, errors.New("Unsupported operation")
	}
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

func createTableOne(stub shim.ChaincodeStubInterface) error {
	// Create table one
	var columnDefsTableOne []*shim.ColumnDefinition
	columnOneTableOneDef := shim.ColumnDefinition{Name: "colOneTableOne",
		Type: shim.ColumnDefinition_STRING, Key: true}
	columnTwoTableOneDef := shim.ColumnDefinition{Name: "colTwoTableOne",
		Type: shim.ColumnDefinition_INT32, Key: false}
	columnThreeTableOneDef := shim.ColumnDefinition{Name: "colThreeTableOne",
		Type: shim.ColumnDefinition_INT32, Key: false}
	columnDefsTableOne = append(columnDefsTableOne, &columnOneTableOneDef)
	columnDefsTableOne = append(columnDefsTableOne, &columnTwoTableOneDef)
	columnDefsTableOne = append(columnDefsTableOne, &columnThreeTableOneDef)
	return stub.CreateTable("tableOne", columnDefsTableOne)
}

<commit_msg>rows2<commit_after>

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
      s "strings"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

// Init create tables for tests
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	// Create table one
	err := createTableOne(stub)
	if err != nil {
		return nil, fmt.Errorf("Error creating table one during init. %s", err)
	}

	return nil, nil
}

func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {

	switch function {

	case "insertRowTableOne":
		if len(args) < 3 {
			return nil, errors.New("insertTableOne failed. Must include 3 column values")
		}
     var str [] string

		for i := 0; i < len(args); i++ {

		str = s.Split(args[i],"-")

		col1Val := str[0]
		col2Int, err := strconv.ParseInt(str[1], 10, 32)
		if err != nil {
			return nil, errors.New("insertTableOne failed. arg[1] must be convertable to int32")
		}
		col2Val := int32(col2Int)
		col3Int, err := strconv.ParseInt(str[2], 10, 32)
		if err != nil {
			return nil, errors.New("insertTableOne failed. arg[2] must be convertable to int32")
		}
		col3Val := int32(col3Int)

		var columns []*shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
		col3 := shim.Column{Value: &shim.Column_Int32{Int32: col3Val}}
		columns = append(columns, &col1)
		columns = append(columns, &col2)
		columns = append(columns, &col3)

		row := shim.Row{Columns: columns}
		ok, err := stub.InsertRow("tableOne", row)
		if err != nil {
			return nil, fmt.Errorf("insertTableOne operation failed. %s", err)
		}
		if !ok {
			return nil, errors.New("insertTableOne operation failed. Row with given key already exists")
		}

	   }

	case "replaceRowTableOne":
		if len(args) < 3 {
			return nil, errors.New("replaceRowTableOne failed. Must include 3 column values")
		}

		col1Val := args[0]
		col2Int, err := strconv.ParseInt(args[1], 10, 32)
		if err != nil {
			return nil, errors.New("replaceRowTableOne failed. arg[1] must be convertable to int32")
		}
		col2Val := int32(col2Int)
		col3Int, err := strconv.ParseInt(args[2], 10, 32)
		if err != nil {
			return nil, errors.New("replaceRowTableOne failed. arg[2] must be convertable to int32")
		}
		col3Val := int32(col3Int)

		var columns []*shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
		col3 := shim.Column{Value: &shim.Column_Int32{Int32: col3Val}}
		columns = append(columns, &col1)
		columns = append(columns, &col2)
		columns = append(columns, &col3)

		row := shim.Row{Columns: columns}
		ok, err := stub.ReplaceRow("tableOne", row)
		if err != nil {
			return nil, fmt.Errorf("replaceRowTableOne operation failed. %s", err)
		}
		if !ok {
			return nil, errors.New("replaceRowTableOne operation failed. Row with given key does not exist")
		}

	case "deleteAndRecreateTableOne":

		err := stub.DeleteTable("tableOne")
		if err != nil {
			return nil, fmt.Errorf("deleteAndRecreateTableOne operation failed. Error deleting table. %s", err)
		}

		err = createTableOne(stub)
		if err != nil {
			return nil, fmt.Errorf("deleteAndRecreateTableOne operation failed. Error creating table. %s", err)
		}

		return nil, nil

	default:
		return nil, errors.New("Unsupported operation")
	}
	return nil, nil
}

// Query callback representing the query of a chaincode
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	switch function {

	case "getRowTableOne":
		if len(args) < 1 {
			return nil, errors.New("getRowTableOne failed. Must include 1 key value")
		}

		col1Val := args[0]
		var columns []shim.Column
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		columns = append(columns, col1)

		row, err := stub.GetRow("tableOne", columns)
		if err != nil {
			return nil, fmt.Errorf("getRowTableOne operation failed. %s", err)
		}

		rowString := fmt.Sprintf("%s", row)
		return []byte(rowString), nil

	case "getRowsTableOne":
	if len(args) < 1 {
			return nil, errors.New("getRowsTableOne failed. Must include at least key values")
		}

		var columns []shim.Column
		col1Val := args[0]
		col1 := shim.Column{Value: &shim.Column_String_{String_: col1Val}}
		columns = append(columns, col1)
	if len(args) > 1 {
			col2Int, err := strconv.ParseInt(args[1], 10, 32)
			if err != nil {
				return nil, errors.New("getRowsTableTwo failed. arg[1] must be convertable to int32")
			}

     col2Val := int32(col2Int)
			col2 := shim.Column{Value: &shim.Column_Int32{Int32: col2Val}}
			columns = append(columns, col2)

         }

         rowChannel, err := stub.GetRows("tableOne", columns)

        if err != nil {
			return nil, fmt.Errorf("getRowsTableTwo operation failed. %s", err)
		}

		var rows []shim.Row
		for {
			select {
			case row, ok := <-rowChannel:
				if !ok {
					rowChannel = nil
				} else {
					rows = append(rows, row)
				}
			}
			if rowChannel == nil {
				break
			}
		}
      jsonRows, err := json.Marshal(rows)
	  if err != nil {
			return nil, fmt.Errorf("getRowsTableTwo operation failed. Error marshaling JSON: %s", err)
		}

		return jsonRows, nil

	default:
		return nil, errors.New("Unsupported operation")
	}
}

func main() {
	err := shim.Start(new(SimpleChaincode))
	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

func createTableOne(stub shim.ChaincodeStubInterface) error {
	// Create table one
	var columnDefsTableOne []*shim.ColumnDefinition
	columnOneTableOneDef := shim.ColumnDefinition{Name: "colOneTableOne",
		Type: shim.ColumnDefinition_STRING, Key: true}
	columnTwoTableOneDef := shim.ColumnDefinition{Name: "colTwoTableOne",
		Type: shim.ColumnDefinition_INT32, Key: false}
	columnThreeTableOneDef := shim.ColumnDefinition{Name: "colThreeTableOne",
		Type: shim.ColumnDefinition_INT32, Key: false}
	columnDefsTableOne = append(columnDefsTableOne, &columnOneTableOneDef)
	columnDefsTableOne = append(columnDefsTableOne, &columnTwoTableOneDef)
	columnDefsTableOne = append(columnDefsTableOne, &columnThreeTableOneDef)
	return stub.CreateTable("tableOne", columnDefsTableOne)
}

<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"math"
	"strings"
	"time"
)

func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {

	// read filer remote storage mount mappings
	if detectErr := option.collectRemoteStorageConf(); detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := option.makeBucketedEventProcessor(filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, option.bucketsDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.bucketsDir)

	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {

	handleCreateBucket := func(entry *filer_pb.Entry) error {
		if !entry.IsDirectory {
			return nil
		}
		remoteConf, found := option.remoteConfs[*option.createBucketAt]
		if !found {
			return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt)
		}

		client, err := remote_storage.GetRemoteStorage(remoteConf)
		if err != nil {
			return err
		}

		glog.V(0).Infof("create bucket %s", entry.Name)
		if err := client.CreateBucket(entry.Name); err != nil {
			return err
		}
		return nil
	}
	handleDeleteBucket := func(entry *filer_pb.Entry) error {
		if !entry.IsDirectory {
			return nil
		}

		client, err := option.findRemoteStorageClient(entry.Name)
		if err != nil {
			return err
		}

		glog.V(0).Infof("delete bucket %s", entry.Name)
		if err := client.DeleteBucket(entry.Name); err != nil {
			return err
		}
		return nil
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry == nil {
			return nil
		}
		if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
			newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
			if readErr != nil {
				return fmt.Errorf("unmarshal mappings: %v", readErr)
			}
			option.mappings = newMappings
		}
		if strings.HasSuffix(message.NewEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
			conf := &remote_pb.RemoteConf{}
			if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
				return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
			}
			option.remoteConfs[conf.Name] = conf
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if message.NewParentPath == option.bucketsDir {
				return handleCreateBucket(message.NewEntry)
			}
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(message.NewParentPath)
			if !ok {
				return nil
			}
			client, err := remote_storage.GetRemoteStorage(remoteStorage)
			if err != nil {
				return err
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(bucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			if resp.Directory == option.bucketsDir {
				return handleDeleteBucket(message.OldEntry)
			}
			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
			if !ok {
				return nil
			}
			client, err := remote_storage.GetRemoteStorage(remoteStorage)
			if err != nil {
				return err
			}
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(bucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			if resp.Directory == option.bucketsDir {
				if message.NewParentPath == option.bucketsDir {
					if message.OldEntry.Name == message.NewEntry.Name {
						return nil
					}
					if err := handleCreateBucket(message.NewEntry); err != nil {
						return err
					}
					if err := handleDeleteBucket(message.OldEntry); err != nil {
						return err
					}
				}
			}
			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
			if oldOk && newOk {
				if !shouldSendToRemote(message.NewEntry) {
					glog.V(2).Infof("skipping updating: %+v", resp)
					return nil
				}
				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
				if err != nil {
					return err
				}
				if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
					// update the same entry
					if message.NewEntry.IsDirectory {
						// update directory property
						return nil
					}
					if filer.IsSameData(message.OldEntry, message.NewEntry) {
						glog.V(2).Infof("update meta: %+v", resp)
						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
					} else {
						newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
						reader := filer.NewFileReader(filerSource, message.NewEntry)
						glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
						remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
						if writeErr != nil {
							return writeErr
						}
						return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
					}
				}
			}

			// the following is entry rename
			if oldOk {
				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
				if err != nil {
					return err
				}
				oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
				if message.OldEntry.IsDirectory {
					return client.RemoveDirectory(oldDest)
				}
				glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
				if err := client.DeleteFile(oldDest); err != nil {
					return err
				}
			}
			if newOk {
				if !shouldSendToRemote(message.NewEntry) {
					glog.V(2).Infof("skipping updating: %+v", resp)
					return nil
				}
				client, err := remote_storage.GetRemoteStorage(newRemoteStorage)
				if err != nil {
					return err
				}
				newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
				if message.NewEntry.IsDirectory {
					return client.WriteDirectory(newDest, message.NewEntry)
				}
				reader := filer.NewFileReader(filerSource, message.NewEntry)
				glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
				remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
				if writeErr != nil {
					return writeErr
				}
				return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
			}
		}

		return nil
	}
	return eachEntryFunc, nil
}

func (option *RemoteSyncOptions)findRemoteStorageClient(bucketName string) (remote_storage.RemoteStorageClient, error) {
	bucket := util.FullPath(option.bucketsDir).Child(bucketName)

	remoteStorageMountLocation, isMounted := option.mappings.Mappings[string(bucket)]
	if !isMounted {
		return nil, fmt.Errorf("%s is not mounted", bucket)
	}
	remoteConf, hasClient := option.remoteConfs[remoteStorageMountLocation.Name]
	if !hasClient {
		return nil, fmt.Errorf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
	}

	client, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return nil, err
	}
	return client, nil
}

func (option *RemoteSyncOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
	bucket, ok = extractBucketPath(option.bucketsDir, actualDir)
	if !ok {
		return "", nil, nil, false
	}
	var isMounted bool
	remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)]
	if !isMounted {
		glog.Warningf("%s is not mounted", bucket)
		return "", nil, nil, false
	}
	var hasClient bool
	remoteConf, hasClient = option.remoteConfs[remoteStorageMountLocation.Name]
	if !hasClient {
		glog.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
		return "", nil, nil, false
	}
	return bucket, remoteStorageMountLocation, remoteConf, true
}

func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
	if !strings.HasPrefix(dir, bucketsDir+"/") {
		return "", false
	}
	parts := strings.SplitN(dir[len(bucketsDir)+1:], "/", 2)
	return util.FullPath(bucketsDir).Child(parts[0]), true
}

func (option *RemoteSyncOptions) collectRemoteStorageConf() (err error) {

	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, *option.filerAddress); err != nil {
		return err
	} else {
		option.mappings = mappings
	}

	option.remoteConfs = make(map[string]*remote_pb.RemoteConf)
	err = filer_pb.List(option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error {
		if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
			return nil
		}
		conf := &remote_pb.RemoteConf{}
		if err := proto.Unmarshal(entry.Content, conf); err != nil {
			return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
		}
		option.remoteConfs[conf.Name] = conf
		return nil
	}, "", false, math.MaxUint32)

	return
}<commit_msg>update remote storage mount mapping<commit_after>package command

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
	"github.com/chrislusf/seaweedfs/weed/remote_storage"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/golang/protobuf/proto"
	"math"
	"strings"
	"time"
)

func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {

	// read filer remote storage mount mappings
	if detectErr := option.collectRemoteStorageConf(); detectErr != nil {
		return fmt.Errorf("read mount info: %v", detectErr)
	}

	eachEntryFunc, err := option.makeBucketedEventProcessor(filerSource)
	if err != nil {
		return err
	}

	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
		lastTime := time.Unix(0, lastTsNs)
		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, option.bucketsDir, lastTsNs)
	})

	lastOffsetTs := collectLastSyncOffset(option, option.bucketsDir)

	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
}

func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {

	handleCreateBucket := func(entry *filer_pb.Entry) error {
		if !entry.IsDirectory {
			return nil
		}
		remoteConf, found := option.remoteConfs[*option.createBucketAt]
		if !found {
			return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt)
		}

		client, err := remote_storage.GetRemoteStorage(remoteConf)
		if err != nil {
			return err
		}

		glog.V(0).Infof("create bucket %s", entry.Name)
		if err := client.CreateBucket(entry.Name); err != nil {
			return err
		}

		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)
		remoteLocation := &remote_pb.RemoteStorageLocation{
			Name:   *option.createBucketAt,
			Bucket: entry.Name,
			Path:   "/",
		}

		return filer.InsertMountMapping(option, string(bucketPath), remoteLocation)

	}
	handleDeleteBucket := func(entry *filer_pb.Entry) error {
		if !entry.IsDirectory {
			return nil
		}

		client, err := option.findRemoteStorageClient(entry.Name)
		if err != nil {
			return err
		}

		glog.V(0).Infof("delete bucket %s", entry.Name)
		if err := client.DeleteBucket(entry.Name); err != nil {
			return err
		}

		bucketPath := util.FullPath(option.bucketsDir).Child(entry.Name)

		return filer.DeleteMountMapping(option, string(bucketPath))
	}

	handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if message.NewEntry != nil {
			// update
			if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
				newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
				if readErr != nil {
					return fmt.Errorf("unmarshal mappings: %v", readErr)
				}
				option.mappings = newMappings
			}
			if strings.HasSuffix(message.NewEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
				conf := &remote_pb.RemoteConf{}
				if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
				}
				option.remoteConfs[conf.Name] = conf
			}
		} else if message.OldEntry != nil {
			// deletion
			if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
				conf := &remote_pb.RemoteConf{}
				if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil {
					return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.OldEntry.Name, err)
				}
				delete(option.remoteConfs, conf.Name)
			}
		}

		return nil
	}

	eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
			return handleEtcRemoteChanges(resp)
		}

		if message.OldEntry == nil && message.NewEntry == nil {
			return nil
		}
		if message.OldEntry == nil && message.NewEntry != nil {
			if message.NewParentPath == option.bucketsDir {
				return handleCreateBucket(message.NewEntry)
			}
			if !filer.HasData(message.NewEntry) {
				return nil
			}
			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(message.NewParentPath)
			if !ok {
				return nil
			}
			client, err := remote_storage.GetRemoteStorage(remoteStorage)
			if err != nil {
				return err
			}
			glog.V(2).Infof("create: %+v", resp)
			if !shouldSendToRemote(message.NewEntry) {
				glog.V(2).Infof("skipping creating: %+v", resp)
				return nil
			}
			dest := toRemoteStorageLocation(bucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
			if message.NewEntry.IsDirectory {
				glog.V(0).Infof("mkdir  %s", remote_storage.FormatLocation(dest))
				return client.WriteDirectory(dest, message.NewEntry)
			}
			glog.V(0).Infof("create %s", remote_storage.FormatLocation(dest))
			reader := filer.NewFileReader(filerSource, message.NewEntry)
			remoteEntry, writeErr := client.WriteFile(dest, message.NewEntry, reader)
			if writeErr != nil {
				return writeErr
			}
			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
		}
		if message.OldEntry != nil && message.NewEntry == nil {
			if resp.Directory == option.bucketsDir {
				return handleDeleteBucket(message.OldEntry)
			}
			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
			if !ok {
				return nil
			}
			client, err := remote_storage.GetRemoteStorage(remoteStorage)
			if err != nil {
				return err
			}
			glog.V(2).Infof("delete: %+v", resp)
			dest := toRemoteStorageLocation(bucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
			if message.OldEntry.IsDirectory {
				glog.V(0).Infof("rmdir  %s", remote_storage.FormatLocation(dest))
				return client.RemoveDirectory(dest)
			}
			glog.V(0).Infof("delete %s", remote_storage.FormatLocation(dest))
			return client.DeleteFile(dest)
		}
		if message.OldEntry != nil && message.NewEntry != nil {
			if resp.Directory == option.bucketsDir {
				if message.NewParentPath == option.bucketsDir {
					if message.OldEntry.Name == message.NewEntry.Name {
						return nil
					}
					if err := handleCreateBucket(message.NewEntry); err != nil {
						return err
					}
					if err := handleDeleteBucket(message.OldEntry); err != nil {
						return err
					}
				}
			}
			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
			if oldOk && newOk {
				if !shouldSendToRemote(message.NewEntry) {
					glog.V(2).Infof("skipping updating: %+v", resp)
					return nil
				}
				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
				if err != nil {
					return err
				}
				if resp.Directory == message.NewParentPath && message.OldEntry.Name == message.NewEntry.Name {
					// update the same entry
					if message.NewEntry.IsDirectory {
						// update directory property
						return nil
					}
					if filer.IsSameData(message.OldEntry, message.NewEntry) {
						glog.V(2).Infof("update meta: %+v", resp)
						oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
						return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
					} else {
						newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
						reader := filer.NewFileReader(filerSource, message.NewEntry)
						glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
						remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
						if writeErr != nil {
							return writeErr
						}
						return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
					}
				}
			}

			// the following is entry rename
			if oldOk {
				client, err := remote_storage.GetRemoteStorage(oldRemoteStorage)
				if err != nil {
					return err
				}
				oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
				if message.OldEntry.IsDirectory {
					return client.RemoveDirectory(oldDest)
				}
				glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
				if err := client.DeleteFile(oldDest); err != nil {
					return err
				}
			}
			if newOk {
				if !shouldSendToRemote(message.NewEntry) {
					glog.V(2).Infof("skipping updating: %+v", resp)
					return nil
				}
				client, err := remote_storage.GetRemoteStorage(newRemoteStorage)
				if err != nil {
					return err
				}
				newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
				if message.NewEntry.IsDirectory {
					return client.WriteDirectory(newDest, message.NewEntry)
				}
				reader := filer.NewFileReader(filerSource, message.NewEntry)
				glog.V(0).Infof("create %s", remote_storage.FormatLocation(newDest))
				remoteEntry, writeErr := client.WriteFile(newDest, message.NewEntry, reader)
				if writeErr != nil {
					return writeErr
				}
				return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
			}
		}

		return nil
	}
	return eachEntryFunc, nil
}

func (option *RemoteSyncOptions)findRemoteStorageClient(bucketName string) (remote_storage.RemoteStorageClient, error) {
	bucket := util.FullPath(option.bucketsDir).Child(bucketName)

	remoteStorageMountLocation, isMounted := option.mappings.Mappings[string(bucket)]
	if !isMounted {
		return nil, fmt.Errorf("%s is not mounted", bucket)
	}
	remoteConf, hasClient := option.remoteConfs[remoteStorageMountLocation.Name]
	if !hasClient {
		return nil, fmt.Errorf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
	}

	client, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return nil, err
	}
	return client, nil
}

func (option *RemoteSyncOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
	bucket, ok = extractBucketPath(option.bucketsDir, actualDir)
	if !ok {
		return "", nil, nil, false
	}
	var isMounted bool
	remoteStorageMountLocation, isMounted = option.mappings.Mappings[string(bucket)]
	if !isMounted {
		glog.Warningf("%s is not mounted", bucket)
		return "", nil, nil, false
	}
	var hasClient bool
	remoteConf, hasClient = option.remoteConfs[remoteStorageMountLocation.Name]
	if !hasClient {
		glog.Warningf("%s mounted to un-configured %+v", bucket, remoteStorageMountLocation)
		return "", nil, nil, false
	}
	return bucket, remoteStorageMountLocation, remoteConf, true
}

func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
	if !strings.HasPrefix(dir, bucketsDir+"/") {
		return "", false
	}
	parts := strings.SplitN(dir[len(bucketsDir)+1:], "/", 2)
	return util.FullPath(bucketsDir).Child(parts[0]), true
}

func (option *RemoteSyncOptions) collectRemoteStorageConf() (err error) {

	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, *option.filerAddress); err != nil {
		return err
	} else {
		option.mappings = mappings
	}

	option.remoteConfs = make(map[string]*remote_pb.RemoteConf)
	err = filer_pb.List(option, filer.DirectoryEtcRemote, "", func(entry *filer_pb.Entry, isLast bool) error {
		if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
			return nil
		}
		conf := &remote_pb.RemoteConf{}
		if err := proto.Unmarshal(entry.Content, conf); err != nil {
			return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
		}
		option.remoteConfs[conf.Name] = conf
		return nil
	}, "", false, math.MaxUint32)

	return
}<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"html/template"
	"net/http"
	"runtime"
	"time"

	"github.com/Jeffail/gabs"
	log "github.com/Sirupsen/logrus"
	"github.com/blevesearch/bleve"
	"github.com/gin-gonic/contrib/ginrus"
	"github.com/gin-gonic/gin"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/exp"
	//"github.com/rcrowley/go-metrics/exp"
	"gopkg.in/tylerb/graceful.v1"
)

var (
	nAlexRequests metrics.Counter
	nRestRequests metrics.Counter
	nHtmlRequests metrics.Counter
)

func init() {
	nAlexRequests = metrics.NewCounter()
	metrics.Register("nAlexRequests", nAlexRequests)

	nRestRequests = metrics.NewCounter()
	metrics.Register("nRestRequests", nRestRequests)

	nHtmlRequests = metrics.NewCounter()
	metrics.Register("nHtmlRequests", nHtmlRequests)

	// Respond with a function call every time they are called
	metrics.NewRegisteredFunctionalGauge("ngoroutines", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumGoroutine()) })
	metrics.NewRegisteredFunctionalGauge("ncgocalls", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumCgoCall()) })
	metrics.NewRegisteredFunctionalGauge("ncpus", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumCPU()) })
}

type ServerConfig struct {
	Port int
	// Include index hash here too
	BuildCommit         string
	BuildBranch         string
	Index               bleve.Index
	ShouldValidateAlexa bool
	template            *template.Template
}

func (s *ServerConfig) VersionString() string {
	return fmt.Sprintf("%s (%s)", s.BuildBranch, s.BuildCommit)
}

func (s *ServerConfig) StartServer() {
	log.WithFields(log.Fields{
		"port":   s.Port,
		"commit": s.BuildCommit,
		"branch": s.BuildBranch,
	}).Info("Starting server")

	var err error
	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		panic(err)
	}

	gin.SetMode(gin.ReleaseMode)
	r := gin.New()
	r.Use(ginrus.Ginrus(log.StandardLogger(), time.RFC3339, true))
	r.Use(gin.Recovery())
	r.GET("/", htmlHandler(s))
	r.GET("/opt/version", versionHandler(s))
	r.GET("/opt/metrics", gin.WrapH(exp.ExpHandler(metrics.DefaultRegistry)))
	r.GET("/search", searchHandler(s))
	r.POST("/alexa/search", alexaSearchHandler(s))

	log.WithFields(log.Fields{
		"port": s.Port,
	}).Info("Starting server")

	srv := &graceful.Server{
		Timeout: 10 * time.Second,
		Server: &http.Server{
			Addr:    fmt.Sprintf(":%d", s.Port),
			Handler: r,
		},
		BeforeShutdown: func() bool {
			log.Warn("Starting shutdown.")
			return true
		},
	}
	srv.ListenAndServe()

	log.Warn("Everything safely closed. Exiting main process.")
}

// Top level handlers

// Return version status information
func versionHandler(s *ServerConfig) gin.HandlerFunc {
	return func(c *gin.Context) {
		status := make(map[string]interface{})
		status["commit"] = s.BuildCommit
		status["branch"] = s.BuildBranch
		c.JSON(200, status)
	}
}

// Respond to search requests
// FIXME: Add tests
// FIXME: Break down into smaller functions
func alexaSearchHandler(s *ServerConfig) gin.HandlerFunc {
	return func(c *gin.Context) {
		nAlexRequests.Inc(1)

		if s.ShouldValidateAlexa {
			if err := s.verifyRequestIsAlexa(c); err != nil {
				// Actual response is set inside this function
				return
			}
		}

		resp := s.getNewResponseTemplate()

		req, err := gabs.ParseJSONBuffer(c.Request.Body)
		if err != nil || req == nil {
			log.WithFields(log.Fields{
				"err": err,
				"req": req,
			}).Error("Error while parsing request body")
			c.JSON(http.StatusBadRequest, resp.Data())
			return
		}

		if s.ShouldValidateAlexa {
			if err := s.validateAppId(c, req); err != nil {
				// Actual response is set inside this function
				return
			}
		}

		t, err := getRequestType(req)
		if err != nil {
			log.WithFields(log.Fields{
				"err": err,
			}).Error("Error getting request type")
			c.JSON(http.StatusBadRequest, resp.Data())
			return
		}

		switch t {
		case intentRequest:
			s.handleIntentRequest(c, req, resp)
		case launchRequest:
			s.handleLaunchRequest(c, req, resp)
		case sessionEndedRequest:
			s.handleSessionEndedRequest(c, req, resp)
		default:
			log.WithFields(log.Fields{
				"type": t,
			}).Error("Unknown request type")
			c.JSON(http.StatusBadRequest, resp.Data())
		}
	}
}

// Home page
// Heavily borrowed from: https://github.com/ekanite/ekanite/blob/master/server_http.go
const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.2.7/semantic.css">
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.2.7/components/card.min.css">

{{ $.Headline }}

Query language reference: bleve

	<form class="ui form" action="/" method="GET">

	    Query
	    <input type="text" name="q" value="{{ $.Query }}">

	    Num Results
		<select name="size" class="ui fluid dropdown">

			<option value="{{ $.Size }}" selected="selected">{{ $.Size }}</option>
			<option value="10">10</option>
			<option value="20">20</option>
			<option value="100">100</option>
		</select>

	  <button class="ui button" type="submit">Search</button>
	</form>
{{ if $.ReturnResults }}

	{{range $nresult, $result := $.Hits }}

				{{ $result.Fields.Book }}

					{{ $result.Fields.Chapter }}:{{ $result.Fields.Verse }}

		      {{ $nresult }}
		      {{ $result.Fields.Version }}

{{ $result.Fields.Text }}

	{{ end }}

{{ end }}

`
<commit_msg>Margin updates<commit_after>package server

import (
	"fmt"
	"html/template"
	"net/http"
	"runtime"
	"time"

	"github.com/Jeffail/gabs"
	log "github.com/Sirupsen/logrus"
	"github.com/blevesearch/bleve"
	"github.com/gin-gonic/contrib/ginrus"
	"github.com/gin-gonic/gin"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/exp"
	//"github.com/rcrowley/go-metrics/exp"
	"gopkg.in/tylerb/graceful.v1"
)

var (
	nAlexRequests metrics.Counter
	nRestRequests metrics.Counter
	nHtmlRequests metrics.Counter
)

func init() {
	nAlexRequests = metrics.NewCounter()
	metrics.Register("nAlexRequests", nAlexRequests)

	nRestRequests = metrics.NewCounter()
	metrics.Register("nRestRequests", nRestRequests)

	nHtmlRequests = metrics.NewCounter()
	metrics.Register("nHtmlRequests", nHtmlRequests)

	// Respond with a function call every time they are called
	metrics.NewRegisteredFunctionalGauge("ngoroutines", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumGoroutine()) })
	metrics.NewRegisteredFunctionalGauge("ncgocalls", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumCgoCall()) })
	metrics.NewRegisteredFunctionalGauge("ncpus", metrics.DefaultRegistry, func() int64 { return int64(runtime.NumCPU()) })
}

type ServerConfig struct {
	Port int
	// Include index hash here too
	BuildCommit         string
	BuildBranch         string
	Index               bleve.Index
	ShouldValidateAlexa bool
	template            *template.Template
}

func (s *ServerConfig) VersionString() string {
	return fmt.Sprintf("%s (%s)", s.BuildBranch, s.BuildCommit)
}

func (s *ServerConfig) StartServer() {
	log.WithFields(log.Fields{
		"port":   s.Port,
		"commit": s.BuildCommit,
		"branch": s.BuildBranch,
	}).Info("Starting server")

	var err error
	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		panic(err)
	}

	gin.SetMode(gin.ReleaseMode)
	r := gin.New()
	r.Use(ginrus.Ginrus(log.StandardLogger(), time.RFC3339, true))
	r.Use(gin.Recovery())
	r.GET("/", htmlHandler(s))
	r.GET("/opt/version", versionHandler(s))
	r.GET("/opt/metrics", gin.WrapH(exp.ExpHandler(metrics.DefaultRegistry)))
	r.GET("/search", searchHandler(s))
	r.POST("/alexa/search", alexaSearchHandler(s))

	log.WithFields(log.Fields{
		"port": s.Port,
	}).Info("Starting server")

	srv := &graceful.Server{
		Timeout: 10 * time.Second,
		Server: &http.Server{
			Addr:    fmt.Sprintf(":%d", s.Port),
			Handler: r,
		},
		BeforeShutdown: func() bool {
			log.Warn("Starting shutdown.")
			return true
		},
	}
	srv.ListenAndServe()

	log.Warn("Everything safely closed. Exiting main process.")
}

// Top level handlers

// Return version status information
func versionHandler(s *ServerConfig) gin.HandlerFunc {
	return func(c *gin.Context) {
		status := make(map[string]interface{})
		status["commit"] = s.BuildCommit
		status["branch"] = s.BuildBranch
		c.JSON(200, status)
	}
}

// Respond to search requests
// FIXME: Add tests
// FIXME: Break down into smaller functions
func alexaSearchHandler(s *ServerConfig) gin.HandlerFunc {
	return func(c *gin.Context) {
		nAlexRequests.Inc(1)

		if s.ShouldValidateAlexa {
			if err := s.verifyRequestIsAlexa(c); err != nil {
				// Actual response is set inside this function
				return
			}
		}

		resp := s.getNewResponseTemplate()

		req, err := gabs.ParseJSONBuffer(c.Request.Body)
		if err != nil || req == nil {
			log.WithFields(log.Fields{
				"err": err,
				"req": req,
			}).Error("Error while parsing request body")
			c.JSON(http.StatusBadRequest, resp.Data())
			return
		}

		if s.ShouldValidateAlexa {
			if err := s.validateAppId(c, req); err != nil {
				// Actual response is set inside this function
				return
			}
		}

		t, err := getRequestType(req)
		if err != nil {
			log.WithFields(log.Fields{
				"err": err,
			}).Error("Error getting request type")
			c.JSON(http.StatusBadRequest, resp.Data())
			return
		}

		switch t {
		case intentRequest:
			s.handleIntentRequest(c, req, resp)
		case launchRequest:
			s.handleLaunchRequest(c, req, resp)
		case sessionEndedRequest:
			s.handleSessionEndedRequest(c, req, resp)
		default:
			log.WithFields(log.Fields{
				"type": t,
			}).Error("Unknown request type")
			c.JSON(http.StatusBadRequest, resp.Data())
		}
	}
}

// Home page
// Heavily borrowed from: https://github.com/ekanite/ekanite/blob/master/server_http.go
const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.2.7/semantic.css">
<link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/semantic-ui/2.2.7/components/card.min.css">

{{ $.Headline }}

Query language reference: bleve

	<form class="ui form" action="/" method="GET">

	    Query
	    <input type="text" name="q" value="{{ $.Query }}">

	    Num Results
		<select name="size" class="ui fluid dropdown">

			<option value="{{ $.Size }}" selected="selected">{{ $.Size }}</option>
			<option value="10">10</option>
			<option value="20">20</option>
			<option value="100">100</option>
		</select>

	  <button class="ui button" type="submit">Search</button>
	</form>
{{ if $.ReturnResults }}

	{{range $nresult, $result := $.Hits }}

				{{ $result.Fields.Book }}

					{{ $result.Fields.Chapter }}:{{ $result.Fields.Verse }}

		      {{ $nresult }}
		      {{ $result.Fields.Version }}

{{ $result.Fields.Text }}

	{{ end }}

{{ end }}

`
<|endoftext|>


<commit_before>package mpawsdynamodb

import (
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace              = "AWS/DynamoDB"
	metricsTypeAverage     = "Average"
	metricsTypeSum         = "Sum"
	metricsTypeMaximum     = "Maximum"
	metricsTypeMinimum     = "Minimum"
	metricsTypeSampleCount = "SampleCount"
)

// has 1 CloudWatch MetricName and corresponding N Mackerel Metrics
type metricsGroup struct {
	CloudWatchName string
	Metrics        []metric
}

type metric struct {
	MackerelName string
	Type         string
}

// DynamoDBPlugin mackerel plugin for aws kinesis
type DynamoDBPlugin struct {
	TableName string
	Prefix    string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p DynamoDBPlugin) MetricKeyPrefix() string {
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *DynamoDBPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

func transformAndAppendDatapoint(dp *cloudwatch.Datapoint, dataType string, label string, stats map[string]interface{}) map[string]interface{} {
	if dp != nil {
		switch dataType {
		case metricsTypeAverage:
			stats[label] = *dp.Average
		case metricsTypeSum:
			stats[label] = *dp.Sum
		case metricsTypeMaximum:
			stats[label] = *dp.Maximum
		case metricsTypeMinimum:
			stats[label] = *dp.Minimum
		case metricsTypeSampleCount:
			stats[label] = *dp.SampleCount
		}
	}
	return stats
}

// fetch metrics which takes "Operation" dimensions querying both ListMetrics and GetMetricsStatistics
func fetchOperationWildcardMetrics(cw cloudwatchiface.CloudWatchAPI, mg metricsGroup, baseDimensions []*cloudwatch.Dimension) (map[string]interface{}, error) {
	// get available dimensions
	dimensionFilters := make([]*cloudwatch.DimensionFilter, len(baseDimensions))
	for i, dimension := range baseDimensions {
		dimensionFilters[i] = &cloudwatch.DimensionFilter{
			Name:  dimension.Name,
			Value: dimension.Value,
		}
	}
	input := &cloudwatch.ListMetricsInput{
		Dimensions: dimensionFilters,
		Namespace:  aws.String(namespace),
		MetricName: aws.String(mg.CloudWatchName),
	}
	// ListMetrics can retrieve up to 500 metrics, but DynamoDB Operations are apparently less than 500
	res, err := cw.ListMetrics(input)
	if err != nil {
		return nil, err
	}

	stats := make(map[string]interface{})

	// get datapoints with retrieved dimensions
	for _, cwMetric := range res.Metrics {
		dimensions := cwMetric.Dimensions
		// extract operation name
		var operation *string
		for _, d := range dimensions {
			if *d.Name == "Operation" {
				operation = d.Value
				break
			}
		}
		if operation == nil {
			log.Printf("Unexpected dimension, skip: %s", dimensions)
			continue
		}

		dp, err := getLastPointFromCloudWatch(cw, mg, dimensions)
		if err != nil {
			return nil, nil
		}
		if dp != nil {
			for _, met := range mg.Metrics {
				label := strings.Replace(met.MackerelName, "#", *operation, 1)
				stats = transformAndAppendDatapoint(dp, met.Type, label, stats)
			}
		}
	}

	return stats, nil
}

// getLastPoint fetches a CloudWatch metric and parse
func getLastPointFromCloudWatch(cw cloudwatchiface.CloudWatchAPI, metric metricsGroup, dimensions []*cloudwatch.Dimension) (*cloudwatch.Datapoint, error) {
	now := time.Now()
	statsInput := make([]*string, len(metric.Metrics))
	for i, typ := range metric.Metrics {
		statsInput[i] = aws.String(typ.Type)
	}
	input := &cloudwatch.GetMetricStatisticsInput{
		// 8 min, since some metrics are aggregated over 5 min
		StartTime:  aws.Time(now.Add(time.Duration(480) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: statsInput,
		Namespace:  aws.String(namespace),
		Dimensions: dimensions,
	}
	response, err := cw.GetMetricStatistics(input)
	if err != nil {
		return nil, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return nil, nil
	}

	latest := new(time.Time)
	var latestDp *cloudwatch.Datapoint
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestDp = dp
	}

	return latestDp, nil
}

var defaultMetricsGroup = []metricsGroup{
	{CloudWatchName: "ConditionalCheckFailedRequests", Metrics: []metric{
		{MackerelName: "ConditionalCheckFailedRequests", Type: metricsTypeSum},
	}},
	{CloudWatchName: "ConsumedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedReadCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedReadCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ConsumedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedWriteCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedWriteCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ProvisionedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedReadCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "ProvisionedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedWriteCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "WriteThrottleEvents", Metrics: []metric{
		{MackerelName: "WriteThrottleEvents", Type: metricsTypeSum},
	}},
}

var operationalMetricsGroup = []metricsGroup{
	{CloudWatchName: "SuccessfulRequestLatency", Metrics: []metric{
		{MackerelName: "SuccessfulRequests.#", Type: metricsTypeSampleCount},
		{MackerelName: "SuccessfulRequestLatency.#.Minimum", Type: metricsTypeMinimum},
		{MackerelName: "SuccessfulRequestLatency.#.Maximum", Type: metricsTypeMaximum},
		{MackerelName: "SuccessfulRequestLatency.#.Average", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ThrottledRequests", Metrics: []metric{
		{MackerelName: "ThrottledRequests.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "ReturnedItemCount", Metrics: []metric{
		{MackerelName: "ReturnedItemCount.#", Type: metricsTypeAverage},
	}},
}

// FetchMetrics fetch the metrics
func (p DynamoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	stats := make(map[string]interface{})

	tableDimensions := []*cloudwatch.Dimension{{
		Name:  aws.String("TableName"),
		Value: aws.String(p.TableName),
	}}
	for _, met := range defaultMetricsGroup {
		dp, err := getLastPointFromCloudWatch(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for _, m := range met.Metrics {
				stats = transformAndAppendDatapoint(dp, m.Type, m.MackerelName, stats)
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	for _, met := range operationalMetricsGroup {
		operationalStats, err := fetchOperationWildcardMetrics(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for name, s := range operationalStats {
				stats[name] = s
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return transformMetrics(stats), nil
}

// TransformMetrics converts some of datapoints to post differences of two metrics
func transformMetrics(stats map[string]interface{}) map[string]interface{} {
	// Although stats are interface{}, those values from cloudwatch.Datapoint are guaranteed to be numerical
	if consumedReadCapacitySum, ok := stats["ConsumedReadCapacityUnitsSum"].(float64); ok {
		stats["ConsumedReadCapacityUnitsNormalized"] = consumedReadCapacitySum / 60.0
	}
	if consumedWriteCapacitySum, ok := stats["ConsumedWriteCapacityUnitsSum"].(float64); ok {
		stats["ConsumedWriteCapacityUnitsNormalized"] = consumedWriteCapacitySum / 60.0
	}
	return stats
}

// GraphDefinition of DynamoDBPlugin
func (p DynamoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)
	labelPrefix = strings.Replace(labelPrefix, "Dynamodb", "DynamoDB", -1)

	var graphdef = map[string]mp.Graphs{
		"ReadCapacity": {
			Label: (labelPrefix + " Read Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedReadCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedReadCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedReadCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"WriteCapacity": {
			Label: (labelPrefix + " Write Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedWriteCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedWriteCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedWriteCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"ThrottledEvents": {
			Label: (labelPrefix + " Throttle Events"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThrottleEvents", Label: "Read"},
				{Name: "WriteThrottleEvents", Label: "Write"},
			},
		},
		"ConditionalCheckFailedRequests": {
			Label: (labelPrefix + " ConditionalCheckFailedRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ConditionalCheckFailedRequests", Label: "Counts"},
			},
		},
		"ThrottledRequests": {
			Label: (labelPrefix + " ThrottledRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"SystemErrors": {
			Label: (labelPrefix + " SystemErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"UserErrors": {
			Label: (labelPrefix + " UserErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1", Stacked: true},
			},
		},
		"ReturnedItemCount": {
			Label: (labelPrefix + " ReturnedItemCount"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequests": {
			Label: (labelPrefix + " SuccessfulRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "*", Label: "%1"},
			},
		},
		"SuccessfulRequestLatency.#": {
			Label: (labelPrefix + " SuccessfulRequestLatency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "Minimum", Label: "Min"},
				{Name: "Maximum", Label: "Max"},
				{Name: "Average", Label: "Average"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optTableName := flag.String("table-name", "", "DynamoDB Table Name")
	optPrefix := flag.String("metric-key-prefix", "dynamodb", "Metric key prefix")
	flag.Parse()

	var plugin DynamoDBPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.TableName = *optTableName
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)

	helper.Run()
}
<commit_msg>[aws-dynamodb] Declare operational metrics explicitly in graphdefs<commit_after>package mpawsdynamodb

import (
	"flag"
	"log"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

const (
	namespace              = "AWS/DynamoDB"
	metricsTypeAverage     = "Average"
	metricsTypeSum         = "Sum"
	metricsTypeMaximum     = "Maximum"
	metricsTypeMinimum     = "Minimum"
	metricsTypeSampleCount = "SampleCount"
)

// has 1 CloudWatch MetricName and corresponding N Mackerel Metrics
type metricsGroup struct {
	CloudWatchName string
	Metrics        []metric
}

type metric struct {
	MackerelName string
	Type         string
}

// DynamoDBPlugin mackerel plugin for aws kinesis
type DynamoDBPlugin struct {
	TableName string
	Prefix    string

	AccessKeyID     string
	SecretAccessKey string
	Region          string
	CloudWatch      *cloudwatch.CloudWatch
}

// MetricKeyPrefix interface for PluginWithPrefix
func (p DynamoDBPlugin) MetricKeyPrefix() string {
	return p.Prefix
}

// prepare creates CloudWatch instance
func (p *DynamoDBPlugin) prepare() error {
	sess, err := session.NewSession()
	if err != nil {
		return err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	p.CloudWatch = cloudwatch.New(sess, config)

	return nil
}

func transformAndAppendDatapoint(dp *cloudwatch.Datapoint, dataType string, label string, stats map[string]interface{}) map[string]interface{} {
	if dp != nil {
		switch dataType {
		case metricsTypeAverage:
			stats[label] = *dp.Average
		case metricsTypeSum:
			stats[label] = *dp.Sum
		case metricsTypeMaximum:
			stats[label] = *dp.Maximum
		case metricsTypeMinimum:
			stats[label] = *dp.Minimum
		case metricsTypeSampleCount:
			stats[label] = *dp.SampleCount
		}
	}
	return stats
}

// fetch metrics which takes "Operation" dimensions querying both ListMetrics and GetMetricsStatistics
func fetchOperationWildcardMetrics(cw cloudwatchiface.CloudWatchAPI, mg metricsGroup, baseDimensions []*cloudwatch.Dimension) (map[string]interface{}, error) {
	// get available dimensions
	dimensionFilters := make([]*cloudwatch.DimensionFilter, len(baseDimensions))
	for i, dimension := range baseDimensions {
		dimensionFilters[i] = &cloudwatch.DimensionFilter{
			Name:  dimension.Name,
			Value: dimension.Value,
		}
	}
	input := &cloudwatch.ListMetricsInput{
		Dimensions: dimensionFilters,
		Namespace:  aws.String(namespace),
		MetricName: aws.String(mg.CloudWatchName),
	}
	// ListMetrics can retrieve up to 500 metrics, but DynamoDB Operations are apparently less than 500
	res, err := cw.ListMetrics(input)
	if err != nil {
		return nil, err
	}

	stats := make(map[string]interface{})

	// get datapoints with retrieved dimensions
	for _, cwMetric := range res.Metrics {
		dimensions := cwMetric.Dimensions
		// extract operation name
		var operation *string
		for _, d := range dimensions {
			if *d.Name == "Operation" {
				operation = d.Value
				break
			}
		}
		if operation == nil {
			log.Printf("Unexpected dimension, skip: %s", dimensions)
			continue
		}

		dp, err := getLastPointFromCloudWatch(cw, mg, dimensions)
		if err != nil {
			return nil, nil
		}
		if dp != nil {
			for _, met := range mg.Metrics {
				label := strings.Replace(met.MackerelName, "#", *operation, 1)
				stats = transformAndAppendDatapoint(dp, met.Type, label, stats)
			}
		}
	}

	return stats, nil
}

// getLastPoint fetches a CloudWatch metric and parse
func getLastPointFromCloudWatch(cw cloudwatchiface.CloudWatchAPI, metric metricsGroup, dimensions []*cloudwatch.Dimension) (*cloudwatch.Datapoint, error) {
	now := time.Now()
	statsInput := make([]*string, len(metric.Metrics))
	for i, typ := range metric.Metrics {
		statsInput[i] = aws.String(typ.Type)
	}
	input := &cloudwatch.GetMetricStatisticsInput{
		// 8 min, since some metrics are aggregated over 5 min
		StartTime:  aws.Time(now.Add(time.Duration(480) * time.Second * -1)),
		EndTime:    aws.Time(now),
		MetricName: aws.String(metric.CloudWatchName),
		Period:     aws.Int64(60),
		Statistics: statsInput,
		Namespace:  aws.String(namespace),
		Dimensions: dimensions,
	}
	response, err := cw.GetMetricStatistics(input)
	if err != nil {
		return nil, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return nil, nil
	}

	latest := new(time.Time)
	var latestDp *cloudwatch.Datapoint
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestDp = dp
	}

	return latestDp, nil
}

var defaultMetricsGroup = []metricsGroup{
	{CloudWatchName: "ConditionalCheckFailedRequests", Metrics: []metric{
		{MackerelName: "ConditionalCheckFailedRequests", Type: metricsTypeSum},
	}},
	{CloudWatchName: "ConsumedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedReadCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedReadCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ConsumedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ConsumedWriteCapacityUnitsSum", Type: metricsTypeSum},
		{MackerelName: "ConsumedWriteCapacityUnitsAverage", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ProvisionedReadCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedReadCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "ProvisionedWriteCapacityUnits", Metrics: []metric{
		{MackerelName: "ProvisionedWriteCapacityUnits", Type: metricsTypeMinimum},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors", Type: metricsTypeSum},
	}},
	{CloudWatchName: "WriteThrottleEvents", Metrics: []metric{
		{MackerelName: "WriteThrottleEvents", Type: metricsTypeSum},
	}},
}

var operationalMetricsGroup = []metricsGroup{
	{CloudWatchName: "SuccessfulRequestLatency", Metrics: []metric{
		{MackerelName: "SuccessfulRequests.#", Type: metricsTypeSampleCount},
		{MackerelName: "SuccessfulRequestLatency.#.Minimum", Type: metricsTypeMinimum},
		{MackerelName: "SuccessfulRequestLatency.#.Maximum", Type: metricsTypeMaximum},
		{MackerelName: "SuccessfulRequestLatency.#.Average", Type: metricsTypeAverage},
	}},
	{CloudWatchName: "ThrottledRequests", Metrics: []metric{
		{MackerelName: "ThrottledRequests.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "SystemErrors", Metrics: []metric{
		{MackerelName: "SystemErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "UserErrors", Metrics: []metric{
		{MackerelName: "UserErrors.#", Type: metricsTypeSampleCount},
	}},
	{CloudWatchName: "ReturnedItemCount", Metrics: []metric{
		{MackerelName: "ReturnedItemCount.#", Type: metricsTypeAverage},
	}},
}

// FetchMetrics fetch the metrics
func (p DynamoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	stats := make(map[string]interface{})

	tableDimensions := []*cloudwatch.Dimension{{
		Name:  aws.String("TableName"),
		Value: aws.String(p.TableName),
	}}
	for _, met := range defaultMetricsGroup {
		dp, err := getLastPointFromCloudWatch(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for _, m := range met.Metrics {
				stats = transformAndAppendDatapoint(dp, m.Type, m.MackerelName, stats)
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	for _, met := range operationalMetricsGroup {
		operationalStats, err := fetchOperationWildcardMetrics(p.CloudWatch, met, tableDimensions)
		if err == nil {
			for name, s := range operationalStats {
				stats[name] = s
			}
		} else {
			log.Printf("%s: %s", met, err)
		}
	}
	return transformMetrics(stats), nil
}

// TransformMetrics converts some of datapoints to post differences of two metrics
func transformMetrics(stats map[string]interface{}) map[string]interface{} {
	// Although stats are interface{}, those values from cloudwatch.Datapoint are guaranteed to be numerical
	if consumedReadCapacitySum, ok := stats["ConsumedReadCapacityUnitsSum"].(float64); ok {
		stats["ConsumedReadCapacityUnitsNormalized"] = consumedReadCapacitySum / 60.0
	}
	if consumedWriteCapacitySum, ok := stats["ConsumedWriteCapacityUnitsSum"].(float64); ok {
		stats["ConsumedWriteCapacityUnitsNormalized"] = consumedWriteCapacitySum / 60.0
	}
	return stats
}

// GraphDefinition of DynamoDBPlugin
func (p DynamoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.Prefix)
	labelPrefix = strings.Replace(labelPrefix, "-", " ", -1)
	labelPrefix = strings.Replace(labelPrefix, "Dynamodb", "DynamoDB", -1)

	var graphdef = map[string]mp.Graphs{
		"ReadCapacity": {
			Label: (labelPrefix + " Read Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedReadCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedReadCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedReadCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"WriteCapacity": {
			Label: (labelPrefix + " Write Capacity Units"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "ProvisionedWriteCapacityUnits", Label: "Provisioned"},
				{Name: "ConsumedWriteCapacityUnitsNormalized", Label: "Consumed"},
				{Name: "ConsumedWriteCapacityUnitsAverage", Label: "Consumed (Average per request)"},
			},
		},
		"ThrottledEvents": {
			Label: (labelPrefix + " Throttle Events"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReadThrottleEvents", Label: "Read"},
				{Name: "WriteThrottleEvents", Label: "Write"},
			},
		},
		"ConditionalCheckFailedRequests": {
			Label: (labelPrefix + " ConditionalCheckFailedRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ConditionalCheckFailedRequests", Label: "Counts"},
			},
		},
		"ThrottledRequests": {
			Label: (labelPrefix + " ThrottledRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ThrottleRequests.PutItem", Label: "PutItem", Stacked: true},
				{Name: "ThrottleRequests.DeleteItem", Label: "DeleteItem", Stacked: true},
				{Name: "ThrottleRequests.UpdateItem", Label: "UpdateItem", Stacked: true},
				{Name: "ThrottleRequests.GetItem", Label: "GetItem", Stacked: true},
				{Name: "ThrottleRequests.BatchGetItem", Label: "BatchGetItem", Stacked: true},
				{Name: "ThrottleRequests.Scan", Label: "Scan", Stacked: true},
				{Name: "ThrottleRequests.Query", Label: "Query", Stacked: true},
				{Name: "ThrottleRequests.BatchWriteItem", Label: "BatchWriteItem", Stacked: true},
			},
		},
		"SystemErrors": {
			Label: (labelPrefix + " SystemErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "SystemErrors.PutItem", Label: "PutItem", Stacked: true},
				{Name: "SystemErrors.DeleteItem", Label: "DeleteItem", Stacked: true},
				{Name: "SystemErrors.UpdateItem", Label: "UpdateItem", Stacked: true},
				{Name: "SystemErrors.GetItem", Label: "GetItem", Stacked: true},
				{Name: "SystemErrors.BatchGetItem", Label: "BatchGetItem", Stacked: true},
				{Name: "SystemErrors.Scan", Label: "Scan", Stacked: true},
				{Name: "SystemErrors.Query", Label: "Query", Stacked: true},
				{Name: "SystemErrors.BatchWriteItem", Label: "BatchWriteItem", Stacked: true},
			},
		},
		"UserErrors": {
			Label: (labelPrefix + " UserErrors"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "UserErrors.PutItem", Label: "PutItem", Stacked: true},
				{Name: "UserErrors.DeleteItem", Label: "DeleteItem", Stacked: true},
				{Name: "UserErrors.UpdateItem", Label: "UpdateItem", Stacked: true},
				{Name: "UserErrors.GetItem", Label: "GetItem", Stacked: true},
				{Name: "UserErrors.BatchGetItem", Label: "BatchGetItem", Stacked: true},
				{Name: "UserErrors.Scan", Label: "Scan", Stacked: true},
				{Name: "UserErrors.Query", Label: "Query", Stacked: true},
				{Name: "UserErrors.BatchWriteItem", Label: "BatchWriteItem", Stacked: true},
			},
		},
		"ReturnedItemCount": {
			Label: (labelPrefix + " ReturnedItemCount"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "ReturnedItemCount.PutItem", Label: "PutItem"},
				{Name: "ReturnedItemCount.DeleteItem", Label: "DeleteItem"},
				{Name: "ReturnedItemCount.UpdateItem", Label: "UpdateItem"},
				{Name: "ReturnedItemCount.GetItem", Label: "GetItem"},
				{Name: "ReturnedItemCount.BatchGetItem", Label: "BatchGetItem"},
				{Name: "ReturnedItemCount.Scan", Label: "Scan"},
				{Name: "ReturnedItemCount.Query", Label: "Query"},
				{Name: "ReturnedItemCount.BatchWriteItem", Label: "BatchWriteItem"},
			},
		},
		"SuccessfulRequests": {
			Label: (labelPrefix + " SuccessfulRequests"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "SuccessfulRequests.PutItem", Label: "PutItem"},
				{Name: "SuccessfulRequests.DeleteItem", Label: "DeleteItem"},
				{Name: "SuccessfulRequests.UpdateItem", Label: "UpdateItem"},
				{Name: "SuccessfulRequests.GetItem", Label: "GetItem"},
				{Name: "SuccessfulRequests.BatchGetItem", Label: "BatchGetItem"},
				{Name: "SuccessfulRequests.Scan", Label: "Scan"},
				{Name: "SuccessfulRequests.Query", Label: "Query"},
				{Name: "SuccessfulRequests.BatchWriteItem", Label: "BatchWriteItem"},
			},
		},
		"SuccessfulRequestLatency.#": {
			Label: (labelPrefix + " SuccessfulRequestLatency"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "Minimum", Label: "Min"},
				{Name: "Maximum", Label: "Max"},
				{Name: "Average", Label: "Average"},
			},
		},
	}
	return graphdef
}

// Do the plugin
func Do() {
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optRegion := flag.String("region", "", "AWS Region")
	optTableName := flag.String("table-name", "", "DynamoDB Table Name")
	optPrefix := flag.String("metric-key-prefix", "dynamodb", "Metric key prefix")
	flag.Parse()

	var plugin DynamoDBPlugin

	plugin.AccessKeyID = *optAccessKeyID
	plugin.SecretAccessKey = *optSecretAccessKey
	plugin.Region = *optRegion
	plugin.TableName = *optTableName
	plugin.Prefix = *optPrefix

	err := plugin.prepare()
	if err != nil {
		log.Fatalln(err)
	}

	helper := mp.NewMackerelPlugin(plugin)

	helper.Run()
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, fromkeith
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice, this
 *   list of conditions and the following disclaimer in the documentation and/or
 *   other materials provided with the distribution.
 *
 * * Neither the name of the fromkeith nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package s3

import (
    "github.com/fromkeith/awsgo"
    "time"
    "fmt"
    "net/url"
    "crypto/hmac"
    "crypto/sha1"
    "encoding/base64"
)

// Generates an expirable link to an s3 object
// Generated link will be: "//s3.amazonaws.com/{bucket}/{path}?{queryargs}"
func GenerateExpirableLink(bucket, path string, expires time.Time, creds awsgo.Credentials) string {
    var header string

    if creds.GetToken() != "" {
        header = fmt.Sprintf("x-amz-security-token:%s\n", creds.GetToken())
    }

    toSign := fmt.Sprintf(
        "GET\n\n\n%d\n%s/%s/%s",
        expires.Unix(),
        header,
        bucket,
        path,
    )
    hmacHasher := hmac.New(sha1.New, []byte(creds.SecretAccessKey))
    hmacHasher.Write([]byte(toSign))
    signature := base64.StdEncoding.EncodeToString(hmacHasher.Sum(nil))

    v := make(url.Values)
    v.Add("AWSAccessKeyId", creds.AccessKeyId)
    v.Add("Expires", fmt.Sprintf("%d", expires.Unix()))
    v.Add("Signature", signature)
    if creds.GetToken() != "" {
        v.Add("x-amz-security-token", creds.GetToken())
    }

    return fmt.Sprintf(
        "//s3.amazonaws.com/%s/%s?%s",
        bucket,
        path,
        v.Encode(),
    )
}

<commit_msg>update expire link to allow a version id to be specified.<commit_after>/*
 * Copyright (c) 2013, fromkeith
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 *   list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice, this
 *   list of conditions and the following disclaimer in the documentation and/or
 *   other materials provided with the distribution.
 *
 * * Neither the name of the fromkeith nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package s3

import (
    "github.com/fromkeith/awsgo"
    "time"
    "fmt"
    "net/url"
    "crypto/hmac"
    "crypto/sha1"
    "encoding/base64"
)

// Generates an expirable link to an s3 object
// Generated link will be: "//s3.amazonaws.com/{bucket}/{path}?{queryargs}"
func GenerateExpirableLinkVersion(bucket, path string, expires time.Time, creds awsgo.Credentials, version string) string {
    var header string

    if creds.GetToken() != "" {
        header = fmt.Sprintf("x-amz-security-token:%s\n", creds.GetToken())
    }
    if version != "" {
        header = fmt.Sprintf("%sx-amz-version-id:%s\n", header, version)
    }

    toSign := fmt.Sprintf(
        "GET\n\n\n%d\n%s/%s/%s",
        expires.Unix(),
        header,
        bucket,
        path,
    )
    hmacHasher := hmac.New(sha1.New, []byte(creds.SecretAccessKey))
    hmacHasher.Write([]byte(toSign))
    signature := base64.StdEncoding.EncodeToString(hmacHasher.Sum(nil))

    v := make(url.Values)
    v.Add("AWSAccessKeyId", creds.AccessKeyId)
    v.Add("Expires", fmt.Sprintf("%d", expires.Unix()))
    v.Add("Signature", signature)
    if creds.GetToken() != "" {
        v.Add("x-amz-security-token", creds.GetToken())
    }
    if version != "" {
        v.Add("x-amz-version-id", version)
    }

    return fmt.Sprintf(
        "//s3.amazonaws.com/%s/%s?%s",
        bucket,
        path,
        v.Encode(),
    )
}

// Generates an expirable link to an s3 object
// Generated link will be: "//s3.amazonaws.com/{bucket}/{path}?{queryargs}"
func GenerateExpirableLink(bucket, path string, expires time.Time, creds awsgo.Credentials) string {
    return GenerateExpirableLinkVersion(bucket, path, expires, creds, "")
}

<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repos contains information about Go source repositories.
package repos

import "fmt"

type Repo struct {
	// GoGerritProject, if non-empty, is the repo's Gerrit project
	// name, such as "go", "net", or "sys".
	GoGerritProject string

	// ImportPath is the repo's import path.
	// It is empty for the main Go repo.
	ImportPath string

	// MirrorToGitHub controls whether this repo is mirrored
	// from Gerrit to GitHub. If true, GoGerritProject and
	// gitHubRepo must both be defined.
	MirrorToGitHub bool

	// showOnDashboard is whether to show the repo on the bottom
	// of build.golang.org in the repo overview section.
	showOnDashboard bool

	// CoordinatorCanBuild reports whether this a repo that the
	// build coordinator knows how to build.
	CoordinatorCanBuild bool

	// gitHubRepo is the "org/repo" of where this repo exists on
	// GitHub. If MirrorToGitHub is true, this is the
	// destination.
	gitHubRepo string

	// WebsiteDesc is the description of the repo for showing on
	// https://golang.org/pkg/#subrepo.
	// It should be plain text. Hostnames may be auto-linkified.
	WebsiteDesc string
}

// ByGerritProject maps from a Gerrit project name ("go", "net", etc)
// to the Repo's information.
var ByGerritProject = map[string]*Repo{ /* initialized below */ }

// ByImportPath maps from an import path ("golang.org/x/net") to the
// Repo's information.
var ByImportPath = map[string]*Repo{ /* initialized below */ }

func init() {
	addMirrored("go", coordinatorCanBuild, noDash)
	addMirrored("dl", importPath("golang.org/dl"), coordinatorCanBuild)
	addMirrored("gddo", importPath("github.com/golang/gddo"))
	addMirrored("gofrontend")
	addMirrored("proposal")
	addMirrored("sublime-build")
	addMirrored("sublime-config")

	x("arch")
	x("benchmarks", desc("benchmarks to measure Go as it is developed"))
	x("blog", desc("blog.golang.org's implementation"))
	x("build", desc("build.golang.org's implementation"))
	x("crypto", desc("additional cryptography packages"))
	x("debug", desc("an experimental debugger for Go"))
	x("example", noDash)
	x("exp", desc("experimental and deprecated packages (handle with care; may change without warning)"))
	x("image", desc("additional imaging packages"))
	x("lint", noDash)
	x("mobile", desc("experimental support for Go on mobile platforms"))
	x("mod")
	x("net", desc("additional networking packages"))
	x("oauth2")
	x("perf", desc("packages and tools for performance measurement, storage, and analysis"))
	x("playground", noDash)
	x("review", desc("a tool for working with Gerrit code reviews"))
	x("scratch", noDash)
	x("sync", desc("additional concurrency primitives"))
	x("sys", desc("packages for making system calls"))
	x("talks")
	x("term")
	x("text", desc("packages for working with text"))
	x("time", desc("additional time packages"))
	x("tools", desc("godoc, goimports, gorename, and other tools"))
	x("tour", noDash, desc("tour.golang.org's implementation"))
	x("vgo", noDash)
	x("website")
	x("xerrors", noDash)

	add(&Repo{GoGerritProject: "gollvm"})
	add(&Repo{GoGerritProject: "grpc-review"})

	add(&Repo{
		GoGerritProject: "protobuf",
		MirrorToGitHub:  true,
		ImportPath:      "google.golang.org/protobuf",
		gitHubRepo:      "protocolbuffers/protobuf-go",
	})
}

type modifyRepo func(*Repo)

// noDash is an option to the x func that marks the repo as hidden on
// the https://build.golang.org/ dashboard.
func noDash(r *Repo) { r.showOnDashboard = false }

func coordinatorCanBuild(r *Repo) { r.CoordinatorCanBuild = true }

func importPath(v string) modifyRepo { return func(r *Repo) { r.ImportPath = v } }

func desc(v string) modifyRepo { return func(r *Repo) { r.WebsiteDesc = v } }

// addMirrored adds a repo that's on Gerrit and mirrored to GitHub.
func addMirrored(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject: proj,
		MirrorToGitHub:  true,
		gitHubRepo:      "golang/" + proj,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

// x adds a golang.org/x repo.
func x(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject:     proj,
		MirrorToGitHub:      true,
		CoordinatorCanBuild: true,
		ImportPath:          "golang.org/x/" + proj,
		gitHubRepo:          "golang/" + proj,
		showOnDashboard:     true,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

func add(r *Repo) {
	if r.MirrorToGitHub {
		if r.gitHubRepo == "" {
			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
		}
		if r.GoGerritProject == "" {
			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no GoGerritProject", r))
		}
	}
	if r.showOnDashboard {
		if !r.CoordinatorCanBuild {
			panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
		}
		if r.GoGerritProject == "" {
			panic(fmt.Sprintf("project %+v is showOnDashboard but has no Gerrit project", r))
		}
	}

	if p := r.GoGerritProject; p != "" {
		if _, dup := ByGerritProject[p]; dup {
			panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
		}
		ByGerritProject[p] = r
	}
	if p := r.ImportPath; p != "" {
		if _, dup := ByImportPath[p]; dup {
			panic(fmt.Sprintf("duplicate import path %q in %+v", p, r))
		}
		ByImportPath[p] = r
	}
}

// ShowOnDashboard reports whether this repo should show up on build.golang.org
// in the list of repos at bottom.
//
// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }

// GitHubRepo returns the "<org>/<repo>" that this repo either lives
// at or is mirrored to. It returns the empty string if this repo has no
// GitHub presence.
func (r *Repo) GitHubRepo() string { return r.gitHubRepo }
<commit_msg>repos: add vscode-go repository<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repos contains information about Go source repositories.
package repos

import "fmt"

type Repo struct {
	// GoGerritProject, if non-empty, is the repo's Gerrit project
	// name, such as "go", "net", or "sys".
	GoGerritProject string

	// ImportPath is the repo's import path.
	// It is empty for the main Go repo and other repos that do not
	// contain Go code.
	ImportPath string

	// MirrorToGitHub controls whether this repo is mirrored
	// from Gerrit to GitHub. If true, GoGerritProject and
	// gitHubRepo must both be defined.
	MirrorToGitHub bool

	// showOnDashboard is whether to show the repo on the bottom
	// of build.golang.org in the repo overview section.
	showOnDashboard bool

	// CoordinatorCanBuild reports whether this a repo that the
	// build coordinator knows how to build.
	CoordinatorCanBuild bool

	// gitHubRepo is the "org/repo" of where this repo exists on
	// GitHub. If MirrorToGitHub is true, this is the
	// destination.
	gitHubRepo string

	// WebsiteDesc is the description of the repo for showing on
	// https://golang.org/pkg/#subrepo.
	// It should be plain text. Hostnames may be auto-linkified.
	WebsiteDesc string
}

// ByGerritProject maps from a Gerrit project name ("go", "net", etc)
// to the Repo's information.
var ByGerritProject = map[string]*Repo{ /* initialized below */ }

// ByImportPath maps from an import path ("golang.org/x/net") to the
// Repo's information.
var ByImportPath = map[string]*Repo{ /* initialized below */ }

func init() {
	addMirrored("go", coordinatorCanBuild, noDash)
	addMirrored("dl", importPath("golang.org/dl"), coordinatorCanBuild)
	addMirrored("gddo", importPath("github.com/golang/gddo"))
	addMirrored("gofrontend")
	addMirrored("proposal")
	addMirrored("sublime-build")
	addMirrored("sublime-config")

	x("arch")
	x("benchmarks", desc("benchmarks to measure Go as it is developed"))
	x("blog", desc("blog.golang.org's implementation"))
	x("build", desc("build.golang.org's implementation"))
	x("crypto", desc("additional cryptography packages"))
	x("debug", desc("an experimental debugger for Go"))
	x("example", noDash)
	x("exp", desc("experimental and deprecated packages (handle with care; may change without warning)"))
	x("image", desc("additional imaging packages"))
	x("lint", noDash)
	x("mobile", desc("experimental support for Go on mobile platforms"))
	x("mod")
	x("net", desc("additional networking packages"))
	x("oauth2")
	x("perf", desc("packages and tools for performance measurement, storage, and analysis"))
	x("playground", noDash)
	x("review", desc("a tool for working with Gerrit code reviews"))
	x("scratch", noDash)
	x("sync", desc("additional concurrency primitives"))
	x("sys", desc("packages for making system calls"))
	x("talks")
	x("term")
	x("text", desc("packages for working with text"))
	x("time", desc("additional time packages"))
	x("tools", desc("godoc, goimports, gorename, and other tools"))
	x("tour", noDash, desc("tour.golang.org's implementation"))
	x("vgo", noDash)
	x("website")
	x("xerrors", noDash)

	add(&Repo{GoGerritProject: "gollvm"})
	add(&Repo{GoGerritProject: "grpc-review"})

	add(&Repo{
		GoGerritProject: "protobuf",
		MirrorToGitHub:  true,
		ImportPath:      "google.golang.org/protobuf",
		gitHubRepo:      "protocolbuffers/protobuf-go",
	})

	add(&Repo{
		GoGerritProject: "vscode-go",
		MirrorToGitHub:  true,
		gitHubRepo:      "golang/vscode-go",
	})
}

type modifyRepo func(*Repo)

// noDash is an option to the x func that marks the repo as hidden on
// the https://build.golang.org/ dashboard.
func noDash(r *Repo) { r.showOnDashboard = false }

func coordinatorCanBuild(r *Repo) { r.CoordinatorCanBuild = true }

func importPath(v string) modifyRepo { return func(r *Repo) { r.ImportPath = v } }

func desc(v string) modifyRepo { return func(r *Repo) { r.WebsiteDesc = v } }

// addMirrored adds a repo that's on Gerrit and mirrored to GitHub.
func addMirrored(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject: proj,
		MirrorToGitHub:  true,
		gitHubRepo:      "golang/" + proj,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

// x adds a golang.org/x repo.
func x(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject:     proj,
		MirrorToGitHub:      true,
		CoordinatorCanBuild: true,
		ImportPath:          "golang.org/x/" + proj,
		gitHubRepo:          "golang/" + proj,
		showOnDashboard:     true,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

func add(r *Repo) {
	if r.MirrorToGitHub {
		if r.gitHubRepo == "" {
			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
		}
		if r.GoGerritProject == "" {
			panic(fmt.Sprintf("project %+v has MirrorToGitHub but no GoGerritProject", r))
		}
	}
	if r.showOnDashboard {
		if !r.CoordinatorCanBuild {
			panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
		}
		if r.GoGerritProject == "" {
			panic(fmt.Sprintf("project %+v is showOnDashboard but has no Gerrit project", r))
		}
	}

	if p := r.GoGerritProject; p != "" {
		if _, dup := ByGerritProject[p]; dup {
			panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
		}
		ByGerritProject[p] = r
	}
	if p := r.ImportPath; p != "" {
		if _, dup := ByImportPath[p]; dup {
			panic(fmt.Sprintf("duplicate import path %q in %+v", p, r))
		}
		ByImportPath[p] = r
	}
}

// ShowOnDashboard reports whether this repo should show up on build.golang.org
// in the list of repos at bottom.
//
// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }

// GitHubRepo returns the "<org>/<repo>" that this repo either lives
// at or is mirrored to. It returns the empty string if this repo has no
// GitHub presence.
func (r *Repo) GitHubRepo() string { return r.gitHubRepo }
<|endoftext|>