<commit_before>// Copyright 2016 OpenConfigd Project.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"fmt"
	"io"
	"net"
	"strconv"
	"strings"

	"github.com/coreswitch/cmd"
	"github.com/coreswitch/component"
	rpc "github.com/coreswitch/openconfigd/proto"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	CliSuccess = iota
	CliSuccessExec
	CliSuccessShow
	CliSuccessModule
	CliSuccessRedirect
	CliSuccessRedirectShow
)

var (
	RIBD_SYNCHRONIZED bool
)

var GrpcModuleMap = map[string]string{}

type openconfigServer struct {
	cmd *cmd.Cmd
}

type registerServer struct {
	cmd *cmd.Cmd
}

type configServer struct {
	cmd *cmd.Cmd
}

func execute(Cmd *cmd.Cmd, mode string, args []string, line string, reply *rpc.ExecReply) (ret string) {
	result, fn, pargs, _ := Cmd.ParseLine(mode, line, &cmd.Param{Command: args})

	fmt.Println(args)
	if RIBD_SYNCHRONIZED {
		if len(args) >= 4 {
			if args[0] == "set" && args[1] == "interfaces" && args[2] == "interface" {
				conf := configActive.LookupByPath([]string{"interfaces", "interface", args[3]})
				if conf == nil {
					ret = "NoMatch\n"
					return
				}
			}
		}
	}
	switch result {
	case cmd.ParseSuccess:
		cb, ok := fn.(func([]string) (int, string))
		if ok {
			inst, instStr := cb(cmd.Interface2String(pargs))
			switch inst {
			case CliSuccess:
				ret = "Success"
			case CliSuccessExec:
				ret = "SuccessExec\n"
				ret += instStr
			case CliSuccessShow:
				reply.Code = rpc.ExecCode_SHOW
				//ret = "SuccessShow\n"
				ret = instStr
			case CliSuccessModule:
				ret = "SuccessModule\n"
				ret += instStr
			case CliSuccessRedirect, CliSuccessRedirectShow:
				if inst == CliSuccessRedirectShow {
					reply.Code = rpc.ExecCode_REDIRECT_SHOW
				} else {
					reply.Code = rpc.ExecCode_REDIRECT
				}
				port, _ := strconv.ParseUint(instStr, 10, 32)
				reply.Port = uint32(port)
				ret = "SuccessRedirect\n"
				ret += instStr
			}
		}
	case cmd.ParseIncomplete:
		ret = "Incomplete\n"
	case cmd.ParseNoMatch:
		ret = "NoMatch\n"
	case cmd.ParseAmbiguous:
		ret = "Ambiguous\n"
	}
	return
}

func complete(Cmd *cmd.Cmd, mode string, args []string, line string, trailing bool) (ret string) {
	result, _, _, comps := Cmd.ParseLine(mode, line, &cmd.Param{Command: args, Complete: true, TrailingSpace: trailing})

	switch result {
	case cmd.ParseSuccess, cmd.ParseIncomplete:
		ret = "Success\n"
	case cmd.ParseNoMatch:
		ret = "NoMatch\n"
	case cmd.ParseAmbiguous:
		ret = "Ambiguous\n"
	}

	for _, comp := range comps {
		var pre string
		if !comp.Dir && !comp.Additive {
			pre = "--"
		} else {
			if comp.Additive {
				pre = "+"
			} else {
				pre = " "
			}
			if comp.Dir {
				pre += ">"
			} else {
				pre += " "
			}
		}
		ret += fmt.Sprintf("%s\t%s\t%s\n", comp.Name, pre, comp.Help)
		//ret += comp.Name + "\t" + "" + "\t" + comp.Help + "\n"
	}

	if result == cmd.ParseSuccess {
		ret += "<cr>\t--\t\n"
	}

	return
}

func unquote(req *rpc.ExecRequest) {
	if req.Type == rpc.ExecType_COMPLETE || req.Type == rpc.ExecType_COMPLETE_TRAILING_SPACE {
		for pos, arg := range req.Args {
			arg, err := strconv.Unquote(arg)
			if err == nil {
				req.Args[pos] = arg
			}
		}
	}
}

func ExecLine(line string) string {
	reply := new(rpc.ExecReply)
	args := strings.Split(line, " ")
	return execute(TopCmd, "configure", args, line, reply)
}

func (s *openconfigServer) DoExec(_ context.Context, req *rpc.ExecRequest) (*rpc.ExecReply, error) {
	reply := new(rpc.ExecReply)

	unquote(req)

	switch req.Type {
	case rpc.ExecType_EXEC:
		reply.Lines = execute(s.cmd, req.Mode, req.Args, req.Line, reply)
	case rpc.ExecType_COMPLETE:
		reply.Lines = complete(s.cmd, req.Mode, req.Args, req.Line, false)
	case rpc.ExecType_COMPLETE_TRAILING_SPACE:
		reply.Lines = complete(s.cmd, req.Mode, req.Args, req.Line+" ", true)
	case rpc.ExecType_COMPLETE_FIRST_COMMANDS:
		reply.Lines = s.cmd.FirstCommands(req.Mode, req.Privilege)
	case rpc.ExecType_COMPLETE_DYNAMIC:
		// Ignore dynamic completion in openconfigd.
	}
	return reply, nil
}

func (s *registerServer) DoRegister(_ context.Context, req *rpc.RegisterRequest) (*rpc.RegisterReply, error) {
	reply := new(rpc.RegisterReply)
	port := GrpcModuleMap[req.Module]
	if port == "" {
		port = "2601"
	} else {
		//fmt.Println("Set port", port)
	}

	inst := CliSuccessRedirect
	if req.Code == rpc.ExecCode_REDIRECT_SHOW {
		inst = CliSuccessRedirectShow
	}

	if mode := s.cmd.LookupMode(req.Mode); mode != nil {
		mode.InstallLine(req.Line,
			func(Args []string) (int, string) {
				return inst, port
			},
			&cmd.Param{Helps: req.Helps, Privilege: req.Privilege, Dynamic: true})
	}
	return reply, nil
}

func (s *registerServer) DoRegisterModule(_ context.Context, req *rpc.RegisterModuleRequest) (*rpc.RegisterModuleReply, error) {
	reply := new(rpc.RegisterModuleReply)
	GrpcModuleMap[req.Module] = req.Port
	return reply, nil
}

func (s *configServer) DoConfig(stream rpc.Config_DoConfigServer) error {
loop:
	for {
		msg, err := stream.Recv()
		if err == io.EOF {
			fmt.Println("EOF")
			break loop
		}
		if err != nil {
			fmt.Println(err)
			break loop
		}
		switch msg.Type {
		case rpc.ConfigType_SUBSCRIBE:
			go SubscribeRemoteAdd(stream, msg)
		case rpc.ConfigType_SUBSCRIBE_MULTI:
			go SubscribeRemoteAddMulti(stream, msg)
		case rpc.ConfigType_SUBSCRIBE_REQUEST:
			go SubscribeAdd(stream, msg)
		case rpc.ConfigType_SET:
			go YangConfigPush(msg.Path)
		case rpc.ConfigType_DELETE:
			go YangConfigPull(msg.Path)
		case rpc.ConfigType_VALIDATE_SUCCESS:
			//fmt.Println("Validate Success")
			SubscribeValidateProcess(stream, msg.Type)
		case rpc.ConfigType_VALIDATE_FAILED:
			//fmt.Println("Validate Failed")
			SubscribeValidateProcess(stream, msg.Type)
		case rpc.ConfigType_API_CALL_FINISHED:
			log.Info("API_SYNC: gRPC message received")
			if ApiSyncCh != nil {
				close(ApiSyncCh)
			}
		}
	}
	if stream != nil {
		SubscribeDelete(stream)
	}

	return nil
}

func DynamicCompletionLocal(commands []string, module string, args []string) []string {
	if len(args) > 0 {
		switch args[0] {
		case "rollback":
			return RollbackCompletion(commands)
		default:
		}
	}
	return []string{}
}

func DynamicCompletion(commands []string, module string, args []string) []string {
	// Local completion check.
	// if module == "local" {
	// 	return DynamicCompletionLocal(commands, module, args)
	// }

	// XXX Need to leverage stream connection. (No need of make a new connection)
	host := ":2601" // Default port.
	port := SubscribePortLookup(module)
	if port != 0 {
		host = fmt.Sprintf(":%d", port)
	}
	conn, err := grpc.Dial(host, grpc.WithInsecure())
	if err != nil {
		fmt.Println("DynamicCompletion: Fail to dial", err)
		return []string{}
	}
	defer conn.Close()

	client := rpc.NewExecClient(conn)
	req := &rpc.ExecRequest{
		Type:     rpc.ExecType_COMPLETE_DYNAMIC,
		Mode:     module,
		Commands: commands,
		Args:     args,
	}

	reply, err := client.DoExec(context.Background(), req)
	if err != nil {
		fmt.Println("client DoExec COMPLETE_DYNAMIC failed:", err)
		return []string{}
	}
	return reply.Candidates
}

// RPC component.
type RpcComponent struct {
	GrpcEndpoint string
}

// RPC component start method.
func (this *RpcComponent) Start() component.Component {
	lis, err := net.Listen("tcp", this.GrpcEndpoint)
	if err != nil {
		grpclog.Fatalf("failed %v", err)
	}
	grpcServer := grpc.NewServer()
	rpc.RegisterExecServer(grpcServer, &openconfigServer{TopCmd})
	rpc.RegisterRegisterServer(grpcServer, &registerServer{TopCmd})
	rpc.RegisterConfigServer(grpcServer, &configServer{TopCmd})
	grpcServer.Serve(lis)
	return this
}

func (this *RpcComponent) Stop() component.Component {
	// fmt.Println("rpc component stop")
	return this
}
<commit_msg>Sort completion result with alphabetical order.<commit_after>// Copyright 2016 OpenConfigd Project.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"fmt"
	"io"
	"net"
	"sort"
	"strconv"
	"strings"

	"github.com/coreswitch/cmd"
	"github.com/coreswitch/component"
	rpc "github.com/coreswitch/openconfigd/proto"
	log "github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

const (
	CliSuccess = iota
	CliSuccessExec
	CliSuccessShow
	CliSuccessModule
	CliSuccessRedirect
	CliSuccessRedirectShow
)

var (
	RIBD_SYNCHRONIZED bool
)

var GrpcModuleMap = map[string]string{}

type openconfigServer struct {
	cmd *cmd.Cmd
}

type registerServer struct {
	cmd *cmd.Cmd
}

type configServer struct {
	cmd *cmd.Cmd
}

func execute(Cmd *cmd.Cmd, mode string, args []string, line string, reply *rpc.ExecReply) (ret string) {
	result, fn, pargs, _ := Cmd.ParseLine(mode, line, &cmd.Param{Command: args})

	fmt.Println(args)
	if RIBD_SYNCHRONIZED {
		if len(args) >= 4 {
			if args[0] == "set" && args[1] == "interfaces" && args[2] == "interface" {
				conf := configActive.LookupByPath([]string{"interfaces", "interface", args[3]})
				if conf == nil {
					ret = "NoMatch\n"
					return
				}
			}
		}
	}
	switch result {
	case cmd.ParseSuccess:
		cb, ok := fn.(func([]string) (int, string))
		if ok {
			inst, instStr := cb(cmd.Interface2String(pargs))
			switch inst {
			case CliSuccess:
				ret = "Success"
			case CliSuccessExec:
				ret = "SuccessExec\n"
				ret += instStr
			case CliSuccessShow:
				reply.Code = rpc.ExecCode_SHOW
				//ret = "SuccessShow\n"
				ret = instStr
			case CliSuccessModule:
				ret = "SuccessModule\n"
				ret += instStr
			case CliSuccessRedirect, CliSuccessRedirectShow:
				if inst == CliSuccessRedirectShow {
					reply.Code = rpc.ExecCode_REDIRECT_SHOW
				} else {
					reply.Code = rpc.ExecCode_REDIRECT
				}
				port, _ := strconv.ParseUint(instStr, 10, 32)
				reply.Port = uint32(port)
				ret = "SuccessRedirect\n"
				ret += instStr
			}
		}
	case cmd.ParseIncomplete:
		ret = "Incomplete\n"
	case cmd.ParseNoMatch:
		ret = "NoMatch\n"
	case cmd.ParseAmbiguous:
		ret = "Ambiguous\n"
	}
	return
}

func complete(Cmd *cmd.Cmd, mode string, args []string, line string, trailing bool) (ret string) {
	result, _, _, comps := Cmd.ParseLine(mode, line, &cmd.Param{Command: args, Complete: true, TrailingSpace: trailing})

	switch result {
	case cmd.ParseSuccess, cmd.ParseIncomplete:
		ret = "Success\n"
	case cmd.ParseNoMatch:
		ret = "NoMatch\n"
	case cmd.ParseAmbiguous:
		ret = "Ambiguous\n"
	}

	sort.Sort(comps)

	for _, comp := range comps {
		var pre string
		if !comp.Dir && !comp.Additive {
			pre = "--"
		} else {
			if comp.Additive {
				pre = "+"
			} else {
				pre = " "
			}
			if comp.Dir {
				pre += ">"
			} else {
				pre += " "
			}
		}
		ret += fmt.Sprintf("%s\t%s\t%s\n", comp.Name, pre, comp.Help)
		//ret += comp.Name + "\t" + "" + "\t" + comp.Help + "\n"
	}

	if result == cmd.ParseSuccess {
		ret += "<cr>\t--\t\n"
	}

	return
}

func unquote(req *rpc.ExecRequest) {
	if req.Type == rpc.ExecType_COMPLETE || req.Type == rpc.ExecType_COMPLETE_TRAILING_SPACE {
		for pos, arg := range req.Args {
			arg, err := strconv.Unquote(arg)
			if err == nil {
				req.Args[pos] = arg
			}
		}
	}
}

func ExecLine(line string) string {
	reply := new(rpc.ExecReply)
	args := strings.Split(line, " ")
	return execute(TopCmd, "configure", args, line, reply)
}

func (s *openconfigServer) DoExec(_ context.Context, req *rpc.ExecRequest) (*rpc.ExecReply, error) {
	reply := new(rpc.ExecReply)

	unquote(req)

	switch req.Type {
	case rpc.ExecType_EXEC:
		reply.Lines = execute(s.cmd, req.Mode, req.Args, req.Line, reply)
	case rpc.ExecType_COMPLETE:
		reply.Lines = complete(s.cmd, req.Mode, req.Args, req.Line, false)
	case rpc.ExecType_COMPLETE_TRAILING_SPACE:
		reply.Lines = complete(s.cmd, req.Mode, req.Args, req.Line+" ", true)
	case rpc.ExecType_COMPLETE_FIRST_COMMANDS:
		reply.Lines = s.cmd.FirstCommands(req.Mode, req.Privilege)
	case rpc.ExecType_COMPLETE_DYNAMIC:
		// Ignore dynamic completion in openconfigd.
	}
	return reply, nil
}

func (s *registerServer) DoRegister(_ context.Context, req *rpc.RegisterRequest) (*rpc.RegisterReply, error) {
	reply := new(rpc.RegisterReply)
	port := GrpcModuleMap[req.Module]
	if port == "" {
		port = "2601"
	} else {
		//fmt.Println("Set port", port)
	}

	inst := CliSuccessRedirect
	if req.Code == rpc.ExecCode_REDIRECT_SHOW {
		inst = CliSuccessRedirectShow
	}

	if mode := s.cmd.LookupMode(req.Mode); mode != nil {
		mode.InstallLine(req.Line,
			func(Args []string) (int, string) {
				return inst, port
			},
			&cmd.Param{Helps: req.Helps, Privilege: req.Privilege, Dynamic: true})
	}
	return reply, nil
}

func (s *registerServer) DoRegisterModule(_ context.Context, req *rpc.RegisterModuleRequest) (*rpc.RegisterModuleReply, error) {
	reply := new(rpc.RegisterModuleReply)
	GrpcModuleMap[req.Module] = req.Port
	return reply, nil
}

func (s *configServer) DoConfig(stream rpc.Config_DoConfigServer) error {
loop:
	for {
		msg, err := stream.Recv()
		if err == io.EOF {
			fmt.Println("EOF")
			break loop
		}
		if err != nil {
			fmt.Println(err)
			break loop
		}
		switch msg.Type {
		case rpc.ConfigType_SUBSCRIBE:
			go SubscribeRemoteAdd(stream, msg)
		case rpc.ConfigType_SUBSCRIBE_MULTI:
			go SubscribeRemoteAddMulti(stream, msg)
		case rpc.ConfigType_SUBSCRIBE_REQUEST:
			go SubscribeAdd(stream, msg)
		case rpc.ConfigType_SET:
			go YangConfigPush(msg.Path)
		case rpc.ConfigType_DELETE:
			go YangConfigPull(msg.Path)
		case rpc.ConfigType_VALIDATE_SUCCESS:
			//fmt.Println("Validate Success")
			SubscribeValidateProcess(stream, msg.Type)
		case rpc.ConfigType_VALIDATE_FAILED:
			//fmt.Println("Validate Failed")
			SubscribeValidateProcess(stream, msg.Type)
		case rpc.ConfigType_API_CALL_FINISHED:
			log.Info("API_SYNC: gRPC message received")
			if ApiSyncCh != nil {
				close(ApiSyncCh)
			}
		}
	}
	if stream != nil {
		SubscribeDelete(stream)
	}

	return nil
}

func DynamicCompletionLocal(commands []string, module string, args []string) []string {
	if len(args) > 0 {
		switch args[0] {
		case "rollback":
			return RollbackCompletion(commands)
		default:
		}
	}
	return []string{}
}

func DynamicCompletion(commands []string, module string, args []string) []string {
	// Local completion check.
	// if module == "local" {
	// 	return DynamicCompletionLocal(commands, module, args)
	// }

	// XXX Need to leverage stream connection. (No need of make a new connection)
	host := ":2601" // Default port.
	port := SubscribePortLookup(module)
	if port != 0 {
		host = fmt.Sprintf(":%d", port)
	}
	conn, err := grpc.Dial(host, grpc.WithInsecure())
	if err != nil {
		fmt.Println("DynamicCompletion: Fail to dial", err)
		return []string{}
	}
	defer conn.Close()

	client := rpc.NewExecClient(conn)
	req := &rpc.ExecRequest{
		Type:     rpc.ExecType_COMPLETE_DYNAMIC,
		Mode:     module,
		Commands: commands,
		Args:     args,
	}

	reply, err := client.DoExec(context.Background(), req)
	if err != nil {
		fmt.Println("client DoExec COMPLETE_DYNAMIC failed:", err)
		return []string{}
	}
	return reply.Candidates
}

// RPC component.
type RpcComponent struct {
	GrpcEndpoint string
}

// RPC component start method.
func (this *RpcComponent) Start() component.Component {
	lis, err := net.Listen("tcp", this.GrpcEndpoint)
	if err != nil {
		grpclog.Fatalf("failed %v", err)
	}
	grpcServer := grpc.NewServer()
	rpc.RegisterExecServer(grpcServer, &openconfigServer{TopCmd})
	rpc.RegisterRegisterServer(grpcServer, &registerServer{TopCmd})
	rpc.RegisterConfigServer(grpcServer, &configServer{TopCmd})
	grpcServer.Serve(lis)
	return this
}

func (this *RpcComponent) Stop() component.Component {
	// fmt.Println("rpc component stop")
	return this
}
<|endoftext|>


<commit_before>package storage

import (
	"github.com/Sirupsen/logrus"
	"github.com/tokubai/kinu/logger"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	awsSession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type S3Storage struct {
	Storage

	client *s3.S3

	region         string
	bucket         string
	bucketBasePath string
}

type S3StorageItem struct {
	StorageItem

	Object *s3.Object
}

func openS3Storage() (Storage, error) {
	s := &S3Storage{}
	err := s.Open()
	if err != nil {
		return nil, logger.ErrorDebug(err)
	}
	return s, nil
}

func (s *S3Storage) Open() error {
	s.region = os.Getenv("KINU_S3_REGION")
	if len(s.region) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_REGION system env is required"}
	}

	s.bucket = os.Getenv("KINU_S3_BUCKET")
	if len(s.bucket) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_BUCKET system env is required"}
	}

	s.bucketBasePath = os.Getenv("KINU_S3_BUCKET_BASE_PATH")
	if len(s.bucketBasePath) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_BUCKET_BASE_PATH system env is required"}
	}

	s.client = s3.New(awsSession.New(), &aws.Config{Region: aws.String(s.region)})

	logger.WithFields(logrus.Fields{
		"bucket":    s.bucket,
		"base_path": s.bucketBasePath,
		"region":    s.region,
	}).Debug("open s3 storage")

	return nil
}

func (s *S3Storage) BuildKey(key string) string {
	return s.bucketBasePath + "/" + key
}

func (s *S3Storage) Fetch(key string) (*Object, error) {
	key = s.BuildKey(key)

	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    key,
	}).Debug("start get object from s3")

	resp, err := s.client.GetObject(params)

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return nil, ErrImageNotFound
	} else if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    key,
	}).Debug("found object from s3")

	defer resp.Body.Close()

	object := &Object{
		Metadata: make(map[string]string, 0),
	}
	for k, v := range resp.Metadata {
		object.Metadata[k] = *v
	}
	object.Body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	return object, nil
}

func (s *S3Storage) PutFromBlob(key string, image []byte, metadata map[string]string) error {
	tmpfile, err := ioutil.TempFile("", "kinu-upload")
	if err != nil {
		return logger.ErrorDebug(err)
	}
	_, err = tmpfile.Write(image)
	if err != nil {
		return logger.ErrorDebug(err)
	}

	defer tmpfile.Close()

	return s.Put(key, tmpfile, metadata)
}

func (s *S3Storage) Put(key string, imageFile io.ReadSeeker, metadata map[string]string) error {
	putMetadata := make(map[string]*string, 0)
	for k, v := range metadata {
		putMetadata[k] = &v
	}

	_, err := s.client.PutObject(&s3.PutObjectInput{
		Bucket:   aws.String(s.bucket),
		Key:      aws.String(s.BuildKey(key)),
		Body:     imageFile,
		Metadata: putMetadata,
	})

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    s.BuildKey(key),
	}).Debug("put to s3")

	if err != nil {
		return logger.ErrorDebug(err)
	}

	return nil
}

func (s *S3Storage) List(key string) ([]StorageItem, error) {
	resp, err := s.client.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(s.bucket),
		Prefix: aws.String(s.BuildKey(key)),
	})

	if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    s.BuildKey(key),
	}).Debug("start list object from s3")

	items := make([]StorageItem, 0)
	for _, object := range resp.Contents {
		logger.WithFields(logrus.Fields{
			"key": &object.Key,
		}).Debug("found object")
		item := S3StorageItem{Object: object}
		items = append(items, &item)
	}

	return items, nil
}

func (s *S3Storage) Move(from string, to string) error {
	fromKey := s.bucket + "/" + from
	toKey := s.bucketBasePath + "/" + to

	_, err := s.client.CopyObject(&s3.CopyObjectInput{
		Bucket:     aws.String(s.bucket),
		CopySource: aws.String(fromKey),
		Key:        aws.String(toKey),
	})

	logger.WithFields(logrus.Fields{
		"from": fromKey,
		"to":   toKey,
	}).Debug("move s3 object start")

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return ErrImageNotFound
	} else if err != nil {
		return logger.ErrorDebug(err)
	}
	_, err = s.client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(fromKey),
	})

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return ErrImageNotFound
	} else if err != nil {
		return logger.ErrorDebug(err)
	}

	return nil
}

func (s *S3StorageItem) IsValid() bool {
	if len(s.Extension()) == 0 {
		return false
	}

	if len(s.ImageSize()) == 0 {
		return false
	}

	return true
}

func (s *S3StorageItem) Key() string {
	return *s.Object.Key
}

func (s *S3StorageItem) Filename() string {
	path := strings.Split(s.Key(), "/")
	return path[len(path)-1]
}

func (s *S3StorageItem) Extension() string {
	path := strings.Split(*s.Object.Key, ".")
	return path[len(path)-1]
}

// KeyFormat: :image_type/:id/:id.:size.:format
func (s *S3StorageItem) ImageSize() string {
	path := strings.Split(s.Key(), ".")
	return path[len(path)-2]
}
<commit_msg>Fix wrong image size metadata saving<commit_after>package storage

import (
	"github.com/Sirupsen/logrus"
	"github.com/tokubai/kinu/logger"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	awsSession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

type S3Storage struct {
	Storage

	client *s3.S3

	region         string
	bucket         string
	bucketBasePath string
}

type S3StorageItem struct {
	StorageItem

	Object *s3.Object
}

func openS3Storage() (Storage, error) {
	s := &S3Storage{}
	err := s.Open()
	if err != nil {
		return nil, logger.ErrorDebug(err)
	}
	return s, nil
}

func (s *S3Storage) Open() error {
	s.region = os.Getenv("KINU_S3_REGION")
	if len(s.region) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_REGION system env is required"}
	}

	s.bucket = os.Getenv("KINU_S3_BUCKET")
	if len(s.bucket) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_BUCKET system env is required"}
	}

	s.bucketBasePath = os.Getenv("KINU_S3_BUCKET_BASE_PATH")
	if len(s.bucketBasePath) == 0 {
		return &ErrInvalidStorageOption{Message: "KINU_S3_BUCKET_BASE_PATH system env is required"}
	}

	s.client = s3.New(awsSession.New(), &aws.Config{Region: aws.String(s.region)})

	logger.WithFields(logrus.Fields{
		"bucket":    s.bucket,
		"base_path": s.bucketBasePath,
		"region":    s.region,
	}).Debug("open s3 storage")

	return nil
}

func (s *S3Storage) BuildKey(key string) string {
	return s.bucketBasePath + "/" + key
}

func (s *S3Storage) Fetch(key string) (*Object, error) {
	key = s.BuildKey(key)

	params := &s3.GetObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(key),
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    key,
	}).Debug("start get object from s3")

	resp, err := s.client.GetObject(params)

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return nil, ErrImageNotFound
	} else if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    key,
	}).Debug("found object from s3")

	defer resp.Body.Close()

	object := &Object{
		Metadata: make(map[string]string, 0),
	}
	for k, v := range resp.Metadata {
		object.Metadata[k] = *v
	}
	object.Body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	return object, nil
}

func (s *S3Storage) PutFromBlob(key string, image []byte, metadata map[string]string) error {
	tmpfile, err := ioutil.TempFile("", "kinu-upload")
	if err != nil {
		return logger.ErrorDebug(err)
	}
	_, err = tmpfile.Write(image)
	if err != nil {
		return logger.ErrorDebug(err)
	}

	defer tmpfile.Close()

	return s.Put(key, tmpfile, metadata)
}

func (s *S3Storage) Put(key string, imageFile io.ReadSeeker, metadata map[string]string) error {
	putMetadata := make(map[string]*string, 0)
	for k, v := range metadata {
		putMetadata[k] = aws.String(v)
	}

	_, err := s.client.PutObject(&s3.PutObjectInput{
		Bucket:   aws.String(s.bucket),
		Key:      aws.String(s.BuildKey(key)),
		Body:     imageFile,
		Metadata: putMetadata,
	})

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    s.BuildKey(key),
	}).Debug("put to s3")

	if err != nil {
		return logger.ErrorDebug(err)
	}

	return nil
}

func (s *S3Storage) List(key string) ([]StorageItem, error) {
	resp, err := s.client.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(s.bucket),
		Prefix: aws.String(s.BuildKey(key)),
	})

	if err != nil {
		return nil, logger.ErrorDebug(err)
	}

	logger.WithFields(logrus.Fields{
		"bucket": s.bucket,
		"key":    s.BuildKey(key),
	}).Debug("start list object from s3")

	items := make([]StorageItem, 0)
	for _, object := range resp.Contents {
		logger.WithFields(logrus.Fields{
			"key": &object.Key,
		}).Debug("found object")
		item := S3StorageItem{Object: object}
		items = append(items, &item)
	}

	return items, nil
}

func (s *S3Storage) Move(from string, to string) error {
	fromKey := s.bucket + "/" + from
	toKey := s.bucketBasePath + "/" + to

	_, err := s.client.CopyObject(&s3.CopyObjectInput{
		Bucket:     aws.String(s.bucket),
		CopySource: aws.String(fromKey),
		Key:        aws.String(toKey),
	})

	logger.WithFields(logrus.Fields{
		"from": fromKey,
		"to":   toKey,
	}).Debug("move s3 object start")

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return ErrImageNotFound
	} else if err != nil {
		return logger.ErrorDebug(err)
	}
	_, err = s.client.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.bucket),
		Key:    aws.String(fromKey),
	})

	if reqerr, ok := err.(awserr.RequestFailure); ok && reqerr.StatusCode() == http.StatusNotFound {
		return ErrImageNotFound
	} else if err != nil {
		return logger.ErrorDebug(err)
	}

	return nil
}

func (s *S3StorageItem) IsValid() bool {
	if len(s.Extension()) == 0 {
		return false
	}

	if len(s.ImageSize()) == 0 {
		return false
	}

	return true
}

func (s *S3StorageItem) Key() string {
	return *s.Object.Key
}

func (s *S3StorageItem) Filename() string {
	path := strings.Split(s.Key(), "/")
	return path[len(path)-1]
}

func (s *S3StorageItem) Extension() string {
	path := strings.Split(*s.Object.Key, ".")
	return path[len(path)-1]
}

// KeyFormat: :image_type/:id/:id.:size.:format
func (s *S3StorageItem) ImageSize() string {
	path := strings.Split(s.Key(), ".")
	return path[len(path)-2]
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"io"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	s3 "github.com/rlmcpherson/s3gof3r"
)

// S3Storage represents a s3 based file storage.
type S3Storage struct {
	prefix string
	bucket *s3.Bucket
	config *s3.Config
}

// NewS3Storage create a new S3Storage base on
// a s3:/// URL.
func NewS3Storage(u *url.URL) *S3Storage {
	k, err := envKeys()
	if err != nil {
		panic(err)
	}
	storage := s3.New(u.Host, k)
	var prefix string
	parts := strings.SplitN(u.Path[1:], "/", 2)
	if len(parts) > 1 {
		prefix = parts[1]
	}
	v := u.Query()
	return &S3Storage{
		prefix: prefix,
		bucket: storage.Bucket(parts[0]),
		config: &s3.Config{
			Concurrency: mustParseInt(v, "concurrency", 10),
			PartSize:    mustParseInt64(v, "part_size", 20000000),
			NTry:        mustParseInt(v, "retry", 10),
			Md5Check:    false,
			Scheme:      "https",
			Client:      s3.ClientWithTimeout(5 * time.Second),
		},
	}
}

// Create creates a new file based on the given filename.
func (s S3Storage) Create(name string) (io.WriteCloser, error) {
	return s.bucket.PutWriter(urlJoin(s.prefix, name), nil, s.config)
}

// Open opens the given filename.
func (s S3Storage) Open(name string) (io.ReadCloser, error) {
	r, _, err := s.bucket.GetReader(urlJoin(s.prefix, name), s.config)
	return r, err
}

func urlJoin(strs ...string) string {
	ss := make([]string, len(strs))
	for i, s := range strs {
		if i == 0 {
			ss[i] = strings.TrimRight(s, "/")
		} else {
			ss[i] = strings.TrimLeft(s, "/")
		}
	}
	return strings.Join(ss, "/")
}

func envKeys() (s3.Keys, error) {
	keys := s3.Keys{
		AccessKey:     os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey:     os.Getenv("AWS_SECRET_ACCESS_KEY"),
		SecurityToken: os.Getenv("AWS_SECURITY_TOKEN"),
	}
	if keys.AccessKey == "" || keys.SecretKey == "" {
		return keys, fmt.Errorf("keys not set in environment: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY")
	}
	return keys, nil
}

func mustParseInt(q url.Values, key string, value int) int {
	v, err := strconv.ParseInt(q.Get(key), 10, 64)
	if err != nil {
		return value
	}
	return int(v)
}

func mustParseInt64(q url.Values, key string, value int64) int64 {
	v, err := strconv.ParseInt(q.Get(key), 10, 64)
	if err != nil {
		return value
	}
	return v
}
<commit_msg>Use s3 encryption<commit_after>package storage

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	s3 "github.com/rlmcpherson/s3gof3r"
)

// S3Storage represents a s3 based file storage.
type S3Storage struct {
	prefix string
	bucket *s3.Bucket
	config *s3.Config
}

// NewS3Storage create a new S3Storage base on
// a s3:/// URL.
func NewS3Storage(u *url.URL) *S3Storage {
	k, err := envKeys()
	if err != nil {
		panic(err)
	}
	storage := s3.New(u.Host, k)
	var prefix string
	parts := strings.SplitN(u.Path[1:], "/", 2)
	if len(parts) > 1 {
		prefix = parts[1]
	}
	v := u.Query()
	return &S3Storage{
		prefix: prefix,
		bucket: storage.Bucket(parts[0]),
		config: &s3.Config{
			Concurrency: mustParseInt(v, "concurrency", 10),
			PartSize:    mustParseInt64(v, "part_size", 20000000),
			NTry:        mustParseInt(v, "retry", 10),
			Md5Check:    false,
			Scheme:      "https",
			Client:      s3.ClientWithTimeout(5 * time.Second),
		},
	}
}

// Create creates a new file based on the given filename.
func (s S3Storage) Create(name string) (io.WriteCloser, error) {
	return s.bucket.PutWriter(urlJoin(s.prefix, name), http.Header{
		"x-amz-server-side-encryption": []string{"AES256"},
	}, s.config)
}

// Open opens the given filename.
func (s S3Storage) Open(name string) (io.ReadCloser, error) {
	r, _, err := s.bucket.GetReader(urlJoin(s.prefix, name), s.config)
	return r, err
}

func urlJoin(strs ...string) string {
	ss := make([]string, len(strs))
	for i, s := range strs {
		if i == 0 {
			ss[i] = strings.TrimRight(s, "/")
		} else {
			ss[i] = strings.TrimLeft(s, "/")
		}
	}
	return strings.Join(ss, "/")
}

func envKeys() (s3.Keys, error) {
	keys := s3.Keys{
		AccessKey:     os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey:     os.Getenv("AWS_SECRET_ACCESS_KEY"),
		SecurityToken: os.Getenv("AWS_SECURITY_TOKEN"),
	}
	if keys.AccessKey == "" || keys.SecretKey == "" {
		return keys, fmt.Errorf("keys not set in environment: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY")
	}
	return keys, nil
}

func mustParseInt(q url.Values, key string, value int) int {
	v, err := strconv.ParseInt(q.Get(key), 10, 64)
	if err != nil {
		return value
	}
	return int(v)
}

func mustParseInt64(q url.Values, key string, value int64) int64 {
	v, err := strconv.ParseInt(q.Get(key), 10, 64)
	if err != nil {
		return value
	}
	return v
}
<|endoftext|>


<commit_before>/*
Brainfuck Interpreter
*/
package main

import (
    "errors"
    "fmt"
    "io/ioutil"
    "os"
)

func cleanup(data []byte) []byte {
    result := make([]byte, 0, len(data))
    for _, c := range data {
        switch c {
        case '>', '<', '+', '-', '.', ',', '[', ']':
            result = append(result, c)
        }
    }
    return result
}

func buildBracemap(data []byte) (bracemap map[int]int, err error) {
    stack := make([]int, 0, 100)
    bracemap = make(map[int]int)
    for pos, c := range data {
        switch c {
        case '[':
            stack = append(stack, pos)
        case ']':
            if len(stack) == 0 {
                err = errors.New("Syntax error: Unmatched closing brace")
                return bracemap, err
            }
            start := stack[len(stack)-1]
            bracemap[start] = pos
            bracemap[pos] = start
            stack = stack[:len(stack)-1]
        }
    }
    if len(stack) != 0 {
        err = errors.New("Syntax error: Not enough closing braces")
        return bracemap, err
    }
    return bracemap, nil
}

func runBrainfuck(data []byte) (tapeLength, instructionCount int, err error) {
    code := cleanup(data)
    bracemap, err := buildBracemap(code)
    if err != nil {
        return 0, 0, err
    }

    tape := make([]byte, 1, 1000)
    codePtr, tapePtr, instructionCount := 0, 0, 0
    for codePtr < len(code) {
        switch code[codePtr] {
        case '>':
            tapePtr++
            if tapePtr == len(tape) {
                tape = append(tape, 0)
            }
        case '<':
            tapePtr--
            if tapePtr == -1 {
                tape = append(tape, 0)
                copy(tape[1:], tape)
                tape[0] = 0
                tapePtr = 0
            }
        case '+':
            tape[tapePtr]++
        case '-':
            tape[tapePtr]--
        case '.':
            fmt.Print(string(tape[tapePtr]))
        case ',':
            fmt.Scanf("%c")
        case '[':
            if tape[tapePtr] == 0 {
                codePtr = bracemap[codePtr]
            }
        case ']':
            if tape[tapePtr] != 0 {
                codePtr = bracemap[codePtr]
            }
        }
        codePtr++
        instructionCount++
    }
    tapeLength = len(tape)
    return tapeLength, instructionCount, nil
}

func main() {
    if len(os.Args) != 2 {
        fmt.Println("Usage:", os.Args[0], "filename")
    } else {
        // read bf file
        data, err := ioutil.ReadFile(os.Args[1])
        if err != nil {
            fmt.Println(err)
            return
        }
        // run code
        tapeLength, instructionCount, err := runBrainfuck(data)
        if err != nil {
            fmt.Println(err)
        } else {
            fmt.Println("")
            fmt.Println("Tape length:", tapeLength)
            fmt.Println("Instruction count:", instructionCount)
        }
    }
}
<commit_msg>added "," instruction (read)<commit_after>/*
Brainfuck Interpreter
*/
package main

import (
    "errors"
    "fmt"
    "io/ioutil"
    "os"
)

func cleanup(data []byte) []byte {
    result := make([]byte, 0, len(data))
    for _, c := range data {
        switch c {
        case '>', '<', '+', '-', '.', ',', '[', ']':
            result = append(result, c)
        }
    }
    return result
}

func buildBracemap(data []byte) (bracemap map[int]int, err error) {
    stack := make([]int, 0, 100)
    bracemap = make(map[int]int)
    for pos, c := range data {
        switch c {
        case '[':
            stack = append(stack, pos)
        case ']':
            if len(stack) == 0 {
                err = errors.New("Syntax error: Unmatched closing brace")
                return bracemap, err
            }
            start := stack[len(stack)-1]
            bracemap[start] = pos
            bracemap[pos] = start
            stack = stack[:len(stack)-1]
        }
    }
    if len(stack) != 0 {
        err = errors.New("Syntax error: Not enough closing braces")
        return bracemap, err
    }
    return bracemap, nil
}

func runBrainfuck(data []byte) (tapeLength, instructionCount int, err error) {
    code := cleanup(data)
    bracemap, err := buildBracemap(code)
    if err != nil {
        return 0, 0, err
    }

    tape := make([]byte, 1, 1000)
    codePtr, tapePtr, instructionCount := 0, 0, 0
    for codePtr < len(code) {
        switch code[codePtr] {
        case '>':
            tapePtr++
            if tapePtr == len(tape) {
                tape = append(tape, 0)
            }
        case '<':
            tapePtr--
            if tapePtr == -1 {
                tape = append(tape, 0)
                copy(tape[1:], tape)
                tape[0] = 0
                tapePtr = 0
            }
        case '+':
            tape[tapePtr]++
        case '-':
            tape[tapePtr]--
        case '.':
            fmt.Print(string(tape[tapePtr]))
        case ',':
            // todo: Test this
            b := make([]uint8, 1)
            _, _ = os.Stdin.Read(b)
            tape[tapePtr] = b[0]
        case '[':
            if tape[tapePtr] == 0 {
                codePtr = bracemap[codePtr]
            }
        case ']':
            if tape[tapePtr] != 0 {
                codePtr = bracemap[codePtr]
            }
        }
        codePtr++
        instructionCount++
    }
    tapeLength = len(tape)
    return tapeLength, instructionCount, nil
}

func main() {
    if len(os.Args) != 2 {
        fmt.Println("Usage:", os.Args[0], "filename")
    } else {
        // read bf file
        data, err := ioutil.ReadFile(os.Args[1])
        if err != nil {
            fmt.Println(err)
            return
        }
        // run code
        tapeLength, instructionCount, err := runBrainfuck(data)
        if err != nil {
            fmt.Println(err)
        } else {
            fmt.Println("")
            fmt.Println("Tape length:", tapeLength)
            fmt.Println("Instruction count:", instructionCount)
        }
    }
}
<|endoftext|>


<commit_before>package brew

import (
	"bytes"
	"context"
	"crypto/sha256"
	"fmt"
	"text/template"
	"strings"

	"github.com/google/go-github/github"
	"github.com/goreleaser/releaser/config"
	"golang.org/x/oauth2"
)

const formulae = `class {{ .Name }} < Formula
  desc "{{ .Desc }}"
  homepage "{{ .Homepage }}"
  url "https://github.com/{{ .Repo }}/releases/download/{{ .Tag }}/{{ .BinaryName }}_Darwin_x86_64.tar.gz"
  head "https://github.com/{{ .Repo }}.git"

  def install
    bin.install "{{ .BinaryName }}"
  end
end
`

type templateData struct {
	Name, Desc, Homepage, Repo, Tag, BinaryName string
}

func Brew(version string, config config.ProjectConfig) error {
	fmt.Println("Updating brew formulae...")
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Token},
	)
	tc := oauth2.NewClient(context.Background(), ts)
	client := github.NewClient(tc)
	parts := strings.Split(config.Brew.Repo, "/")

	tmpl, err := template.New(config.BinaryName).Parse(formulae)
	if err != nil {
		return err
	}

	data, err := dataFor(version, config, client)
	if err != nil {
		return err
	}

	var out bytes.Buffer
	tmpl.Execute(&out, data)

	var sha *string
	file, _, _, err := client.Repositories.GetContents(
		parts[0], parts[1], config.BinaryName+".rb", &github.RepositoryContentGetOptions{},
	)
	if err == nil {
		sha = file.SHA
	} else {
		sha = github.String(fmt.Sprintf("%s", sha256.Sum256(out.Bytes())))
	}

	_, _, err = client.Repositories.UpdateFile(
		parts[0],
		parts[1],
		config.BinaryName+".rb",
		&github.RepositoryContentFileOptions{
			Committer: &github.CommitAuthor{
				Name:  github.String("goreleaserbot"),
				Email: github.String("bot@goreleaser"),
			},
			Content: out.Bytes(),
			Message: github.String(config.BinaryName + " version " + version),
			SHA:     sha,
		},
	)
	return err
}

func dataFor(version string, config config.ProjectConfig, client *github.Client) (result templateData, err error) {
	var homepage string
	var description string
	parts := strings.Split(config.Repo, "/")
	rep, _, err := client.Repositories.Get(parts[0], parts[1])
	if err != nil {
		return result, err
	}
	if rep.Homepage == nil {
		homepage = *rep.HTMLURL
	} else {
		homepage = *rep.Homepage
	}
	if rep.Description == nil {
		description = "TODO"
	} else {
		description = *rep.Description
	}
	return templateData{
		Name:       formulaNameFor(config.BinaryName),
		Desc:       description,
		Homepage:   homepage,
		Repo:       config.Repo,
		Tag:        version,
		BinaryName: config.BinaryName,
	}, err
}

func formulaNameFor(name string) string {
	name = strings.Replace(name, "-", " ", -1)
	name = strings.Replace(name, "_", " ", -1)
	return strings.Replace(strings.Title(name), " ", "", -1)
}<commit_msg>linuxbrew compat<commit_after>package brew

import (
	"bytes"
	"context"
	"crypto/sha256"
	"fmt"
	"text/template"
	"strings"

	"github.com/google/go-github/github"
	"github.com/goreleaser/releaser/config"
	"golang.org/x/oauth2"
)

const formulae = `class {{ .Name }} < Formula
  desc "{{ .Desc }}"
  homepage "{{ .Homepage }}"
  url "https://github.com/{{ .Repo }}/releases/download/{{ .Tag }}/{{ .BinaryName }}_#{%x(uname -s).gsub(/\n/, '')}_#{%x(uname -m).gsub(/\n/, '')}.tar.gz"
  head "https://github.com/{{ .Repo }}.git"

  def install
    bin.install "{{ .BinaryName }}"
  end
end
`

type templateData struct {
	Name, Desc, Homepage, Repo, Tag, BinaryName string
}

func Brew(version string, config config.ProjectConfig) error {
	fmt.Println("Updating brew formulae...")
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: config.Token},
	)
	tc := oauth2.NewClient(context.Background(), ts)
	client := github.NewClient(tc)
	parts := strings.Split(config.Brew.Repo, "/")

	tmpl, err := template.New(config.BinaryName).Parse(formulae)
	if err != nil {
		return err
	}

	data, err := dataFor(version, config, client)
	if err != nil {
		return err
	}

	var out bytes.Buffer
	tmpl.Execute(&out, data)

	var sha *string
	file, _, _, err := client.Repositories.GetContents(
		parts[0], parts[1], config.BinaryName+".rb", &github.RepositoryContentGetOptions{},
	)
	if err == nil {
		sha = file.SHA
	} else {
		sha = github.String(fmt.Sprintf("%s", sha256.Sum256(out.Bytes())))
	}

	_, _, err = client.Repositories.UpdateFile(
		parts[0],
		parts[1],
		config.BinaryName+".rb",
		&github.RepositoryContentFileOptions{
			Committer: &github.CommitAuthor{
				Name:  github.String("goreleaserbot"),
				Email: github.String("bot@goreleaser"),
			},
			Content: out.Bytes(),
			Message: github.String(config.BinaryName + " version " + version),
			SHA:     sha,
		},
	)
	return err
}

func dataFor(version string, config config.ProjectConfig, client *github.Client) (result templateData, err error) {
	var homepage string
	var description string
	parts := strings.Split(config.Repo, "/")
	rep, _, err := client.Repositories.Get(parts[0], parts[1])
	if err != nil {
		return result, err
	}
	if rep.Homepage == nil {
		homepage = *rep.HTMLURL
	} else {
		homepage = *rep.Homepage
	}
	if rep.Description == nil {
		description = "TODO"
	} else {
		description = *rep.Description
	}
	return templateData{
		Name:       formulaNameFor(config.BinaryName),
		Desc:       description,
		Homepage:   homepage,
		Repo:       config.Repo,
		Tag:        version,
		BinaryName: config.BinaryName,
	}, err
}

func formulaNameFor(name string) string {
	name = strings.Replace(name, "-", " ", -1)
	name = strings.Replace(name, "_", " ", -1)
	return strings.Replace(strings.Title(name), " ", "", -1)
}<|endoftext|>


<commit_before>package number

// IntegerGenerator is the interface implemented by objects providing sequence
// of integers.
type IntegerGenerator interface {

	// NextValue returns the next value from the generator.
	NextValue() int64
}

// ConstantIntegerGenerator is an integer generator that always returns the same
// value.
type ConstantIntegerGenerator struct {
	constant int64
}

// NewConstantIntegerGenerator returns an IntegerGenerator that always returns
// the provided constant.
func NewConstantIntegerGenerator(constant int64) IntegerGenerator {
	return ConstantIntegerGenerator{constant: constant}
}

// NextValue returns the next value from the generator.
func (g ConstantIntegerGenerator) NextValue() int64 {
	return g.constant
}

// ProbabilityGenerator is the interface implemented by objects providing sequence
// of probabilities (floating points numbers between 0 and 1).
type ProbabilityGenerator interface {

	// NextValue returns the next value from the generator.
	NextValue() Probability
}

// ConstantProbabilityGenerator is a probability generator that always returns
// the same value.
type ConstantProbabilityGenerator struct {
	constant Probability
}

// NewConstantProbabilityGenerator returns a ProbabilityGenerator that always
// returns the provided constant.
func NewConstantProbabilityGenerator(constant Probability) ProbabilityGenerator {
	return ConstantProbabilityGenerator{constant: constant}
}

// NextValue returns the next value from the generator.
func (g ConstantProbabilityGenerator) NextValue() Probability {
	return g.constant
}
<commit_msg>Add Float64Generator and variants<commit_after>package number

import (
	"math"
	"math/rand"
	"time"
)

// IntegerGenerator is the interface implemented by objects providing sequence
// of integers.
type IntegerGenerator interface {

	// NextValue returns the next value from the generator.
	NextValue() int64
}

// ConstantIntegerGenerator is an integer generator that always returns the same
// value.
type ConstantIntegerGenerator struct {
	constant int64
}

// NewConstantIntegerGenerator returns an IntegerGenerator that always returns
// the provided constant.
func NewConstantIntegerGenerator(constant int64) IntegerGenerator {
	return ConstantIntegerGenerator{constant: constant}
}

// NextValue returns the next value from the generator.
func (g ConstantIntegerGenerator) NextValue() int64 {
	return g.constant
}

// Float64Generator is the interface implemented by objects providing sequence
// of float64
type Float64Generator interface {

	// NextValue returns the next value from the generator.
	NextValue() float64
}

// ConstantFloat64Generator is an integer generator that always returns the same
// value.
type ConstantFloat64Generator struct {
	constant float64
}

// NewConstantFloat64Generator returns an Float64Generator that always returns
// the provided constant.
func NewConstantFloat64Generator(constant float64) Float64Generator {
	return ConstantFloat64Generator{constant: constant}
}

// NextValue returns the next value from the generator.
func (g ConstantFloat64Generator) NextValue() float64 {
	return g.constant
}

// BoundedFloat64Generator produces float64 bounded in a given range
type BoundedFloat64Generator struct {
	min, max float64    // both min and max are exclusive
	rng      *rand.Rand // PRNG used to generate the values
}

// NewBoundedFloat64Generator returns a BoundedFloat64Generator that uses a
// default uniform PRNG.
func NewBoundedFloat64Generator(min, max float64) *BoundedFloat64Generator {
	g := &BoundedFloat64Generator{min: min, max: max}
	g.SetRNG(rand.New(rand.NewSource(int64(time.Now().UnixNano()))))
	return g
}

// SetRNG sets the pseudo random number generator used to produce the values.
func (g *BoundedFloat64Generator) SetRNG(rng *rand.Rand) {
	g.rng = rng
}

// NextValue returns the next value from the generator.
func (g *BoundedFloat64Generator) NextValue() float64 {
	return g.rng.Float64() + math.SmallestNonzeroFloat64
}

// ProbabilityGenerator is the interface implemented by objects providing sequence
// of probabilities (floating points numbers between 0 and 1).
type ProbabilityGenerator interface {

	// NextValue returns the next value from the generator.
	NextValue() Probability
}

// ConstantProbabilityGenerator is a probability generator that always returns
// the same value.
type ConstantProbabilityGenerator struct {
	constant Probability
}

// NewConstantProbabilityGenerator returns a ProbabilityGenerator that always
// returns the provided constant.
func NewConstantProbabilityGenerator(constant Probability) ProbabilityGenerator {
	return ConstantProbabilityGenerator{constant: constant}
}

// NextValue returns the next value from the generator.
func (g ConstantProbabilityGenerator) NextValue() Probability {
	return g.constant
}
<|endoftext|>


<commit_before>package store_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/store"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func (s *StoreSuite) dummyBranch(c *C, suffix string) bzrDir {
	tmpDir := c.MkDir()
	if suffix != "" {
		tmpDir = filepath.Join(tmpDir, suffix)
		err := os.MkdirAll(tmpDir, 0755)
		c.Assert(err, IsNil)
	}
	branch := bzrDir(tmpDir)
	branch.init()

	copyCharmDir(branch.path(), testing.Charms.Dir("dummy"))
	branch.add()
	branch.commit("Imported charm.")
	return branch
}

var urls = []*charm.URL{
	charm.MustParseURL("cs:~joe/oneiric/dummy"),
	charm.MustParseURL("cs:oneiric/dummy"),
}

type fakePlugin struct {
	oldEnv string
}

func (p *fakePlugin) install(dir string, content string) {
	if p.oldEnv == "" {
		p.oldEnv = os.Getenv("BZR_PLUGINS_AT")
	}
	err := ioutil.WriteFile(filepath.Join(dir, "__init__.py"), []byte(content), 0644)
	if err != nil {
		panic(err)
	}
	os.Setenv("BZR_PLUGINS_AT", "fakePlugin@" + dir)
}

func (p *fakePlugin) uninstall() {
	os.Setenv("BZR_PLUGINS_AT", p.oldEnv)
}

func (s *StoreSuite) TestPublish(c *C) {
	// Ensure that the streams are parsed separately by inserting
	// garbage on stderr. The wanted information is still there.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n")`)
	defer plugin.uninstall()

	branch := s.dummyBranch(c, "")

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)

	for _, url := range urls {
		info, rc, err := s.store.OpenCharm(url)
		c.Assert(err, IsNil)
		defer rc.Close()
		c.Assert(info.Revision(), Equals, 0)
		c.Assert(info.Meta().Name, Equals, "dummy")

		data, err := ioutil.ReadAll(rc)
		c.Assert(err, IsNil)

		bundle, err := charm.ReadBundleBytes(data)
		c.Assert(err, IsNil)
		c.Assert(bundle.Revision(), Equals, 0)
		c.Assert(bundle.Meta().Name, Equals, "dummy")
	}

	// Attempt to publish the same content again while providing the wrong
	// tip revision. It must pick the real revision from the branch and
	// note this was previously published.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Bump the content revision and lie again about the known tip revision.
	// This time, though, pretend it's the same as the real branch revision
	// previously published. It must error and not publish the new revision
	// because it will use the revision provided as a parameter to check if
	// publishing was attempted before. This is the mechanism that enables
	// stopping fast without having to download every single branch. Real
	// revision is picked in the next scan.
	digest1 := branch.digest()
	branch.change()
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), digest1)
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Now allow it to publish the new content by providing an unseen revision.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)
	digest2 := branch.digest()

	info, err := s.store.CharmInfo(urls[0])
	c.Assert(err, IsNil)
	c.Assert(info.Revision(), Equals, 1)
	c.Assert(info.Meta().Name, Equals, "dummy")

	// There are two events published, for each of the successful attempts.
	// The failures are ignored given that they are artifacts of the
	// publishing mechanism rather than actual problems.
	_, err = s.store.CharmEvent(urls[0], "wrong-rev")
	c.Assert(err, Equals, store.ErrNotFound)
	for i, digest := range []string{digest1, digest2} {
		event, err := s.store.CharmEvent(urls[0], digest)
		c.Assert(err, IsNil)
		c.Assert(event.Kind, Equals, store.EventPublished)
		c.Assert(event.Revision, Equals, i)
		c.Assert(event.Errors, IsNil)
		c.Assert(event.Warnings, IsNil)
	}
}

func (s *StoreSuite) TestPublishErrorFromBzr(c *C) {
	// In TestPublish we ensure that the streams are parsed
	// separately by inserting garbage on stderr. Now make
	// sure that stderr isn't simply trashed, as we want to
	// know about what a real error tells us.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n"); sys.exit(1)`)
	defer plugin.uninstall()

	branch := s.dummyBranch(c, "")

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, "(?s).*STDERR STUFF.*")
}

func (s *StoreSuite) TestPublishErrorInCharm(c *C) {
	branch := s.dummyBranch(c, "")

	// Corrupt the charm.
	branch.remove("metadata.yaml")
	branch.commit("Removed metadata.yaml.")

	// Attempt to publish the erroneous content.
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, ".*/metadata.yaml: no such file or directory")

	// The event should be logged as well, since this was an error in the charm
	// that won't go away and must be communicated to the author.
	event, err := s.store.CharmEvent(urls[0], branch.digest())
	c.Assert(err, IsNil)
	c.Assert(event.Kind, Equals, store.EventPublishError)
	c.Assert(event.Revision, Equals, 0)
	c.Assert(event.Errors, NotNil)
	c.Assert(event.Errors[0], Matches, ".*/metadata.yaml: no such file or directory")
	c.Assert(event.Warnings, IsNil)
}

type bzrDir string

func (dir bzrDir) path(args ...string) string {
	return filepath.Join(append([]string{string(dir)}, args...)...)
}

func (dir bzrDir) run(args ...string) []byte {
	cmd := exec.Command("bzr", args...)
	cmd.Dir = string(dir)
	output, err := cmd.Output()
	if err != nil {
		panic(fmt.Sprintf("command failed: bzr %s\n%s", strings.Join(args, " "), output))
	}
	return output
}

func (dir bzrDir) init() {
	dir.run("init")
}

func (dir bzrDir) add(paths ...string) {
	dir.run(append([]string{"add"}, paths...)...)
}

func (dir bzrDir) remove(paths ...string) {
	dir.run(append([]string{"rm"}, paths...)...)
}

func (dir bzrDir) commit(msg string) {
	dir.run("commit", "-m", msg)
}

func (dir bzrDir) write(path string, data string) {
	err := ioutil.WriteFile(dir.path(path), []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func (dir bzrDir) change() {
	t := time.Now().String()
	dir.write("timestamp", t)
	dir.add("timestamp")
	dir.commit("Revision bumped at " + t)
}

func (dir bzrDir) digest() string {
	output := dir.run("revision-info")
	f := bytes.Fields(output)
	if len(f) != 2 {
		panic("revision-info returned bad output: " + string(output))
	}
	return string(f[1])
}

func copyCharmDir(dst string, dir *charm.Dir) {
	var b bytes.Buffer
	err := dir.BundleTo(&b)
	if err != nil {
		panic(err)
	}
	bundle, err := charm.ReadBundleBytes(b.Bytes())
	if err != nil {
		panic(err)
	}
	err = bundle.ExpandTo(dst)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Test of oldEnv was crackful.<commit_after>package store_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/store"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func (s *StoreSuite) dummyBranch(c *C, suffix string) bzrDir {
	tmpDir := c.MkDir()
	if suffix != "" {
		tmpDir = filepath.Join(tmpDir, suffix)
		err := os.MkdirAll(tmpDir, 0755)
		c.Assert(err, IsNil)
	}
	branch := bzrDir(tmpDir)
	branch.init()

	copyCharmDir(branch.path(), testing.Charms.Dir("dummy"))
	branch.add()
	branch.commit("Imported charm.")
	return branch
}

var urls = []*charm.URL{
	charm.MustParseURL("cs:~joe/oneiric/dummy"),
	charm.MustParseURL("cs:oneiric/dummy"),
}

type fakePlugin struct {
	oldEnv string
}

func (p *fakePlugin) install(dir string, content string) {
	p.oldEnv = os.Getenv("BZR_PLUGINS_AT")
	err := ioutil.WriteFile(filepath.Join(dir, "__init__.py"), []byte(content), 0644)
	if err != nil {
		panic(err)
	}
	os.Setenv("BZR_PLUGINS_AT", "fakePlugin@" + dir)
}

func (p *fakePlugin) uninstall() {
	os.Setenv("BZR_PLUGINS_AT", p.oldEnv)
}

func (s *StoreSuite) TestPublish(c *C) {
	// Ensure that the streams are parsed separately by inserting
	// garbage on stderr. The wanted information is still there.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n")`)
	defer plugin.uninstall()

	branch := s.dummyBranch(c, "")

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)

	for _, url := range urls {
		info, rc, err := s.store.OpenCharm(url)
		c.Assert(err, IsNil)
		defer rc.Close()
		c.Assert(info.Revision(), Equals, 0)
		c.Assert(info.Meta().Name, Equals, "dummy")

		data, err := ioutil.ReadAll(rc)
		c.Assert(err, IsNil)

		bundle, err := charm.ReadBundleBytes(data)
		c.Assert(err, IsNil)
		c.Assert(bundle.Revision(), Equals, 0)
		c.Assert(bundle.Meta().Name, Equals, "dummy")
	}

	// Attempt to publish the same content again while providing the wrong
	// tip revision. It must pick the real revision from the branch and
	// note this was previously published.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Bump the content revision and lie again about the known tip revision.
	// This time, though, pretend it's the same as the real branch revision
	// previously published. It must error and not publish the new revision
	// because it will use the revision provided as a parameter to check if
	// publishing was attempted before. This is the mechanism that enables
	// stopping fast without having to download every single branch. Real
	// revision is picked in the next scan.
	digest1 := branch.digest()
	branch.change()
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), digest1)
	c.Assert(err, Equals, store.ErrRedundantUpdate)

	// Now allow it to publish the new content by providing an unseen revision.
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, IsNil)
	digest2 := branch.digest()

	info, err := s.store.CharmInfo(urls[0])
	c.Assert(err, IsNil)
	c.Assert(info.Revision(), Equals, 1)
	c.Assert(info.Meta().Name, Equals, "dummy")

	// There are two events published, for each of the successful attempts.
	// The failures are ignored given that they are artifacts of the
	// publishing mechanism rather than actual problems.
	_, err = s.store.CharmEvent(urls[0], "wrong-rev")
	c.Assert(err, Equals, store.ErrNotFound)
	for i, digest := range []string{digest1, digest2} {
		event, err := s.store.CharmEvent(urls[0], digest)
		c.Assert(err, IsNil)
		c.Assert(event.Kind, Equals, store.EventPublished)
		c.Assert(event.Revision, Equals, i)
		c.Assert(event.Errors, IsNil)
		c.Assert(event.Warnings, IsNil)
	}
}

func (s *StoreSuite) TestPublishErrorFromBzr(c *C) {
	// In TestPublish we ensure that the streams are parsed
	// separately by inserting garbage on stderr. Now make
	// sure that stderr isn't simply trashed, as we want to
	// know about what a real error tells us.
	plugin := fakePlugin{}
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n"); sys.exit(1)`)
	defer plugin.uninstall()

	branch := s.dummyBranch(c, "")

	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, "(?s).*STDERR STUFF.*")
}

func (s *StoreSuite) TestPublishErrorInCharm(c *C) {
	branch := s.dummyBranch(c, "")

	// Corrupt the charm.
	branch.remove("metadata.yaml")
	branch.commit("Removed metadata.yaml.")

	// Attempt to publish the erroneous content.
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
	c.Assert(err, ErrorMatches, ".*/metadata.yaml: no such file or directory")

	// The event should be logged as well, since this was an error in the charm
	// that won't go away and must be communicated to the author.
	event, err := s.store.CharmEvent(urls[0], branch.digest())
	c.Assert(err, IsNil)
	c.Assert(event.Kind, Equals, store.EventPublishError)
	c.Assert(event.Revision, Equals, 0)
	c.Assert(event.Errors, NotNil)
	c.Assert(event.Errors[0], Matches, ".*/metadata.yaml: no such file or directory")
	c.Assert(event.Warnings, IsNil)
}

type bzrDir string

func (dir bzrDir) path(args ...string) string {
	return filepath.Join(append([]string{string(dir)}, args...)...)
}

func (dir bzrDir) run(args ...string) []byte {
	cmd := exec.Command("bzr", args...)
	cmd.Dir = string(dir)
	output, err := cmd.Output()
	if err != nil {
		panic(fmt.Sprintf("command failed: bzr %s\n%s", strings.Join(args, " "), output))
	}
	return output
}

func (dir bzrDir) init() {
	dir.run("init")
}

func (dir bzrDir) add(paths ...string) {
	dir.run(append([]string{"add"}, paths...)...)
}

func (dir bzrDir) remove(paths ...string) {
	dir.run(append([]string{"rm"}, paths...)...)
}

func (dir bzrDir) commit(msg string) {
	dir.run("commit", "-m", msg)
}

func (dir bzrDir) write(path string, data string) {
	err := ioutil.WriteFile(dir.path(path), []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func (dir bzrDir) change() {
	t := time.Now().String()
	dir.write("timestamp", t)
	dir.add("timestamp")
	dir.commit("Revision bumped at " + t)
}

func (dir bzrDir) digest() string {
	output := dir.run("revision-info")
	f := bytes.Fields(output)
	if len(f) != 2 {
		panic("revision-info returned bad output: " + string(output))
	}
	return string(f[1])
}

func copyCharmDir(dst string, dir *charm.Dir) {
	var b bytes.Buffer
	err := dir.BundleTo(&b)
	if err != nil {
		panic(err)
	}
	bundle, err := charm.ReadBundleBytes(b.Bytes())
	if err != nil {
		panic(err)
	}
	err = bundle.ExpandTo(dst)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2015-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package object

import (
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"

	"net/http"
	"net/url"

	"github.com/vmware/govmomi/property"
	"github.com/vmware/govmomi/session"
	"github.com/vmware/govmomi/vim25"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

// DatastoreNoSuchDirectoryError is returned when a directory could not be found.
type DatastoreNoSuchDirectoryError struct {
	verb    string
	subject string
}

func (e DatastoreNoSuchDirectoryError) Error() string {
	return fmt.Sprintf("cannot %s '%s': No such directory", e.verb, e.subject)
}

// DatastoreNoSuchFileError is returned when a file could not be found.
type DatastoreNoSuchFileError struct {
	verb    string
	subject string
}

func (e DatastoreNoSuchFileError) Error() string {
	return fmt.Sprintf("cannot %s '%s': No such file", e.verb, e.subject)
}

type Datastore struct {
	Common
}

func NewDatastore(c *vim25.Client, ref types.ManagedObjectReference) *Datastore {
	return &Datastore{
		Common: NewCommon(c, ref),
	}
}

func (d Datastore) Path(path string) string {
	name := d.Name()
	if name == "" {
		panic("expected non-empty name")
	}

	return fmt.Sprintf("[%s] %s", name, path)
}

// URL for datastore access over HTTP
func (d Datastore) URL(ctx context.Context, dc *Datacenter, path string) (*url.URL, error) {
	var mdc mo.Datacenter
	if err := dc.Properties(ctx, dc.Reference(), []string{"name"}, &mdc); err != nil {
		return nil, err
	}

	var mds mo.Datastore
	if err := d.Properties(ctx, d.Reference(), []string{"name"}, &mds); err != nil {
		return nil, err
	}

	u := d.c.URL()

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
		Path:   fmt.Sprintf("/folder/%s", path),
		RawQuery: url.Values{
			"dcPath": []string{mdc.Name},
			"dsName": []string{mds.Name},
		}.Encode(),
	}, nil
}

func (d Datastore) Browser(ctx context.Context) (*HostDatastoreBrowser, error) {
	var do mo.Datastore

	err := d.Properties(ctx, d.Reference(), []string{"browser"}, &do)
	if err != nil {
		return nil, err
	}

	return NewHostDatastoreBrowser(d.c, do.Browser), nil
}

func (d Datastore) useServiceTicketHostName(name string) bool {
	// No need if talking directly to ESX.
	if !d.c.IsVC() {
		return false
	}

	// If version happens to be < 5.1
	if name == "" {
		return false
	}

	// If the HostSystem is using DHCP on a network without dynamic DNS,
	// HostSystem.Config.Network.DnsConfig.HostName is set to "localhost" by default.
	// This resolves to "localhost.localdomain" by default via /etc/hosts on ESX.
	// In that case, we will stick with the HostSystem.Name which is the IP address that
	// was used to connect the host to VC.
	if name == "localhost.localdomain" {
		return false
	}

	// An escape hatch, as it is still possible to have HostName that doesn't resolve via DNS,
	// or resolves to an address that isn't reachable.
	env := os.Getenv("GOVMOMI_USE_SERVICE_TICKET_HOSTNAME")
	if env == "0" || env == "false" {
		return false
	}

	return true
}

// ServiceTicket obtains a ticket via AcquireGenericServiceTicket and returns it an http.Cookie with the url.URL
// that can be used along with the ticket cookie to access the given path.
func (d Datastore) ServiceTicket(ctx context.Context, path string, method string) (*url.URL, *http.Cookie, error) {
	// We are uploading to an ESX host
	u := &url.URL{
		Scheme: d.c.URL().Scheme,
		Host:   d.c.URL().Host,
		Path:   fmt.Sprintf("/folder/%s", path),
		RawQuery: url.Values{
			"dsName": []string{d.Name()},
		}.Encode(),
	}

	// If connected to VC, the ticket request must be for an ESX host.
	if d.c.IsVC() {
		hosts, err := d.AttachedHosts(ctx)
		if err != nil {
			return nil, nil, err
		}

		if len(hosts) == 0 {
			return nil, nil, fmt.Errorf("no hosts attached to datastore %#v", d.Reference())
		}

		// Pick a random attached host
		host := hosts[rand.Intn(len(hosts))]
		name, err := host.ObjectName(ctx)
		if err != nil {
			return nil, nil, err
		}
		u.Host = name
	}

	spec := types.SessionManagerHttpServiceRequestSpec{
		Url: u.String(),
		// See SessionManagerHttpServiceRequestSpecMethod enum
		Method: fmt.Sprintf("http%s%s", method[0:1], strings.ToLower(method[1:])),
	}

	sm := session.NewManager(d.Client())

	ticket, err := sm.AcquireGenericServiceTicket(ctx, &spec)
	if err != nil {
		return nil, nil, err
	}

	cookie := &http.Cookie{
		Name:  "vmware_cgi_ticket",
		Value: ticket.Id,
	}

	if d.useServiceTicketHostName(ticket.HostName) {
		u.Host = ticket.HostName
	}

	return u, cookie, nil
}

func (d Datastore) uploadTicket(ctx context.Context, path string, param *soap.Upload) (*url.URL, *soap.Upload, error) {
	p := soap.DefaultUpload
	if param != nil {
		p = *param // copy
	}

	u, ticket, err := d.ServiceTicket(ctx, path, p.Method)
	if err != nil {
		return nil, nil, err
	}

	p.Ticket = ticket

	return u, &p, nil
}

func (d Datastore) downloadTicket(ctx context.Context, path string, param *soap.Download) (*url.URL, *soap.Download, error) {
	p := soap.DefaultDownload
	if param != nil {
		p = *param // copy
	}

	u, ticket, err := d.ServiceTicket(ctx, path, p.Method)
	if err != nil {
		return nil, nil, err
	}

	p.Ticket = ticket

	return u, &p, nil
}

// Upload via soap.Upload with an http service ticket
func (d Datastore) Upload(ctx context.Context, f io.Reader, path string, param *soap.Upload) error {
	u, p, err := d.uploadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().Upload(f, u, p)
}

// UploadFile via soap.Upload with an http service ticket
func (d Datastore) UploadFile(ctx context.Context, file string, path string, param *soap.Upload) error {
	u, p, err := d.uploadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().UploadFile(file, u, p)
}

// Download via soap.Download with an http service ticket
func (d Datastore) Download(ctx context.Context, path string, param *soap.Download) (io.ReadCloser, int64, error) {
	u, p, err := d.downloadTicket(ctx, path, param)
	if err != nil {
		return nil, 0, err
	}
	return d.Client().Download(u, p)
}

// DownloadFile via soap.Download with an http service ticket
func (d Datastore) DownloadFile(ctx context.Context, path string, file string, param *soap.Download) error {
	u, p, err := d.downloadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().DownloadFile(file, u, p)
}

// AttachedHosts returns hosts that have this Datastore attached, accessible and writable.
func (d Datastore) AttachedHosts(ctx context.Context) ([]*HostSystem, error) {
	var ds mo.Datastore
	var hosts []*HostSystem

	pc := property.DefaultCollector(d.Client())
	err := pc.RetrieveOne(ctx, d.Reference(), []string{"host"}, &ds)
	if err != nil {
		return nil, err
	}

	mounts := make(map[types.ManagedObjectReference]types.DatastoreHostMount)
	var refs []types.ManagedObjectReference
	for _, host := range ds.Host {
		refs = append(refs, host.Key)
		mounts[host.Key] = host
	}

	var hs []mo.HostSystem
	err = pc.Retrieve(ctx, refs, []string{"runtime.connectionState", "runtime.powerState"}, &hs)
	if err != nil {
		return nil, err
	}

	for _, host := range hs {
		if host.Runtime.ConnectionState == types.HostSystemConnectionStateConnected &&
			host.Runtime.PowerState == types.HostSystemPowerStatePoweredOn {

			mount := mounts[host.Reference()]
			info := mount.MountInfo

			if *info.Mounted && *info.Accessible && info.AccessMode == string(types.HostMountModeReadWrite) {
				hosts = append(hosts, NewHostSystem(d.Client(), mount.Key))
			}
		}
	}

	return hosts, nil
}

// AttachedHosts returns hosts that have this Datastore attached, accessible and writable and are members of the given cluster.
func (d Datastore) AttachedClusterHosts(ctx context.Context, cluster *ComputeResource) ([]*HostSystem, error) {
	var hosts []*HostSystem

	clusterHosts, err := cluster.Hosts(ctx)
	if err != nil {
		return nil, err
	}

	attachedHosts, err := d.AttachedHosts(ctx)
	if err != nil {
		return nil, err
	}

	refs := make(map[types.ManagedObjectReference]bool)
	for _, host := range attachedHosts {
		refs[host.Reference()] = true
	}

	for _, host := range clusterHosts {
		if refs[host.Reference()] {
			hosts = append(hosts, host)
		}
	}

	return hosts, nil
}

func (d Datastore) Stat(ctx context.Context, file string) (types.BaseFileInfo, error) {
	b, err := d.Browser(ctx)
	if err != nil {
		return nil, err
	}

	spec := types.HostDatastoreBrowserSearchSpec{
		Details: &types.FileQueryFlags{
			FileType:     true,
			FileSize:     true,
			Modification: true,
			FileOwner:    types.NewBool(true),
		},
		MatchPattern: []string{path.Base(file)},
	}

	dsPath := d.Path(path.Dir(file))
	task, err := b.SearchDatastore(context.TODO(), dsPath, &spec)
	if err != nil {
		return nil, err
	}

	info, err := task.WaitForResult(context.TODO(), nil)
	if err != nil {
		if info == nil || info.Error != nil {
			_, ok := info.Error.Fault.(*types.FileNotFound)
			if ok {
				// FileNotFound means the base path doesn't exist.
				return nil, DatastoreNoSuchDirectoryError{"stat", dsPath}
			}
		}

		return nil, err
	}

	res := info.Result.(types.HostDatastoreBrowserSearchResults)
	if len(res.File) == 0 {
		// File doesn't exist
		return nil, DatastoreNoSuchFileError{"stat", d.Path(file)}
	}

	return res.File[0], nil

}

// Type returns the type of file system volume.
func (d Datastore) Type(ctx context.Context) (types.HostFileSystemVolumeFileSystemType, error) {
	var mds mo.Datastore

	if err := d.Properties(ctx, d.Reference(), []string{"summary.type"}, &mds); err != nil {
		return types.HostFileSystemVolumeFileSystemType(""), err
	}
	return types.HostFileSystemVolumeFileSystemType(mds.Summary.Type), nil
}
<commit_msg>Disable datastore service ticket hostname usage<commit_after>/*
Copyright (c) 2015-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package object

import (
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"

	"net/http"
	"net/url"

	"github.com/vmware/govmomi/property"
	"github.com/vmware/govmomi/session"
	"github.com/vmware/govmomi/vim25"
	"github.com/vmware/govmomi/vim25/mo"
	"github.com/vmware/govmomi/vim25/soap"
	"github.com/vmware/govmomi/vim25/types"
	"golang.org/x/net/context"
)

// DatastoreNoSuchDirectoryError is returned when a directory could not be found.
type DatastoreNoSuchDirectoryError struct {
	verb    string
	subject string
}

func (e DatastoreNoSuchDirectoryError) Error() string {
	return fmt.Sprintf("cannot %s '%s': No such directory", e.verb, e.subject)
}

// DatastoreNoSuchFileError is returned when a file could not be found.
type DatastoreNoSuchFileError struct {
	verb    string
	subject string
}

func (e DatastoreNoSuchFileError) Error() string {
	return fmt.Sprintf("cannot %s '%s': No such file", e.verb, e.subject)
}

type Datastore struct {
	Common
}

func NewDatastore(c *vim25.Client, ref types.ManagedObjectReference) *Datastore {
	return &Datastore{
		Common: NewCommon(c, ref),
	}
}

func (d Datastore) Path(path string) string {
	name := d.Name()
	if name == "" {
		panic("expected non-empty name")
	}

	return fmt.Sprintf("[%s] %s", name, path)
}

// URL for datastore access over HTTP
func (d Datastore) URL(ctx context.Context, dc *Datacenter, path string) (*url.URL, error) {
	var mdc mo.Datacenter
	if err := dc.Properties(ctx, dc.Reference(), []string{"name"}, &mdc); err != nil {
		return nil, err
	}

	var mds mo.Datastore
	if err := d.Properties(ctx, d.Reference(), []string{"name"}, &mds); err != nil {
		return nil, err
	}

	u := d.c.URL()

	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
		Path:   fmt.Sprintf("/folder/%s", path),
		RawQuery: url.Values{
			"dcPath": []string{mdc.Name},
			"dsName": []string{mds.Name},
		}.Encode(),
	}, nil
}

func (d Datastore) Browser(ctx context.Context) (*HostDatastoreBrowser, error) {
	var do mo.Datastore

	err := d.Properties(ctx, d.Reference(), []string{"browser"}, &do)
	if err != nil {
		return nil, err
	}

	return NewHostDatastoreBrowser(d.c, do.Browser), nil
}

func (d Datastore) useServiceTicketHostName(name string) bool {
	// No need if talking directly to ESX.
	if !d.c.IsVC() {
		return false
	}

	// If version happens to be < 5.1
	if name == "" {
		return false
	}

	// If the HostSystem is using DHCP on a network without dynamic DNS,
	// HostSystem.Config.Network.DnsConfig.HostName is set to "localhost" by default.
	// This resolves to "localhost.localdomain" by default via /etc/hosts on ESX.
	// In that case, we will stick with the HostSystem.Name which is the IP address that
	// was used to connect the host to VC.
	if name == "localhost.localdomain" {
		return false
	}

	// Still possible to have HostName that don't resolve via DNS,
	// so we default to false.
	key := "GOVMOMI_USE_SERVICE_TICKET_HOSTNAME"

	val := d.c.URL().Query().Get(key)
	if val == "" {
		val = os.Getenv(key)
	}

	if val == "1" || val == "true" {
		return true
	}

	return false
}

// ServiceTicket obtains a ticket via AcquireGenericServiceTicket and returns it an http.Cookie with the url.URL
// that can be used along with the ticket cookie to access the given path.
func (d Datastore) ServiceTicket(ctx context.Context, path string, method string) (*url.URL, *http.Cookie, error) {
	// We are uploading to an ESX host
	u := &url.URL{
		Scheme: d.c.URL().Scheme,
		Host:   d.c.URL().Host,
		Path:   fmt.Sprintf("/folder/%s", path),
		RawQuery: url.Values{
			"dsName": []string{d.Name()},
		}.Encode(),
	}

	// If connected to VC, the ticket request must be for an ESX host.
	if d.c.IsVC() {
		hosts, err := d.AttachedHosts(ctx)
		if err != nil {
			return nil, nil, err
		}

		if len(hosts) == 0 {
			return nil, nil, fmt.Errorf("no hosts attached to datastore %#v", d.Reference())
		}

		// Pick a random attached host
		host := hosts[rand.Intn(len(hosts))]
		name, err := host.ObjectName(ctx)
		if err != nil {
			return nil, nil, err
		}
		u.Host = name
	}

	spec := types.SessionManagerHttpServiceRequestSpec{
		Url: u.String(),
		// See SessionManagerHttpServiceRequestSpecMethod enum
		Method: fmt.Sprintf("http%s%s", method[0:1], strings.ToLower(method[1:])),
	}

	sm := session.NewManager(d.Client())

	ticket, err := sm.AcquireGenericServiceTicket(ctx, &spec)
	if err != nil {
		return nil, nil, err
	}

	cookie := &http.Cookie{
		Name:  "vmware_cgi_ticket",
		Value: ticket.Id,
	}

	if d.useServiceTicketHostName(ticket.HostName) {
		u.Host = ticket.HostName
	}

	return u, cookie, nil
}

func (d Datastore) uploadTicket(ctx context.Context, path string, param *soap.Upload) (*url.URL, *soap.Upload, error) {
	p := soap.DefaultUpload
	if param != nil {
		p = *param // copy
	}

	u, ticket, err := d.ServiceTicket(ctx, path, p.Method)
	if err != nil {
		return nil, nil, err
	}

	p.Ticket = ticket

	return u, &p, nil
}

func (d Datastore) downloadTicket(ctx context.Context, path string, param *soap.Download) (*url.URL, *soap.Download, error) {
	p := soap.DefaultDownload
	if param != nil {
		p = *param // copy
	}

	u, ticket, err := d.ServiceTicket(ctx, path, p.Method)
	if err != nil {
		return nil, nil, err
	}

	p.Ticket = ticket

	return u, &p, nil
}

// Upload via soap.Upload with an http service ticket
func (d Datastore) Upload(ctx context.Context, f io.Reader, path string, param *soap.Upload) error {
	u, p, err := d.uploadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().Upload(f, u, p)
}

// UploadFile via soap.Upload with an http service ticket
func (d Datastore) UploadFile(ctx context.Context, file string, path string, param *soap.Upload) error {
	u, p, err := d.uploadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().UploadFile(file, u, p)
}

// Download via soap.Download with an http service ticket
func (d Datastore) Download(ctx context.Context, path string, param *soap.Download) (io.ReadCloser, int64, error) {
	u, p, err := d.downloadTicket(ctx, path, param)
	if err != nil {
		return nil, 0, err
	}
	return d.Client().Download(u, p)
}

// DownloadFile via soap.Download with an http service ticket
func (d Datastore) DownloadFile(ctx context.Context, path string, file string, param *soap.Download) error {
	u, p, err := d.downloadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().DownloadFile(file, u, p)
}

// AttachedHosts returns hosts that have this Datastore attached, accessible and writable.
func (d Datastore) AttachedHosts(ctx context.Context) ([]*HostSystem, error) {
	var ds mo.Datastore
	var hosts []*HostSystem

	pc := property.DefaultCollector(d.Client())
	err := pc.RetrieveOne(ctx, d.Reference(), []string{"host"}, &ds)
	if err != nil {
		return nil, err
	}

	mounts := make(map[types.ManagedObjectReference]types.DatastoreHostMount)
	var refs []types.ManagedObjectReference
	for _, host := range ds.Host {
		refs = append(refs, host.Key)
		mounts[host.Key] = host
	}

	var hs []mo.HostSystem
	err = pc.Retrieve(ctx, refs, []string{"runtime.connectionState", "runtime.powerState"}, &hs)
	if err != nil {
		return nil, err
	}

	for _, host := range hs {
		if host.Runtime.ConnectionState == types.HostSystemConnectionStateConnected &&
			host.Runtime.PowerState == types.HostSystemPowerStatePoweredOn {

			mount := mounts[host.Reference()]
			info := mount.MountInfo

			if *info.Mounted && *info.Accessible && info.AccessMode == string(types.HostMountModeReadWrite) {
				hosts = append(hosts, NewHostSystem(d.Client(), mount.Key))
			}
		}
	}

	return hosts, nil
}

// AttachedHosts returns hosts that have this Datastore attached, accessible and writable and are members of the given cluster.
func (d Datastore) AttachedClusterHosts(ctx context.Context, cluster *ComputeResource) ([]*HostSystem, error) {
	var hosts []*HostSystem

	clusterHosts, err := cluster.Hosts(ctx)
	if err != nil {
		return nil, err
	}

	attachedHosts, err := d.AttachedHosts(ctx)
	if err != nil {
		return nil, err
	}

	refs := make(map[types.ManagedObjectReference]bool)
	for _, host := range attachedHosts {
		refs[host.Reference()] = true
	}

	for _, host := range clusterHosts {
		if refs[host.Reference()] {
			hosts = append(hosts, host)
		}
	}

	return hosts, nil
}

func (d Datastore) Stat(ctx context.Context, file string) (types.BaseFileInfo, error) {
	b, err := d.Browser(ctx)
	if err != nil {
		return nil, err
	}

	spec := types.HostDatastoreBrowserSearchSpec{
		Details: &types.FileQueryFlags{
			FileType:     true,
			FileSize:     true,
			Modification: true,
			FileOwner:    types.NewBool(true),
		},
		MatchPattern: []string{path.Base(file)},
	}

	dsPath := d.Path(path.Dir(file))
	task, err := b.SearchDatastore(context.TODO(), dsPath, &spec)
	if err != nil {
		return nil, err
	}

	info, err := task.WaitForResult(context.TODO(), nil)
	if err != nil {
		if info == nil || info.Error != nil {
			_, ok := info.Error.Fault.(*types.FileNotFound)
			if ok {
				// FileNotFound means the base path doesn't exist.
				return nil, DatastoreNoSuchDirectoryError{"stat", dsPath}
			}
		}

		return nil, err
	}

	res := info.Result.(types.HostDatastoreBrowserSearchResults)
	if len(res.File) == 0 {
		// File doesn't exist
		return nil, DatastoreNoSuchFileError{"stat", d.Path(file)}
	}

	return res.File[0], nil

}

// Type returns the type of file system volume.
func (d Datastore) Type(ctx context.Context) (types.HostFileSystemVolumeFileSystemType, error) {
	var mds mo.Datastore

	if err := d.Properties(ctx, d.Reference(), []string{"summary.type"}, &mds); err != nil {
		return types.HostFileSystemVolumeFileSystemType(""), err
	}
	return types.HostFileSystemVolumeFileSystemType(mds.Summary.Type), nil
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	"github.com/admpub/errors"
	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}

	onInstalled []func(ctx echo.Context) error
)

func OnInstalled(cb func(ctx echo.Context) error) {
	if cb == nil {
		return
	}
	onInstalled = append(onInstalled, cb)
}

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, installer func(string) error) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	var sqlStr string
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {

		err = ctx.MustBind(&config.DefaultConfig.DB)
		if err != nil {
			return err
		}
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
		if config.DefaultConfig.DB.Type == `sqlite` {
			config.DefaultConfig.DB.User = ``
			config.DefaultConfig.DB.Password = ``
			config.DefaultConfig.DB.Host = ``
			if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
				config.DefaultConfig.DB.Database += `.db`
			}
		}
		//连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			err = createDatabase(err)
		}
		if err != nil {
			return err
		}
		//创建数据库数据
		installer, ok := config.DBInstallers[config.DefaultConfig.DB.Type]
		if !ok {
			err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
			return err
		}

		adminUser := ctx.Form(`adminUser`)
		adminPass := ctx.Form(`adminPass`)
		adminEmail := ctx.Form(`adminEmail`)
		if len(adminUser) == 0 {
			err = ctx.E(`管理员用户名不能为空`)
			return err
		}
		if !com.IsUsername(adminUser) {
			err = ctx.E(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`)
			return err
		}
		if len(adminPass) < 8 {
			err = ctx.E(`管理员密码不能少于8个字符`)
			return err
		}
		if len(adminEmail) == 0 {
			err = ctx.E(`管理员邮箱不能为空`)
			return err
		}
		if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
			err = ctx.E(`管理员邮箱格式不正确`)
			return err
		}
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			log.Info(color.GreenString(`[installer]`), `Execute SQL file: `, sqlFile)
			err = install(ctx, sqlFile, installer)
			if err != nil {
				return err
			}
		}

		// 重新连接数据库
		log.Info(color.GreenString(`[installer]`), `Reconnect the database`)
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			return err
		}

		// 添加创始人
		m := model.NewUser(ctx)
		log.Info(color.GreenString(`[installer]`), `Create Administrator`)
		err = m.Register(adminUser, adminPass, adminEmail, ``)
		if err != nil {
			return errors.WithMessage(err, `Create Administrator`)
		}

		// 生成安全密钥
		log.Info(color.GreenString(`[installer]`), `Generate a security key`)
		config.DefaultConfig.InitSecretKey()

		// 保存数据库账号到配置文件
		log.Info(color.GreenString(`[installer]`), `Save the configuration file`)
		err = config.DefaultConfig.SaveToFile()
		if err != nil {
			return err
		}

		for _, cb := range onInstalled {
			log.Info(color.GreenString(`[installer]`), `Execute Hook: `, com.FuncName(cb))
			if err = cb(ctx); err != nil {
				return err
			}
		}

		// 生成锁文件
		log.Info(color.GreenString(`[installer]`), `Generated file: `, lockFile)
		err = config.SetInstalled(lockFile)
		if err != nil {
			return err
		}

		time.Sleep(1 * time.Second) // 等1秒

		// 启动
		log.Info(color.GreenString(`[installer]`), `Start up`)
		config.DefaultCLIConfig.RunStartup()

		// 升级
		if err := Upgrade(); err != nil {
			log.Error(err)
		}

		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		}
		if err != nil {
			goto DIE
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/fatih/color"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	"github.com/admpub/errors"
	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
	"github.com/admpub/nging/application/registry/settings"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}

	onInstalled []func(ctx echo.Context) error
)

func OnInstalled(cb func(ctx echo.Context) error) {
	if cb == nil {
		return
	}
	onInstalled = append(onInstalled, cb)
}

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, installer func(string) error) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	var sqlStr string
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {

		err = ctx.MustBind(&config.DefaultConfig.DB)
		if err != nil {
			return err
		}
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
		config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
		if config.DefaultConfig.DB.Type == `sqlite` {
			config.DefaultConfig.DB.User = ``
			config.DefaultConfig.DB.Password = ``
			config.DefaultConfig.DB.Host = ``
			if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
				config.DefaultConfig.DB.Database += `.db`
			}
		}
		//连接数据库
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			err = createDatabase(err)
		}
		if err != nil {
			return err
		}
		//创建数据库数据
		installer, ok := config.DBInstallers[config.DefaultConfig.DB.Type]
		if !ok {
			err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
			return err
		}

		adminUser := ctx.Form(`adminUser`)
		adminPass := ctx.Form(`adminPass`)
		adminEmail := ctx.Form(`adminEmail`)
		if len(adminUser) == 0 {
			err = ctx.E(`管理员用户名不能为空`)
			return err
		}
		if !com.IsUsername(adminUser) {
			err = ctx.E(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`)
			return err
		}
		if len(adminPass) < 8 {
			err = ctx.E(`管理员密码不能少于8个字符`)
			return err
		}
		if len(adminEmail) == 0 {
			err = ctx.E(`管理员邮箱不能为空`)
			return err
		}
		if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
			err = ctx.E(`管理员邮箱格式不正确`)
			return err
		}
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			log.Info(color.GreenString(`[installer]`), `Execute SQL file: `, sqlFile)
			err = install(ctx, sqlFile, installer)
			if err != nil {
				return err
			}
		}

		// 重新连接数据库
		log.Info(color.GreenString(`[installer]`), `Reconnect the database`)
		err = config.ConnectDB(config.DefaultConfig)
		if err != nil {
			return err
		}

		// 添加创始人
		m := model.NewUser(ctx)
		log.Info(color.GreenString(`[installer]`), `Create Administrator`)
		err = m.Register(adminUser, adminPass, adminEmail, ``)
		if err != nil {
			return errors.WithMessage(err, `Create Administrator`)
		}

		// 生成安全密钥
		log.Info(color.GreenString(`[installer]`), `Generate a security key`)
		config.DefaultConfig.InitSecretKey()

		// 保存数据库账号到配置文件
		log.Info(color.GreenString(`[installer]`), `Save the configuration file`)
		err = config.DefaultConfig.SaveToFile()
		if err != nil {
			return err
		}

		for _, cb := range onInstalled {
			log.Info(color.GreenString(`[installer]`), `Execute Hook: `, com.FuncName(cb))
			if err = cb(ctx); err != nil {
				return err
			}
		}

		// 生成锁文件
		log.Info(color.GreenString(`[installer]`), `Generated file: `, lockFile)
		err = config.SetInstalled(lockFile)
		if err != nil {
			return err
		}

		time.Sleep(1 * time.Second) // 等1秒
		settings.Init()

		// 启动
		log.Info(color.GreenString(`[installer]`), `Start up`)
		config.DefaultCLIConfig.RunStartup()

		// 升级
		if err := Upgrade(); err != nil {
			log.Error(err)
		}

		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		}
		if err != nil {
			goto DIE
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<|endoftext|>


<commit_before>package stringutil

import (
	"strings"
)

func StringAfter(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[pos+len(find):]
}

func StringBefore(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[:pos]
}

func Trim(content string) string {
	runes := []rune(content)
	if len(runes) > 0 {
		if trimableChar(runes[0]) {
			return Trim(content[1:])
		}
		if trimableChar(runes[len(content)-1]) {
			return Trim(content[:len(content)-1])
		}
	}
	return content
}

func trimableChar(c rune) bool {
	return c == ' ' || c == '\n' || c == '\r'
}

func StringLess(a, b string) bool {
	for i := 0; i < len(a) && i < len(b); i++ {
		if a[i] < b[i] {
			return true
		}
		if a[i] > b[i] {
			return false
		}
	}
	return len(a) < len(b)
}
<commit_msg>update readme<commit_after>package Stringutil

import (
	"strings"
)

func StringAfter(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[pos+len(find):]
}

func StringBefore(content string, find string) string {
	if len(find) == 0 {
		return content
	}
	pos := strings.Index(content, find)
	if pos == -1 {
		return ""
	}
	return content[:pos]
}

func Trim(content string) string {
	runes := []rune(content)
	if len(runes) > 0 {
		if trimableChar(runes[0]) {
			return Trim(content[1:])
		}
		if trimableChar(runes[len(content)-1]) {
			return Trim(content[:len(content)-1])
		}
	}
	return content
}

func trimableChar(c rune) bool {
	return c == ' ' || c == '\n' || c == '\r'
}

func StringLess(a, b string) bool {
	for i := 0; i < len(a) && i < len(b); i++ {
		if a[i] < b[i] {
			return true
		}
		if a[i] > b[i] {
			return false
		}
	}
	return len(a) < len(b)
}
<|endoftext|>