<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/smtp"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func main() {
	sourceRoot := os.Args[1]
	if sourceRoot == "" {
		fmt.Fprintf(os.Stderr, "expected path to build sources as first argument")
		os.Exit(1)
	}

	var buildTokens = map[string]string{
		"${BUILD_ID}":            os.Getenv("BUILD_ID"),
		"${BUILD_NAME}":          os.Getenv("BUILD_NAME"),
		"${BUILD_JOB_NAME}":      os.Getenv("BUILD_JOB_NAME"),
		"${BUILD_PIPELINE_NAME}": os.Getenv("BUILD_PIPELINE_NAME"),
		"${ATC_EXTERNAL_URL}":    os.Getenv("ATC_EXTERNAL_URL"),
	}

	var indata struct {
		Source struct {
			SMTP struct {
				Host      string
				Port      string
				Username  string
				Password  string
				Anonymous bool `json:"anonymous"`
			}
			From string
			To   []string
		}
		Params struct {
			Subject       string
			Body          string
			SendEmptyBody bool `json:"send_empty_body"`
			Headers       string
			To            string `json:"to"`
		}
	}

	inbytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}

	err = json.Unmarshal(inbytes, &indata)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error parsing input as JSON: %s", err)
		os.Exit(1)
	}

	if indata.Source.SMTP.Host == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.smtp.host"`)
		os.Exit(1)
	}

	if indata.Source.SMTP.Port == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.smtp.port"`)
		os.Exit(1)
	}

	if indata.Source.From == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.from"`)
		os.Exit(1)
	}

	if len(indata.Source.To) == 0 && len(indata.Params.To) == 0 {
		fmt.Fprintf(os.Stderr, `missing required field "source.to" or "params.to". Must specify at least one`)
		os.Exit(1)
	}

	if indata.Params.Subject == "" {
		fmt.Fprintf(os.Stderr, `missing required field "params.subject"`)
		os.Exit(1)
	}

	if indata.Source.SMTP.Anonymous == false {
		if indata.Source.SMTP.Username == "" {
			fmt.Fprintf(os.Stderr, `missing required field "source.smtp.username" if anonymous specify anonymous: true`)
			os.Exit(1)
		}

		if indata.Source.SMTP.Password == "" {
			fmt.Fprintf(os.Stderr, `missing required field "source.smtp.password" if anonymous specify anonymous: true`)
			os.Exit(1)
		}
	}

	replaceTokens := func(sourceString string) string {
		for k, v := range buildTokens {
			sourceString = strings.Replace(sourceString, k, v, -1)
		}
		return sourceString
	}

	readSource := func(sourcePath string) (string, error) {
		if !filepath.IsAbs(sourcePath) {
			sourcePath = filepath.Join(sourceRoot, sourcePath)
		}
		var bytes []byte
		bytes, err = ioutil.ReadFile(sourcePath)
		return replaceTokens(string(bytes)), err
	}

	subject, err := readSource(indata.Params.Subject)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	subject = strings.Trim(subject, "\n")

	var headers string
	if indata.Params.Headers != "" {
		headers, err = readSource(indata.Params.Headers)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
		headers = strings.Trim(headers, "\n")
	}

	var body string
	if indata.Params.Body != "" {
		body, err = readSource(indata.Params.Body)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
	}

	if indata.Params.To != "" {
		var toList string
		toList, err = readSource(indata.Params.To)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
		if len(toList) > 0 {
			toListArray := strings.Split(toList, ",")
			for _, toAddress := range toListArray {
				indata.Source.To = append(indata.Source.To, strings.TrimSpace(toAddress))
			}
		}
	}

	type MetadataItem struct {
		Name  string
		Value string
	}
	var outdata struct {
		Version struct {
			Time time.Time
		} `json:"version"`
		Metadata []MetadataItem
	}
	outdata.Version.Time = time.Now().UTC()
	outdata.Metadata = []MetadataItem{
		{Name: "smtp_host", Value: indata.Source.SMTP.Host},
		{Name: "subject", Value: subject},
	}
	outbytes, err := json.Marshal(outdata)
	if err != nil {
		panic(err)
	}

	var messageData []byte
	messageData = append(messageData, []byte("To: "+strings.Join(indata.Source.To, ", ")+"\n")...)
	messageData = append(messageData, []byte("From: "+indata.Source.From+"\n")...)
	if headers != "" {
		messageData = append(messageData, []byte(headers+"\n")...)
	}
	messageData = append(messageData, []byte("Subject: "+subject+"\n")...)

	messageData = append(messageData, []byte("\n")...)
	messageData = append(messageData, []byte(body)...)

	if indata.Params.SendEmptyBody == false && len(body) == 0 {
		fmt.Fprintf(os.Stderr, "Message not sent because the message body is empty and send_empty_body parameter was set to false. Github readme: https://github.com/pivotal-cf/email-resource")
		fmt.Printf("%s", []byte(outbytes))
		return
	}

	if indata.Source.SMTP.Anonymous {
		var c *smtp.Client
		var wc io.WriteCloser
		c, err = smtp.Dial(fmt.Sprintf("%s:%s", indata.Source.SMTP.Host, indata.Source.SMTP.Port))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Dialing: "+err.Error())
			os.Exit(1)
		}
		defer c.Close()
		c.Mail(indata.Source.From)

		for _, toAddress := range indata.Source.To {
			c.Rcpt(toAddress)
		}
		// Send the email body.
		wc, err = c.Data()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Getting writer context: "+err.Error())
			os.Exit(1)
		}
		defer wc.Close()
		_, err = wc.Write(messageData)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Writing bytes: "+err.Error())
			os.Exit(1)
		}

	} else {
		err = smtp.SendMail(
			fmt.Sprintf("%s:%s", indata.Source.SMTP.Host, indata.Source.SMTP.Port),
			smtp.PlainAuth(
				"",
				indata.Source.SMTP.Username,
				indata.Source.SMTP.Password,
				indata.Source.SMTP.Host,
			),
			indata.Source.From,
			indata.Source.To,
			messageData,
		)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to send an email using SMTP server %s on port %s: %v",
			indata.Source.SMTP.Host, indata.Source.SMTP.Port, err)
		os.Exit(1)
	}

	fmt.Printf("%s", []byte(outbytes))
}
<commit_msg>update to add version to out<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/smtp"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var (
	//VERSION -
	VERSION string
)

func main() {
	sourceRoot := os.Args[1]
	if sourceRoot == "" {
		fmt.Fprintf(os.Stderr, "expected path to build sources as first argument")
		os.Exit(1)
	}

	var buildTokens = map[string]string{
		"${BUILD_ID}":            os.Getenv("BUILD_ID"),
		"${BUILD_NAME}":          os.Getenv("BUILD_NAME"),
		"${BUILD_JOB_NAME}":      os.Getenv("BUILD_JOB_NAME"),
		"${BUILD_PIPELINE_NAME}": os.Getenv("BUILD_PIPELINE_NAME"),
		"${ATC_EXTERNAL_URL}":    os.Getenv("ATC_EXTERNAL_URL"),
	}

	var indata struct {
		Source struct {
			SMTP struct {
				Host      string
				Port      string
				Username  string
				Password  string
				Anonymous bool `json:"anonymous"`
			}
			From string
			To   []string
		}
		Params struct {
			Subject       string
			Body          string
			SendEmptyBody bool `json:"send_empty_body"`
			Headers       string
			To            string `json:"to"`
		}
	}

	inbytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}

	err = json.Unmarshal(inbytes, &indata)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error parsing input as JSON: %s", err)
		os.Exit(1)
	}

	if indata.Source.SMTP.Host == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.smtp.host"`)
		os.Exit(1)
	}

	if indata.Source.SMTP.Port == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.smtp.port"`)
		os.Exit(1)
	}

	if indata.Source.From == "" {
		fmt.Fprintf(os.Stderr, `missing required field "source.from"`)
		os.Exit(1)
	}

	if len(indata.Source.To) == 0 && len(indata.Params.To) == 0 {
		fmt.Fprintf(os.Stderr, `missing required field "source.to" or "params.to". Must specify at least one`)
		os.Exit(1)
	}

	if indata.Params.Subject == "" {
		fmt.Fprintf(os.Stderr, `missing required field "params.subject"`)
		os.Exit(1)
	}

	if indata.Source.SMTP.Anonymous == false {
		if indata.Source.SMTP.Username == "" {
			fmt.Fprintf(os.Stderr, `missing required field "source.smtp.username" if anonymous specify anonymous: true`)
			os.Exit(1)
		}

		if indata.Source.SMTP.Password == "" {
			fmt.Fprintf(os.Stderr, `missing required field "source.smtp.password" if anonymous specify anonymous: true`)
			os.Exit(1)
		}
	}

	replaceTokens := func(sourceString string) string {
		for k, v := range buildTokens {
			sourceString = strings.Replace(sourceString, k, v, -1)
		}
		return sourceString
	}

	readSource := func(sourcePath string) (string, error) {
		if !filepath.IsAbs(sourcePath) {
			sourcePath = filepath.Join(sourceRoot, sourcePath)
		}
		var bytes []byte
		bytes, err = ioutil.ReadFile(sourcePath)
		return replaceTokens(string(bytes)), err
	}

	subject, err := readSource(indata.Params.Subject)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	subject = strings.Trim(subject, "\n")

	var headers string
	if indata.Params.Headers != "" {
		headers, err = readSource(indata.Params.Headers)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
		headers = strings.Trim(headers, "\n")
	}

	var body string
	if indata.Params.Body != "" {
		body, err = readSource(indata.Params.Body)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
	}

	if indata.Params.To != "" {
		var toList string
		toList, err = readSource(indata.Params.To)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(1)
		}
		if len(toList) > 0 {
			toListArray := strings.Split(toList, ",")
			for _, toAddress := range toListArray {
				indata.Source.To = append(indata.Source.To, strings.TrimSpace(toAddress))
			}
		}
	}

	type MetadataItem struct {
		Name  string
		Value string
	}
	var outdata struct {
		Version struct {
			Time time.Time
		} `json:"version"`
		Metadata []MetadataItem
	}
	outdata.Version.Time = time.Now().UTC()
	outdata.Metadata = []MetadataItem{
		{Name: "smtp_host", Value: indata.Source.SMTP.Host},
		{Name: "subject", Value: subject},
		{Name: "version", Value: VERSION},
	}
	outbytes, err := json.Marshal(outdata)
	if err != nil {
		panic(err)
	}

	var messageData []byte
	messageData = append(messageData, []byte("To: "+strings.Join(indata.Source.To, ", ")+"\n")...)
	messageData = append(messageData, []byte("From: "+indata.Source.From+"\n")...)
	if headers != "" {
		messageData = append(messageData, []byte(headers+"\n")...)
	}
	messageData = append(messageData, []byte("Subject: "+subject+"\n")...)

	messageData = append(messageData, []byte("\n")...)
	messageData = append(messageData, []byte(body)...)

	if indata.Params.SendEmptyBody == false && len(body) == 0 {
		fmt.Fprintf(os.Stderr, "Message not sent because the message body is empty and send_empty_body parameter was set to false. Github readme: https://github.com/pivotal-cf/email-resource")
		fmt.Printf("%s", []byte(outbytes))
		return
	}

	if indata.Source.SMTP.Anonymous {
		var c *smtp.Client
		var wc io.WriteCloser
		c, err = smtp.Dial(fmt.Sprintf("%s:%s", indata.Source.SMTP.Host, indata.Source.SMTP.Port))
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Dialing: "+err.Error())
			os.Exit(1)
		}
		defer c.Close()
		c.Mail(indata.Source.From)

		for _, toAddress := range indata.Source.To {
			c.Rcpt(toAddress)
		}
		// Send the email body.
		wc, err = c.Data()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Getting writer context: "+err.Error())
			os.Exit(1)
		}
		defer wc.Close()
		_, err = wc.Write(messageData)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Writing bytes: "+err.Error())
			os.Exit(1)
		}

	} else {
		err = smtp.SendMail(
			fmt.Sprintf("%s:%s", indata.Source.SMTP.Host, indata.Source.SMTP.Port),
			smtp.PlainAuth(
				"",
				indata.Source.SMTP.Username,
				indata.Source.SMTP.Password,
				indata.Source.SMTP.Host,
			),
			indata.Source.From,
			indata.Source.To,
			messageData,
		)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to send an email using SMTP server %s on port %s: %v",
			indata.Source.SMTP.Host, indata.Source.SMTP.Port, err)
		os.Exit(1)
	}

	fmt.Printf("%s", []byte(outbytes))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/gogap/spirit"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"
	"time"
)

var (
	ErrNoURNPackageSourceFound = errors.New("no urn packages source found")
	ErrConfigFileNameIsEmpty   = errors.New("config file name is empty")
)

type SpiritHelper struct {
	conf           spirit.SpiritConfig
	configFile     string
	configFileName string
	originalConfig []byte

	RefURNs     []string
	RefPackages []Package
}

func (p *SpiritHelper) LoadSpiritConfig(filename string) (err error) {

	if filename == "" {
		err = ErrConfigFileNameIsEmpty
		return
	}

	if fi, e := os.Stat(filename); e != nil {
		err = e
		return
	} else {
		p.configFile = filename
		p.configFileName = fi.Name()
	}

	if p.originalConfig, err = ioutil.ReadFile(filename); err != nil {
		return
	}

	if err = json.Unmarshal(p.originalConfig, &p.conf); err != nil {
		return
	}

	return
}

func (p *SpiritHelper) CreateProject(createOpts CreateOptions, tmplArgs map[string]interface{}) (err error) {
	if err = createOpts.Validate(); err != nil {
		return
	}

	goSrc := path.Join(createOpts.GoPath, "src")

	if err = p.parse(goSrc, createOpts.Sources); err != nil {
		return
	}

	// download packages
	if createOpts.GetPackages {
		if err = p.GetPackages(createOpts.PackagesRevision, createOpts.UpdatePackages); err != nil {
			return
		}
	}

	// make project dir
	projectPath := path.Join(goSrc, createOpts.ProjectPath)
	if path.IsAbs(createOpts.ProjectPath) {
		projectPath = createOpts.ProjectPath
	}

	if fi, e := os.Stat(projectPath); e != nil {
		if !strings.Contains(e.Error(), "no such file or directory") &&
			!os.IsNotExist(e) {
			err = e
			return
		}
	} else if !fi.IsDir() {
		err = fmt.Errorf("your project path %s already exist, but it is not a directory", projectPath)
		return
	} else if createOpts.ForceWrite {
		spirit.Logger().Warnf("project path %s already exist, it will be overwrite", projectPath)
	} else {
		err = fmt.Errorf("your project path %s already exist", projectPath)
		return
	}

	if err = os.MkdirAll(projectPath, os.FileMode(0755)); err != nil {
		if !os.IsNotExist(err) {
			return
		} else if !createOpts.ForceWrite {
			return
		}
		err = nil
	}

	// render code template
	tmplPathFmt := "github.com/gogap/spirit-tool/template/%s/main.go"
	tmplArgsPathFmt := "github.com/gogap/spirit-tool/template/%s/args.json"

	tmplPath := path.Join(goSrc, fmt.Sprintf(tmplPathFmt, createOpts.TemplateName))
	spirit.Logger().Infof("using template of %s: %s", createOpts.TemplateName, tmplPath)

	tmplArgsPath := path.Join(goSrc, fmt.Sprintf(tmplArgsPathFmt, createOpts.TemplateName))
	spirit.Logger().Infof("using template args of %s: %s", createOpts.TemplateName, tmplArgsPath)

	var tmpl *template.Template
	if tmpl, err = template.New("main.go").Option("missingkey=error").Delims("//<-", "->//").ParseFiles(tmplPath); err != nil {
		return
	}

	internalArgs := map[string]interface{}{}
	if argData, e := ioutil.ReadFile(tmplArgsPath); e == nil {
		if err = json.Unmarshal(argData, &internalArgs); err != nil {
			return
		}
	}

	if tmplArgs != nil {
		for k, v := range tmplArgs {
			internalArgs[k] = v
		}
	}

	buffer := &bytes.Buffer{}
	if err = tmpl.Execute(buffer, map[string]interface{}{
		"create_options":  createOpts,
		"packages":        p.RefPackages,
		"config":          p.configFile,
		"config_filename": p.configFileName,
		"create_time":     time.Now(),
		"args":            internalArgs}); err != nil {
		return
	}

	srcPath := path.Join(projectPath, "main.go")
	if err = ioutil.WriteFile(srcPath, buffer.Bytes(), os.FileMode(0644)); err != nil {
		return
	}

	confPath := path.Join(projectPath, p.configFileName)
	if err = ioutil.WriteFile(confPath, p.originalConfig, os.FileMode(0644)); err != nil {
		return
	}

	// format code for sort import packages order
	if _, err = execCommand("go fmt " + srcPath); err != nil {
		return
	}

	spirit.Logger().Infof("project created at %s\n", projectPath)

	return
}

func (p *SpiritHelper) GetPackages(pkgRevision map[string]string, update bool) (err error) {
	for _, pkg := range p.RefPackages {
		if pkgRevision != nil {
			if revision, exist := pkgRevision[pkg.URI]; exist {
				pkg.Revision = revision
			}
		}
		if err = pkg.Get(update); err != nil {
			return
		}
	}
	return
}

func (p *SpiritHelper) RunProject(createOpts CreateOptions, detach bool, envs []string, tmplArgs map[string]interface{}) (err error) {

	if err = p.CreateProject(createOpts, tmplArgs); err != nil {
		return
	}

	if _, err = execCommandWithDir("go build -o main "+path.Join(createOpts.ProjectPath, "main.go"), createOpts.ProjectPath); err != nil {
		return
	}

	if cmder, e := execute(path.Join(createOpts.ProjectPath, "main"), createOpts.ProjectPath, !detach, envs); e != nil {
		err = e
		return
	} else if !detach {
		cmder.Wait()
	}

	return
}

func (p *SpiritHelper) parse(gosrc string, sources []string) (err error) {
	if sources == nil || len(sources) == 0 {
		err = ErrNoURNPackageSourceFound
		return
	}

	var urns []string

	if urns = parseActorsUsingURN(
		p.conf.InputTranslators,
		p.conf.OutputTranslators,
		p.conf.Inboxes,
		p.conf.Outboxes,
		p.conf.Receivers,
		p.conf.Senders,
		p.conf.Routers,
		p.conf.Components,
		p.conf.LabelMatchers,
		p.conf.URNRewriters,
	); err != nil {
		return
	}

	for _, readerPool := range p.conf.ReaderPools {
		urns = append(urns, parseActorUsingURN(readerPool.ActorConfig)...)
		if readerPool.Reader != nil {
			urns = append(urns, parseActorUsingURN(*readerPool.Reader)...)
		}
	}

	for _, writerPool := range p.conf.WriterPools {
		urns = append(urns, parseActorUsingURN(writerPool.ActorConfig)...)
		if writerPool.Writer != nil {
			urns = append(urns, parseActorUsingURN(*writerPool.Writer)...)
		}
	}

	p.RefURNs = urns

	if p.RefPackages, err = urnsToPackages(gosrc, urns, sources...); err != nil {
		return
	}

	return
}

func parseActorsUsingURN(confs ...[]spirit.ActorConfig) (urns []string) {
	for _, conf := range confs {
		for _, c := range conf {
			urns = append(urns, c.URN)
		}
	}
	return
}

func parseActorUsingURN(actorConfs ...spirit.ActorConfig) (urns []string) {
	for _, conf := range actorConfs {
		urns = append(urns, conf.URN)
	}
	return
}

func urnsToPackages(gosrc string, urns []string, sourceFiles ...string) (packages []Package, err error) {
	urnPkgMap := map[string]string{}

	for _, sourceFile := range sourceFiles {
		var data []byte

		if data, err = ioutil.ReadFile(sourceFile); err != nil {
			return
		}

		sourceConf := SourceConfig{}
		if err = json.Unmarshal(data, &sourceConf); err != nil {
			return
		}

		for _, urnPkg := range sourceConf.Packages {
			if oldVal, exist := urnPkgMap[urnPkg.URN]; exist {
				if oldVal != urnPkg.Pkg {
					err = fmt.Errorf("source have duplicate urn pkg, urn:%s, pkg1:%s, pkg2: %s, file: %s", urnPkg.URN, oldVal, urnPkg.Pkg, sourceFile)
					return
				}
			}
			urnPkgMap[urnPkg.URN] = urnPkg.Pkg
		}
	}

	pkgs := map[string]bool{}

	for _, urn := range urns {
		if pkg, exist := urnPkgMap[urn]; !exist {
			err = fmt.Errorf("no package from any source of urn: %s", urn)
			return
		} else {
			pkgs[pkg] = true
		}
	}

	for pkg, _ := range pkgs {
		packages = append(packages, Package{gosrc: gosrc, URI: pkg, Revision: ""})
	}

	return
}
<commit_msg>print pid while run detach<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/gogap/spirit"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"
	"time"
)

var (
	ErrNoURNPackageSourceFound = errors.New("no urn packages source found")
	ErrConfigFileNameIsEmpty   = errors.New("config file name is empty")
)

type SpiritHelper struct {
	conf           spirit.SpiritConfig
	configFile     string
	configFileName string
	originalConfig []byte

	RefURNs     []string
	RefPackages []Package
}

func (p *SpiritHelper) LoadSpiritConfig(filename string) (err error) {

	if filename == "" {
		err = ErrConfigFileNameIsEmpty
		return
	}

	if fi, e := os.Stat(filename); e != nil {
		err = e
		return
	} else {
		p.configFile = filename
		p.configFileName = fi.Name()
	}

	if p.originalConfig, err = ioutil.ReadFile(filename); err != nil {
		return
	}

	if err = json.Unmarshal(p.originalConfig, &p.conf); err != nil {
		return
	}

	return
}

func (p *SpiritHelper) CreateProject(createOpts CreateOptions, tmplArgs map[string]interface{}) (err error) {
	if err = createOpts.Validate(); err != nil {
		return
	}

	goSrc := path.Join(createOpts.GoPath, "src")

	if err = p.parse(goSrc, createOpts.Sources); err != nil {
		return
	}

	// download packages
	if createOpts.GetPackages {
		if err = p.GetPackages(createOpts.PackagesRevision, createOpts.UpdatePackages); err != nil {
			return
		}
	}

	// make project dir
	projectPath := path.Join(goSrc, createOpts.ProjectPath)
	if path.IsAbs(createOpts.ProjectPath) {
		projectPath = createOpts.ProjectPath
	}

	if fi, e := os.Stat(projectPath); e != nil {
		if !strings.Contains(e.Error(), "no such file or directory") &&
			!os.IsNotExist(e) {
			err = e
			return
		}
	} else if !fi.IsDir() {
		err = fmt.Errorf("your project path %s already exist, but it is not a directory", projectPath)
		return
	} else if createOpts.ForceWrite {
		spirit.Logger().Warnf("project path %s already exist, it will be overwrite", projectPath)
	} else {
		err = fmt.Errorf("your project path %s already exist", projectPath)
		return
	}

	if err = os.MkdirAll(projectPath, os.FileMode(0755)); err != nil {
		if !os.IsNotExist(err) {
			return
		} else if !createOpts.ForceWrite {
			return
		}
		err = nil
	}

	// render code template
	tmplPathFmt := "github.com/gogap/spirit-tool/template/%s/main.go"
	tmplArgsPathFmt := "github.com/gogap/spirit-tool/template/%s/args.json"

	tmplPath := path.Join(goSrc, fmt.Sprintf(tmplPathFmt, createOpts.TemplateName))
	spirit.Logger().Infof("using template of %s: %s", createOpts.TemplateName, tmplPath)

	tmplArgsPath := path.Join(goSrc, fmt.Sprintf(tmplArgsPathFmt, createOpts.TemplateName))
	spirit.Logger().Infof("using template args of %s: %s", createOpts.TemplateName, tmplArgsPath)

	var tmpl *template.Template
	if tmpl, err = template.New("main.go").Option("missingkey=error").Delims("//<-", "->//").ParseFiles(tmplPath); err != nil {
		return
	}

	internalArgs := map[string]interface{}{}
	if argData, e := ioutil.ReadFile(tmplArgsPath); e == nil {
		if err = json.Unmarshal(argData, &internalArgs); err != nil {
			return
		}
	}

	if tmplArgs != nil {
		for k, v := range tmplArgs {
			internalArgs[k] = v
		}
	}

	buffer := &bytes.Buffer{}
	if err = tmpl.Execute(buffer, map[string]interface{}{
		"create_options":  createOpts,
		"packages":        p.RefPackages,
		"config":          p.configFile,
		"config_filename": p.configFileName,
		"create_time":     time.Now(),
		"args":            internalArgs}); err != nil {
		return
	}

	srcPath := path.Join(projectPath, "main.go")
	if err = ioutil.WriteFile(srcPath, buffer.Bytes(), os.FileMode(0644)); err != nil {
		return
	}

	confPath := path.Join(projectPath, p.configFileName)
	if err = ioutil.WriteFile(confPath, p.originalConfig, os.FileMode(0644)); err != nil {
		return
	}

	// format code for sort import packages order
	if _, err = execCommand("go fmt " + srcPath); err != nil {
		return
	}

	spirit.Logger().Infof("project created at %s\n", projectPath)

	return
}

func (p *SpiritHelper) GetPackages(pkgRevision map[string]string, update bool) (err error) {
	for _, pkg := range p.RefPackages {
		if pkgRevision != nil {
			if revision, exist := pkgRevision[pkg.URI]; exist {
				pkg.Revision = revision
			}
		}
		if err = pkg.Get(update); err != nil {
			return
		}
	}
	return
}

func (p *SpiritHelper) RunProject(createOpts CreateOptions, detach bool, envs []string, tmplArgs map[string]interface{}) (err error) {

	if err = p.CreateProject(createOpts, tmplArgs); err != nil {
		return
	}

	if _, err = execCommandWithDir("go build -o main "+path.Join(createOpts.ProjectPath, "main.go"), createOpts.ProjectPath); err != nil {
		return
	}

	if cmder, e := execute(path.Join(createOpts.ProjectPath, "main"), createOpts.ProjectPath, !detach, envs); e != nil {
		err = e
		return
	} else if !detach {
		cmder.Wait()
	} else {
		spirit.Logger().Infof("PID: %d\n", cmder.Process.Pid)
	}

	return
}

func (p *SpiritHelper) parse(gosrc string, sources []string) (err error) {
	if sources == nil || len(sources) == 0 {
		err = ErrNoURNPackageSourceFound
		return
	}

	var urns []string

	if urns = parseActorsUsingURN(
		p.conf.InputTranslators,
		p.conf.OutputTranslators,
		p.conf.Inboxes,
		p.conf.Outboxes,
		p.conf.Receivers,
		p.conf.Senders,
		p.conf.Routers,
		p.conf.Components,
		p.conf.LabelMatchers,
		p.conf.URNRewriters,
	); err != nil {
		return
	}

	for _, readerPool := range p.conf.ReaderPools {
		urns = append(urns, parseActorUsingURN(readerPool.ActorConfig)...)
		if readerPool.Reader != nil {
			urns = append(urns, parseActorUsingURN(*readerPool.Reader)...)
		}
	}

	for _, writerPool := range p.conf.WriterPools {
		urns = append(urns, parseActorUsingURN(writerPool.ActorConfig)...)
		if writerPool.Writer != nil {
			urns = append(urns, parseActorUsingURN(*writerPool.Writer)...)
		}
	}

	p.RefURNs = urns

	if p.RefPackages, err = urnsToPackages(gosrc, urns, sources...); err != nil {
		return
	}

	return
}

func parseActorsUsingURN(confs ...[]spirit.ActorConfig) (urns []string) {
	for _, conf := range confs {
		for _, c := range conf {
			urns = append(urns, c.URN)
		}
	}
	return
}

func parseActorUsingURN(actorConfs ...spirit.ActorConfig) (urns []string) {
	for _, conf := range actorConfs {
		urns = append(urns, conf.URN)
	}
	return
}

func urnsToPackages(gosrc string, urns []string, sourceFiles ...string) (packages []Package, err error) {
	urnPkgMap := map[string]string{}

	for _, sourceFile := range sourceFiles {
		var data []byte

		if data, err = ioutil.ReadFile(sourceFile); err != nil {
			return
		}

		sourceConf := SourceConfig{}
		if err = json.Unmarshal(data, &sourceConf); err != nil {
			return
		}

		for _, urnPkg := range sourceConf.Packages {
			if oldVal, exist := urnPkgMap[urnPkg.URN]; exist {
				if oldVal != urnPkg.Pkg {
					err = fmt.Errorf("source have duplicate urn pkg, urn:%s, pkg1:%s, pkg2: %s, file: %s", urnPkg.URN, oldVal, urnPkg.Pkg, sourceFile)
					return
				}
			}
			urnPkgMap[urnPkg.URN] = urnPkg.Pkg
		}
	}

	pkgs := map[string]bool{}

	for _, urn := range urns {
		if pkg, exist := urnPkgMap[urn]; !exist {
			err = fmt.Errorf("no package from any source of urn: %s", urn)
			return
		} else {
			pkgs[pkg] = true
		}
	}

	for pkg, _ := range pkgs {
		packages = append(packages, Package{gosrc: gosrc, URI: pkg, Revision: ""})
	}

	return
}
<|endoftext|>


<commit_before>package doarama_test

import (
	"bytes"
	"testing"
	"time"

	"github.com/twpayne/go-doarama"
)

func TestWrite(t *testing.T) {
	for _, tc := range []struct {
		samples []doarama.Sample
		wantGPX string
		wantIGC string
	}{
		{
			samples: []doarama.Sample{
				doarama.Sample{
					Time: doarama.NewTimestamp(time.Date(2015, 7, 5, 9, 30, 0, 0, time.UTC)),
					Coords: doarama.Coords{
						Latitude:  47.79885,
						Longitude: 13.04840,
						Altitude:  430,
					},
				},
				doarama.Sample{
					Time: doarama.NewTimestamp(time.Date(2015, 7, 5, 11, 15, 0, 0, time.UTC)),
					Coords: doarama.Coords{
						Latitude:  47.80413,
						Longitude: 13.11091,
						Altitude:  1272,
					},
				},
			},
			wantGPX: "" +
				"<gpx version=\"1.1\" creator=\"https://github.com/twpayne/go-doarama\">" +
				"<trk>" +
				"<trkseg>" +
				"<trkpt lat=\"47.798850\" lon=\"13.048400\">" +
				"<ele>430.000000</ele>" +
				"<time>2015-07-05T09:30:00Z</time>" +
				"</trkpt>" +
				"<trkpt lat=\"47.804130\" lon=\"13.110910\">" +
				"<ele>1272.000000</ele>" +
				"<time>2015-07-05T11:15:00Z</time>" +
				"</trkpt>" +
				"</trkseg>" +
				"</trk>" +
				"</gpx>",
			wantIGC: "" +
				"HFDTE050715\r\n" +
				"B0930004747931N01302904EA0043000430\r\n" +
				"B1115004748247N01306654EA0127201272\r\n",
		},
	} {
		bGPX := &bytes.Buffer{}
		if err := doarama.WriteGPX(bGPX, tc.samples); err != nil {
			t.Errorf("doarama.WriteGPX(b, %#v) == %v, want nil", tc.samples, err)
		}
		if bGPX.String() != tc.wantGPX {
			t.Errorf("doarama.WriteGPX(b, %#v) wrote %#v, want %#v", tc.samples, bGPX.String(), tc.wantGPX)
		}
		bIGC := &bytes.Buffer{}
		if err := doarama.WriteIGC(bIGC, tc.samples); err != nil {
			t.Errorf("doarama.WriteIGC(b, %#v) == %v, want nil", tc.samples, err)
		}
		if bIGC.String() != tc.wantIGC {
			t.Errorf("doarama.WriteIGC(b, %#v) wrote %#v, want %#v", tc.samples, bIGC.String(), tc.wantIGC)
		}
	}
}
<commit_msg>Simplify formatting<commit_after>package doarama_test

import (
	"bytes"
	"testing"
	"time"

	"github.com/twpayne/go-doarama"
)

func TestWrite(t *testing.T) {
	for _, tc := range []struct {
		samples []doarama.Sample
		wantGPX string
		wantIGC string
	}{
		{
			samples: []doarama.Sample{
				{
					Time: doarama.NewTimestamp(time.Date(2015, 7, 5, 9, 30, 0, 0, time.UTC)),
					Coords: doarama.Coords{
						Latitude:  47.79885,
						Longitude: 13.04840,
						Altitude:  430,
					},
				},
				{
					Time: doarama.NewTimestamp(time.Date(2015, 7, 5, 11, 15, 0, 0, time.UTC)),
					Coords: doarama.Coords{
						Latitude:  47.80413,
						Longitude: 13.11091,
						Altitude:  1272,
					},
				},
			},
			wantGPX: "" +
				"<gpx version=\"1.1\" creator=\"https://github.com/twpayne/go-doarama\">" +
				"<trk>" +
				"<trkseg>" +
				"<trkpt lat=\"47.798850\" lon=\"13.048400\">" +
				"<ele>430.000000</ele>" +
				"<time>2015-07-05T09:30:00Z</time>" +
				"</trkpt>" +
				"<trkpt lat=\"47.804130\" lon=\"13.110910\">" +
				"<ele>1272.000000</ele>" +
				"<time>2015-07-05T11:15:00Z</time>" +
				"</trkpt>" +
				"</trkseg>" +
				"</trk>" +
				"</gpx>",
			wantIGC: "" +
				"HFDTE050715\r\n" +
				"B0930004747931N01302904EA0043000430\r\n" +
				"B1115004748247N01306654EA0127201272\r\n",
		},
	} {
		bGPX := &bytes.Buffer{}
		if err := doarama.WriteGPX(bGPX, tc.samples); err != nil {
			t.Errorf("doarama.WriteGPX(b, %#v) == %v, want nil", tc.samples, err)
		}
		if bGPX.String() != tc.wantGPX {
			t.Errorf("doarama.WriteGPX(b, %#v) wrote %#v, want %#v", tc.samples, bGPX.String(), tc.wantGPX)
		}
		bIGC := &bytes.Buffer{}
		if err := doarama.WriteIGC(bIGC, tc.samples); err != nil {
			t.Errorf("doarama.WriteIGC(b, %#v) == %v, want nil", tc.samples, err)
		}
		if bIGC.String() != tc.wantIGC {
			t.Errorf("doarama.WriteIGC(b, %#v) wrote %#v, want %#v", tc.samples, bIGC.String(), tc.wantIGC)
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName     = "quiet"
	silentOptionName    = "silent"
	progressOptionName  = "progress"
	trickleOptionName   = "trickle"
	wrapOptionName      = "wrap-with-directory"
	hiddenOptionName    = "hidden"
	onlyHashOptionName  = "only-hash"
	chunkerOptionName   = "chunker"
	pinOptionName       = "pin"
	rawLeavesOptionName = "raw-leaves"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
			return nil
		}

		// ipfs cli progress bar defaults to true
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()
		rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		dserv := n.DAG
		local, _, _ := req.Option("local").Bool()
		if local {
			offlineexch := offline.Exchange(n.Blockstore)
			bserv := blockservice.New(n.Blockstore, offlineexch)
			dserv = dag.NewDAGService(bserv)
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, prgFound, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		silent, _, err := req.Option(silentOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		var showProgressBar bool
		if prgFound {
			showProgressBar = progress
		} else if !quiet && !silent {
			showProgressBar = true
		}

		var bar *pb.ProgressBar
		if showProgressBar {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if showProgressBar {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !showProgressBar {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if showProgressBar {
					bar.Update()
				}
			case size := <-sizeChan:
				if showProgressBar {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<commit_msg>add cmd: clean up default logic of --progress option<commit_after>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName     = "quiet"
	silentOptionName    = "silent"
	progressOptionName  = "progress"
	trickleOptionName   = "trickle"
	wrapOptionName      = "wrap-with-directory"
	hiddenOptionName    = "hidden"
	onlyHashOptionName  = "only-hash"
	chunkerOptionName   = "chunker"
	pinOptionName       = "pin"
	rawLeavesOptionName = "raw-leaves"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		quiet, _, _ := req.Option(quietOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()

		if quiet || silent {
			return nil
		}

		// ipfs cli progress bar defaults to true unless quiet or silent is used
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()
		rawblks, _, _ := req.Option(rawLeavesOptionName).Bool()

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		dserv := n.DAG
		local, _, _ := req.Option("local").Bool()
		if local {
			offlineexch := offline.Exchange(n.Blockstore)
			bserv := blockservice.New(n.Blockstore, offlineexch)
			dserv = dag.NewDAGService(bserv)
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, _, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		var bar *pb.ProgressBar
		if progress {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if progress {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !progress {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if progress {
					bar.Update()
				}
			case size := <-sizeChan:
				if progress {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName    = "quiet"
	silentOptionName   = "silent"
	progressOptionName = "progress"
	trickleOptionName  = "trickle"
	wrapOptionName     = "wrap-with-directory"
	hiddenOptionName   = "hidden"
	onlyHashOptionName = "only-hash"
	chunkerOptionName  = "chunker"
	pinOptionName      = "pin"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output.").Default(false),
		cmds.BoolOption(silentOptionName, "Write no output.").Default(false),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation.").Default(false),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk.").Default(false),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object.").Default(false),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
			return nil
		}

		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, n.DAG)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, _, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		silent, _, err := req.Option(silentOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		if !quiet && !silent {
			progress = true
		}

		var bar *pb.ProgressBar
		if progress {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if progress {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !progress {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if progress {
					bar.Update()
				}
			case size := <-sizeChan:
				if progress {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<commit_msg>ipfs/add: make --local flag not write out providers<commit_after>package commands

import (
	"fmt"
	"io"

	"github.com/ipfs/go-ipfs/core/coreunix"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"

	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName    = "quiet"
	silentOptionName   = "silent"
	progressOptionName = "progress"
	trickleOptionName  = "trickle"
	wrapOptionName     = "wrap-with-directory"
	hiddenOptionName   = "hidden"
	onlyHashOptionName = "only-hash"
	chunkerOptionName  = "chunker"
	pinOptionName      = "pin"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output.").Default(false),
		cmds.BoolOption(silentOptionName, "Write no output.").Default(false),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation.").Default(false),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk.").Default(false),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object.").Default(false),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add.").Default(false),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet {
			return nil
		}

		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		dserv := n.DAG
		local, _, _ := req.Option("local").Bool()
		if local {
			offlineexch := offline.Exchange(n.Blockstore)
			bserv := blockservice.New(n.Blockstore, offlineexch)
			dserv = dag.NewDAGService(bserv)
		}

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, _, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		silent, _, err := req.Option(silentOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		if !quiet && !silent {
			progress = true
		}

		var bar *pb.ProgressBar
		if progress {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if progress {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !progress {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if progress {
					bar.Update()
				}
			case size := <-sizeChan:
				if progress {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"
	"strings"

	logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log"

	cmds "github.com/ipfs/go-ipfs/commands"
)

// Golang os.Args overrides * and replaces the character argument with
// an array which includes every file in the user's CWD. As a
// workaround, we use 'all' instead. The util library still uses * so
// we convert it at this step.
var logAllKeyword = "all"

var LogCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with the daemon log output.",
		ShortDescription: `
'ipfs log' contains utility commands to affect or read the logging
output of a running daemon.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"level": logLevelCmd,
		"ls":    logLsCmd,
		"tail":  logTailCmd,
	},
}

var logLevelCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Change the logging level.",
		ShortDescription: `
'ipfs log level' is a utility command used to change the logging
output of a running daemon.
`,
	},

	Arguments: []cmds.Argument{
		// TODO use a different keyword for 'all' because all can theoretically
		// clash with a subsystem name
		cmds.StringArg("subsystem", true, false, fmt.Sprintf("The subsystem logging identifier. Use '%s' for all subsystems.", logAllKeyword)),
		cmds.StringArg("level", true, false, `The log level, with 'debug' the most verbose and 'critical' the least verbose.
			One of: debug, info, notice, warning, error, critical.
		`),
	},
	Run: func(req cmds.Request, res cmds.Response) {

		args := req.Arguments()
		subsystem, level := args[0], args[1]

		if subsystem == logAllKeyword {
			subsystem = "*"
		}

		if err := logging.SetLogLevel(subsystem, level); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		s := fmt.Sprintf("Changed log level of '%s' to '%s'\n", subsystem, level)
		log.Info(s)
		res.SetOutput(&MessageOutput{s})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: MessageTextMarshaler,
	},
	Type: MessageOutput{},
}

var logLsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List the logging subsystems.",
		ShortDescription: `
'ipfs log ls' is a utility command used to list the logging
subsystems of a running daemon.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		output := strings.Join(logging.GetSubsystems(), "\n")
		res.SetOutput(&MessageOutput{output + "\n"})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: MessageTextMarshaler,
	},
	Type: MessageOutput{},
}

var logTailCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Read the logs.",
		ShortDescription: `
'ipfs log tail' is a utility command used to read log output as it is written.
`,
	},

	Run: func(req cmds.Request, res cmds.Response) {
		ctx := req.Context()
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			<-ctx.Done()
		}()
		logging.WriterGroup.AddWriter(w)
		res.SetOutput(r)
	},
}
<commit_msg>Marshal log ls better<commit_after>package commands

import (
	"fmt"
	"io"

	logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log"

	cmds "github.com/ipfs/go-ipfs/commands"
)

// Golang os.Args overrides * and replaces the character argument with
// an array which includes every file in the user's CWD. As a
// workaround, we use 'all' instead. The util library still uses * so
// we convert it at this step.
var logAllKeyword = "all"

var LogCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with the daemon log output.",
		ShortDescription: `
'ipfs log' contains utility commands to affect or read the logging
output of a running daemon.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"level": logLevelCmd,
		"ls":    logLsCmd,
		"tail":  logTailCmd,
	},
}

var logLevelCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Change the logging level.",
		ShortDescription: `
'ipfs log level' is a utility command used to change the logging
output of a running daemon.
`,
	},

	Arguments: []cmds.Argument{
		// TODO use a different keyword for 'all' because all can theoretically
		// clash with a subsystem name
		cmds.StringArg("subsystem", true, false, fmt.Sprintf("The subsystem logging identifier. Use '%s' for all subsystems.", logAllKeyword)),
		cmds.StringArg("level", true, false, `The log level, with 'debug' the most verbose and 'critical' the least verbose.
			One of: debug, info, notice, warning, error, critical.
		`),
	},
	Run: func(req cmds.Request, res cmds.Response) {

		args := req.Arguments()
		subsystem, level := args[0], args[1]

		if subsystem == logAllKeyword {
			subsystem = "*"
		}

		if err := logging.SetLogLevel(subsystem, level); err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		s := fmt.Sprintf("Changed log level of '%s' to '%s'\n", subsystem, level)
		log.Info(s)
		res.SetOutput(&MessageOutput{s})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: MessageTextMarshaler,
	},
	Type: MessageOutput{},
}

var logLsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List the logging subsystems.",
		ShortDescription: `
'ipfs log ls' is a utility command used to list the logging
subsystems of a running daemon.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		res.SetOutput(&stringList{logging.GetSubsystems()})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: stringListMarshaler,
	},
	Type: stringList{},
}

var logTailCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Read the logs.",
		ShortDescription: `
'ipfs log tail' is a utility command used to read log output as it is written.
`,
	},

	Run: func(req cmds.Request, res cmds.Response) {
		ctx := req.Context()
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			<-ctx.Done()
		}()
		logging.WriterGroup.AddWriter(w)
		res.SetOutput(r)
	},
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package subtle implements functions that are often useful in cryptographic
// code but require careful thought to use correctly.
package subtle

// ConstantTimeCompare returns 1 iff the two slices, x
// and y, have equal contents. The time taken is a function of the length of
// the slices and is independent of the contents.
func ConstantTimeCompare(x, y []byte) int {
	if len(x) != len(y) {
		return 0
	}

	var v byte

	for i := 0; i < len(x); i++ {
		v |= x[i] ^ y[i]
	}

	return ConstantTimeByteEq(v, 0)
}

// ConstantTimeSelect returns x if v is 1 and y if v is 0.
// Its behavior is undefined if v takes any other value.
func ConstantTimeSelect(v, x, y int) int { return ^(v-1)&x | (v-1)&y }

// ConstantTimeByteEq returns 1 if x == y and 0 otherwise.
func ConstantTimeByteEq(x, y uint8) int {
	z := ^(x ^ y)
	z &= z >> 4
	z &= z >> 2
	z &= z >> 1

	return int(z)
}

// ConstantTimeEq returns 1 if x == y and 0 otherwise.
func ConstantTimeEq(x, y int32) int {
	z := ^(x ^ y)
	z &= z >> 16
	z &= z >> 8
	z &= z >> 4
	z &= z >> 2
	z &= z >> 1

	return int(z & 1)
}

// ConstantTimeCopy copies the contents of y into x (a slice of equal length)
// if v == 1. If v == 0, x is left unchanged. Its behavior is undefined if v
// takes any other value.
func ConstantTimeCopy(v int, x, y []byte) {
	if len(x) != len(y) {
		panic("subtle: slices have different lengths")
	}

	xmask := byte(v - 1)
	ymask := byte(^(v - 1))
	for i := 0; i < len(x); i++ {
		x[i] = x[i]&xmask | y[i]&ymask
	}
}

// ConstantTimeLessOrEq returns 1 if x <= y and 0 otherwise.
// Its behavior is undefined if x or y are negative or > 2**31 - 1.
func ConstantTimeLessOrEq(x, y int) int {
	x32 := int32(x)
	y32 := int32(y)
	return int(((x32 - y32 - 1) >> 31) & 1)
}
<commit_msg>crypto/subtle: expand abbreviation to eliminate confusion<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package subtle implements functions that are often useful in cryptographic
// code but require careful thought to use correctly.
package subtle

// ConstantTimeCompare returns 1 if and only if the two slices, x
// and y, have equal contents. The time taken is a function of the length of
// the slices and is independent of the contents.
func ConstantTimeCompare(x, y []byte) int {
	if len(x) != len(y) {
		return 0
	}

	var v byte

	for i := 0; i < len(x); i++ {
		v |= x[i] ^ y[i]
	}

	return ConstantTimeByteEq(v, 0)
}

// ConstantTimeSelect returns x if v is 1 and y if v is 0.
// Its behavior is undefined if v takes any other value.
func ConstantTimeSelect(v, x, y int) int { return ^(v-1)&x | (v-1)&y }

// ConstantTimeByteEq returns 1 if x == y and 0 otherwise.
func ConstantTimeByteEq(x, y uint8) int {
	z := ^(x ^ y)
	z &= z >> 4
	z &= z >> 2
	z &= z >> 1

	return int(z)
}

// ConstantTimeEq returns 1 if x == y and 0 otherwise.
func ConstantTimeEq(x, y int32) int {
	z := ^(x ^ y)
	z &= z >> 16
	z &= z >> 8
	z &= z >> 4
	z &= z >> 2
	z &= z >> 1

	return int(z & 1)
}

// ConstantTimeCopy copies the contents of y into x (a slice of equal length)
// if v == 1. If v == 0, x is left unchanged. Its behavior is undefined if v
// takes any other value.
func ConstantTimeCopy(v int, x, y []byte) {
	if len(x) != len(y) {
		panic("subtle: slices have different lengths")
	}

	xmask := byte(v - 1)
	ymask := byte(^(v - 1))
	for i := 0; i < len(x); i++ {
		x[i] = x[i]&xmask | y[i]&ymask
	}
}

// ConstantTimeLessOrEq returns 1 if x <= y and 0 otherwise.
// Its behavior is undefined if x or y are negative or > 2**31 - 1.
func ConstantTimeLessOrEq(x, y int) int {
	x32 := int32(x)
	y32 := int32(y)
	return int(((x32 - y32 - 1) >> 31) & 1)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"io/ioutil"
	"log"
	"time"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/BurntSushi/toml"
)

type Config struct {
	Upstream string
	ReqFanFactor int
	TimeoutMS int
	MemcacheHosts []string
	MemcacheSeconds int
}

func snd (c chan string, s string) {
	go func () { c <- s }()
}

// start starts Charm up and returns a done channel for the done message
func start(confPath string) (chan string) {
	done := make(chan string)

	// print a welcome message
	log.Print("Charm is starting up.")
	// read the config file
	log.Printf(".   . Reading %s", confPath)
	tomlData, err := ioutil.ReadFile(confPath)
	if err != nil {
		snd(done, fmt.Sprintf("Could not read file at %s", confPath))
		return done
	}
	// populate the config struct
	log.Print(".   . Loading config")
	var conf Config
	_, err = toml.Decode(string(tomlData), &conf)
	if err != nil {
		snd(done, "Could not decode config")
		return done
	}
	// report on the configuration
	log.Print("Charm is configured!")
	log.Printf(".   . Stabilizing %v", conf.Upstream)
	log.Printf(".   . with %v duplicate requests", conf.ReqFanFactor)
	log.Printf(".   . and a %v milisecond timeout.", conf.TimeoutMS)
	log.Printf(
		".   . memcached at %v for %v seconds.",
		conf.MemcacheHosts,
		conf.MemcacheSeconds,
	)
	go run(conf, done)
	return done
}

type stableTransport struct {
	wrappedTransport http.RoundTripper
	reqFanFactor int
	cacheResponse chan *http.Response
}

// stableTransport.RoundTrip makes many round trips and returns the first
// response
func (t *stableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	c := make(chan *http.Response)
	if t.wrappedTransport == nil {
		t.wrappedTransport = http.DefaultTransport
	}
	// fan out requests, send responses to the channel, log errors, don't
	// wait very long for someone to recieve our response
	for i := 0; i < t.reqFanFactor; i++ {
		go func () {
			resp, err := t.wrappedTransport.RoundTrip(r)
			if err != nil {
				log.Printf("transport-error: %v", err)
			} else {
				select {
				case c <- resp:
					// they were still waiting for the first
					// response and they recieved it from c
					return
				case <-time.After(1 * time.Millisecond):
					// no one was waiting to recieve from c
					// so this is not the first response
					return
				}
			}
		}()
	}

	// wait to reviece the first response
	first := <-c
	// send first response to be cached if they want it
	go func () {
		select {
		case t.cacheResponse <- first:
			return
		case <-time.After(15 * time.Millisecond):
			return
		}
	}()
	// return the first response to the handler
	return first, nil
}

// cacheKey returns a string to be used as the cache key for a request
func cacheKey(r *http.Request) (string, error) {
	// We need to be careful here.
	// There is serious potential to accidently ignore permissions if we
	// cache requests too broadly. For example, if our cache key is the path
	// and a super-admin cache-misses on /some/restricted/path then a
	// restricted user could be given the cached result from that super
	//  admin request.

	keyStr := ""
	keyStr += r.Method
	keyStr += r.URL.Host
	keyStr += r.URL.Path
	keyStr += r.URL.RawQuery

	// TODO: extract which headers to cache on into a config option
	keyStr += r.Header["X-Forwarded-Email"][0]

	key := sha256.Sum224([]byte(keyStr))
	return string(key[:sha256.Size224]), nil
}

// Conf.ServeHTTP checks memcache then proxies/caches with a stable transport
func (conf Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// check memcache
	mc := memcache.New(conf.MemcacheHosts...)
	key, err := cacheKey(r)
	if err != nil {
		log.Println("cache key error for Request(%v)", r)
	} else {
		item, err := mc.Get(key)
		if err == nil { //cache hit
			// get the cached response
			response, err := http.ReadResponse(
				bufio.NewReader(bytes.NewReader(item.Value)),
				r,
			)
			if err == nil {
				// write the response to the response writer
				response.Write(w)
				return
			}
		}
	}
	// cache miss
	upstreamURL, err := url.Parse(conf.Upstream)
	if err != nil {
		log.Fatal("error parsing Upstream URL", conf.Upstream)
	}
	responseChan := make(chan *http.Response)
	proxy := httputil.NewSingleHostReverseProxy(upstreamURL)
	proxy.Transport = &stableTransport{
		proxy.Transport,
		conf.ReqFanFactor,
		responseChan,
	}
	proxy.ServeHTTP(w, r)

	// if the transport has a response waiting on the channel, cache it if
	// we have a cache
	if cacheKey != nil {
		select {
		case resp := <- responseChan:
			dump, err := httputil.DumpResponse(resp, true)
			if err != nil {
				log.Println("error dumping response: %v", err)
				return
			}
			item := &memcache.Item{
				Key: key,
				Value: dump,
				Expiration: int32(conf.MemcacheSeconds),
			}
			mc.Set(item)
		case <-time.After(1 * time.Millisecond):
			return
		}
	}
}

// run
func run(conf Config, done chan string) {
	// serve the config under a timeout
	timeout := time.Duration(conf.TimeoutMS) * time.Millisecond
	log.Fatal(http.ListenAndServe(
		":8000",
		http.TimeoutHandler(conf, timeout, "upstream timeout"),
	))
}

func main() {
	// start Charm,
	done := start("/secret/charm.conf")
	// when Charm is done, log the message and quit.
        log.Print(<-done)
}
<commit_msg>serve http after caching<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/sha256"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"io/ioutil"
	"log"
	"time"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/BurntSushi/toml"
)

type Config struct {
	Upstream string
	ReqFanFactor int
	TimeoutMS int
	MemcacheHosts []string
	MemcacheSeconds int
}

func snd (c chan string, s string) {
	go func () { c <- s }()
}

// start starts Charm up and returns a done channel for the done message
func start(confPath string) (chan string) {
	done := make(chan string)

	// print a welcome message
	log.Print("Charm is starting up.")
	// read the config file
	log.Printf(".   . Reading %s", confPath)
	tomlData, err := ioutil.ReadFile(confPath)
	if err != nil {
		snd(done, fmt.Sprintf("Could not read file at %s", confPath))
		return done
	}
	// populate the config struct
	log.Print(".   . Loading config")
	var conf Config
	_, err = toml.Decode(string(tomlData), &conf)
	if err != nil {
		snd(done, "Could not decode config")
		return done
	}
	// report on the configuration
	log.Print("Charm is configured!")
	log.Printf(".   . Stabilizing %v", conf.Upstream)
	log.Printf(".   . with %v duplicate requests", conf.ReqFanFactor)
	log.Printf(".   . and a %v milisecond timeout.", conf.TimeoutMS)
	log.Printf(
		".   . memcached at %v for %v seconds.",
		conf.MemcacheHosts,
		conf.MemcacheSeconds,
	)
	go run(conf, done)
	return done
}

type stableTransport struct {
	wrappedTransport http.RoundTripper
	reqFanFactor int
	cacheResponse chan *http.Response
}

// stableTransport.RoundTrip makes many round trips and returns the first
// response
func (t *stableTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	c := make(chan *http.Response)
	if t.wrappedTransport == nil {
		t.wrappedTransport = http.DefaultTransport
	}
	// fan out requests, send responses to the channel, log errors, don't
	// wait very long for someone to recieve our response
	for i := 0; i < t.reqFanFactor; i++ {
		go func () {
			resp, err := t.wrappedTransport.RoundTrip(r)
			if err != nil {
				log.Printf("transport-error: %v", err)
			} else {
				select {
				case c <- resp:
					// they were still waiting for the first
					// response and they recieved it from c
					return
				case <-time.After(1 * time.Millisecond):
					// no one was waiting to recieve from c
					// so this is not the first response
					return
				}
			}
		}()
	}

	// wait to reviece the first response
	first := <-c
	// send first response to be cached if they want it
	go func () {
		select {
		case t.cacheResponse <- first:
			return
		case <-time.After(15 * time.Millisecond):
			return
		}
	}()
	// return the first response to the handler
	return first, nil
}

// cacheKey returns a string to be used as the cache key for a request
func cacheKey(r *http.Request) (string, error) {
	// We need to be careful here.
	// There is serious potential to accidently ignore permissions if we
	// cache requests too broadly. For example, if our cache key is the path
	// and a super-admin cache-misses on /some/restricted/path then a
	// restricted user could be given the cached result from that super
	//  admin request.

	keyStr := ""
	keyStr += r.Method
	keyStr += r.URL.Host
	keyStr += r.URL.Path
	keyStr += r.URL.RawQuery

	// TODO: extract which headers to cache on into a config option
	keyStr += r.Header["X-Forwarded-Email"][0]

	key := sha256.Sum224([]byte(keyStr))
	return string(key[:sha256.Size224]), nil
}

// Conf.ServeHTTP checks memcache then proxies/caches with a stable transport
func (conf Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// check memcache
	mc := memcache.New(conf.MemcacheHosts...)
	key, err := cacheKey(r)
	if err != nil {
		log.Println("cache key error for Request(%v)", r)
	} else {
		item, err := mc.Get(key)
		if err == nil { //cache hit
			// get the cached response
			response, err := http.ReadResponse(
				bufio.NewReader(bytes.NewReader(item.Value)),
				r,
			)
			if err == nil {
				// write the response to the response writer
				response.Write(w)
				return
			}
		}
	}
	// cache miss
	upstreamURL, err := url.Parse(conf.Upstream)
	if err != nil {
		log.Fatal("error parsing Upstream URL", conf.Upstream)
	}
	responseChan := make(chan *http.Response)
	proxy := httputil.NewSingleHostReverseProxy(upstreamURL)
	proxy.Transport = &stableTransport{
		proxy.Transport,
		conf.ReqFanFactor,
		responseChan,
	}

	// if the transport has a response waiting on the channel, cache it if
	// we have a cache
	if cacheKey != nil {
		select {
		case resp := <- responseChan:
			dump, err := httputil.DumpResponse(resp, true)
			if err != nil {
				log.Println("error dumping response:", err)
				return
			}
			item := &memcache.Item{
				Key: key,
				Value: dump,
				Expiration: int32(conf.MemcacheSeconds),
			}
			mc.Set(item)
		case <-time.After(1 * time.Millisecond):
			return
		}
	}

	// TODO: figure out how to serve this before caching the response
	//       we currently get "read on closed response body" if we try
	//       that.
	proxy.ServeHTTP(w, r)
}

// run
func run(conf Config, done chan string) {
	// serve the config under a timeout
	timeout := time.Duration(conf.TimeoutMS) * time.Millisecond
	log.Fatal(http.ListenAndServe(
		":8000",
		http.TimeoutHandler(conf, timeout, "upstream timeout"),
	))
}

func main() {
	// start Charm,
	done := start("/secret/charm.conf")
	// when Charm is done, log the message and quit.
        log.Print(<-done)
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"os"
	"strings"

	"github.com/rancher/rancher/pkg/controllers/management/drivers/kontainerdriver"
	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func addKontainerDrivers(management *config.ManagementContext) error {
	// create binary drop location if not exists
	err := os.MkdirAll(kontainerdriver.DriverDir, 0777)
	if err != nil {
		return fmt.Errorf("error creating binary drop folder: %v", err)
	}

	creator := driverCreator{
		driversLister: management.Management.KontainerDrivers("").Controller().Lister(),
		drivers:       management.Management.KontainerDrivers(""),
	}

	if err := cleanupImportDriver(creator); err != nil {
		return err
	}

	if err := creator.add("rancherKubernetesEngine"); err != nil {
		return err
	}

	if err := creator.add("googleKubernetesEngine"); err != nil {
		return err
	}

	if err := creator.add("azureKubernetesService"); err != nil {
		return err
	}

	if err := creator.add("amazonElasticContainerService"); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"baiducloudcontainerengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-baidu/0.2.0/kontainer-engine-driver-baidu-linux",
		"4613e3be3ae5487b0e21dfa761b95de2144f80f98bf76847411e5fcada343d5e",
		"https://drivers.rancher.cn/kontainer-engine-driver-baidu/0.2.0/component.js",
		false,
		"drivers.rancher.cn", "*.baidubce.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"aliyunkubernetescontainerservice",
		"https://drivers.rancher.cn/kontainer-engine-driver-aliyun/0.2.5/kontainer-engine-driver-aliyun-linux",
		"31aa0a44450c5a5eb128dd0956292dfd91aab726d1a548f6d527a9212a27db9b",
		"",
		false,
		"*.aliyuncs.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"tencentkubernetesengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-tencent/0.2.3/kontainer-engine-driver-tencent-linux",
		"144f785473290ee2f63cf35da0c6bde12bc307878078500a47a0a8d04422ae53",
		"",
		false,
		"*.tencentcloudapi.com", "*.qcloud.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"huaweicontainercloudengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-huawei/0.1.2/kontainer-engine-driver-huawei-linux",
		"0b6c1dfaa477a60a3bd9f8a60a55fcafd883866c2c5c387aec75b95d6ba81d45",
		"",
		false,
		"*.myhuaweicloud.com",
	); err != nil {
		return err
	}
	if err := creator.addCustomDriver(
		"oraclecontainerengine",
		"https://github.com/rancher-plugins/kontainer-engine-driver-oke/releases/download/v1.4.0/kontainer-engine-driver-oke-linux",
		"a42bd8b14e780edc426119f143acbc54a93a07f8b62f193b9e261ddd85db6e9c",
		"",
		false,
		"*.oraclecloud.com",
	); err != nil {
		return err
	}

	return nil
}

func cleanupImportDriver(creator driverCreator) error {
	var err error
	if _, err = creator.driversLister.Get("", "import"); err == nil {
		err = creator.drivers.Delete("import", &v1.DeleteOptions{})
	}

	if !errors.IsNotFound(err) {
		return err
	}

	return nil
}

type driverCreator struct {
	driversLister v3.KontainerDriverLister
	drivers       v3.KontainerDriverInterface
}

func (c *driverCreator) add(name string) error {
	logrus.Infof("adding kontainer driver %v", name)

	driver, err := c.driversLister.Get("", name)
	if err != nil {
		if errors.IsNotFound(err) {
			_, err = c.drivers.Create(&v3.KontainerDriver{
				ObjectMeta: v1.ObjectMeta{
					Name:      strings.ToLower(name),
					Namespace: "",
				},
				Spec: v3.KontainerDriverSpec{
					URL:     "",
					BuiltIn: true,
					Active:  true,
				},
				Status: v3.KontainerDriverStatus{
					DisplayName: name,
				},
			})
			if err != nil && !errors.IsAlreadyExists(err) {
				return fmt.Errorf("error creating driver: %v", err)
			}
		} else {
			return fmt.Errorf("error getting driver: %v", err)
		}
	} else {
		driver.Spec.URL = ""

		_, err = c.drivers.Update(driver)
		if err != nil {
			return fmt.Errorf("error updating driver: %v", err)
		}
	}

	return nil
}

func (c *driverCreator) addCustomDriver(name, url, checksum, uiURL string, active bool, domains ...string) error {
	logrus.Infof("adding kontainer driver %v", name)
	_, err := c.driversLister.Get("", name)
	if err != nil {
		if errors.IsNotFound(err) {
			_, err = c.drivers.Create(&v3.KontainerDriver{
				ObjectMeta: v1.ObjectMeta{
					Name: strings.ToLower(name),
				},
				Spec: v3.KontainerDriverSpec{
					URL:              url,
					BuiltIn:          false,
					Active:           active,
					Checksum:         checksum,
					UIURL:            uiURL,
					WhitelistDomains: domains,
				},
				Status: v3.KontainerDriverStatus{
					DisplayName: name,
				},
			})
			if err != nil && !errors.IsAlreadyExists(err) {
				return fmt.Errorf("error creating driver: %v", err)
			}
		} else {
			return fmt.Errorf("error getting driver: %v", err)
		}
	}
	return nil
}
<commit_msg>Update TKE cluster driver<commit_after>package app

import (
	"fmt"
	"os"
	"strings"

	"github.com/rancher/rancher/pkg/controllers/management/drivers/kontainerdriver"
	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/errors"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func addKontainerDrivers(management *config.ManagementContext) error {
	// create binary drop location if not exists
	err := os.MkdirAll(kontainerdriver.DriverDir, 0777)
	if err != nil {
		return fmt.Errorf("error creating binary drop folder: %v", err)
	}

	creator := driverCreator{
		driversLister: management.Management.KontainerDrivers("").Controller().Lister(),
		drivers:       management.Management.KontainerDrivers(""),
	}

	if err := cleanupImportDriver(creator); err != nil {
		return err
	}

	if err := creator.add("rancherKubernetesEngine"); err != nil {
		return err
	}

	if err := creator.add("googleKubernetesEngine"); err != nil {
		return err
	}

	if err := creator.add("azureKubernetesService"); err != nil {
		return err
	}

	if err := creator.add("amazonElasticContainerService"); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"baiducloudcontainerengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-baidu/0.2.0/kontainer-engine-driver-baidu-linux",
		"4613e3be3ae5487b0e21dfa761b95de2144f80f98bf76847411e5fcada343d5e",
		"https://drivers.rancher.cn/kontainer-engine-driver-baidu/0.2.0/component.js",
		false,
		"drivers.rancher.cn", "*.baidubce.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"aliyunkubernetescontainerservice",
		"https://drivers.rancher.cn/kontainer-engine-driver-aliyun/0.2.5/kontainer-engine-driver-aliyun-linux",
		"31aa0a44450c5a5eb128dd0956292dfd91aab726d1a548f6d527a9212a27db9b",
		"",
		false,
		"*.aliyuncs.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"tencentkubernetesengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-tencent/0.3.0/kontainer-engine-driver-tencent-linux",
		"ad5406502daf826874889963d7bdaed78db4689f147889ecf97394bc4e8d3d76",
		"",
		false,
		"*.tencentcloudapi.com", "*.qcloud.com",
	); err != nil {
		return err
	}

	if err := creator.addCustomDriver(
		"huaweicontainercloudengine",
		"https://drivers.rancher.cn/kontainer-engine-driver-huawei/0.1.2/kontainer-engine-driver-huawei-linux",
		"0b6c1dfaa477a60a3bd9f8a60a55fcafd883866c2c5c387aec75b95d6ba81d45",
		"",
		false,
		"*.myhuaweicloud.com",
	); err != nil {
		return err
	}
	if err := creator.addCustomDriver(
		"oraclecontainerengine",
		"https://github.com/rancher-plugins/kontainer-engine-driver-oke/releases/download/v1.4.0/kontainer-engine-driver-oke-linux",
		"a42bd8b14e780edc426119f143acbc54a93a07f8b62f193b9e261ddd85db6e9c",
		"",
		false,
		"*.oraclecloud.com",
	); err != nil {
		return err
	}

	return nil
}

func cleanupImportDriver(creator driverCreator) error {
	var err error
	if _, err = creator.driversLister.Get("", "import"); err == nil {
		err = creator.drivers.Delete("import", &v1.DeleteOptions{})
	}

	if !errors.IsNotFound(err) {
		return err
	}

	return nil
}

type driverCreator struct {
	driversLister v3.KontainerDriverLister
	drivers       v3.KontainerDriverInterface
}

func (c *driverCreator) add(name string) error {
	logrus.Infof("adding kontainer driver %v", name)

	driver, err := c.driversLister.Get("", name)
	if err != nil {
		if errors.IsNotFound(err) {
			_, err = c.drivers.Create(&v3.KontainerDriver{
				ObjectMeta: v1.ObjectMeta{
					Name:      strings.ToLower(name),
					Namespace: "",
				},
				Spec: v3.KontainerDriverSpec{
					URL:     "",
					BuiltIn: true,
					Active:  true,
				},
				Status: v3.KontainerDriverStatus{
					DisplayName: name,
				},
			})
			if err != nil && !errors.IsAlreadyExists(err) {
				return fmt.Errorf("error creating driver: %v", err)
			}
		} else {
			return fmt.Errorf("error getting driver: %v", err)
		}
	} else {
		driver.Spec.URL = ""

		_, err = c.drivers.Update(driver)
		if err != nil {
			return fmt.Errorf("error updating driver: %v", err)
		}
	}

	return nil
}

func (c *driverCreator) addCustomDriver(name, url, checksum, uiURL string, active bool, domains ...string) error {
	logrus.Infof("adding kontainer driver %v", name)
	_, err := c.driversLister.Get("", name)
	if err != nil {
		if errors.IsNotFound(err) {
			_, err = c.drivers.Create(&v3.KontainerDriver{
				ObjectMeta: v1.ObjectMeta{
					Name: strings.ToLower(name),
				},
				Spec: v3.KontainerDriverSpec{
					URL:              url,
					BuiltIn:          false,
					Active:           active,
					Checksum:         checksum,
					UIURL:            uiURL,
					WhitelistDomains: domains,
				},
				Status: v3.KontainerDriverStatus{
					DisplayName: name,
				},
			})
			if err != nil && !errors.IsAlreadyExists(err) {
				return fmt.Errorf("error creating driver: %v", err)
			}
		} else {
			return fmt.Errorf("error getting driver: %v", err)
		}
	}
	return nil
}
<|endoftext|>