<commit_before>package impl

import (
	"fmt"
	"strings"
	"time"
	"github.com/vitesse-ftian/dggo/vitessedata/proto/xdrive"
	"vitessedata/plugin"
//	"github.com/Shopify/sarama"
	"github.com/wvanbergen/kafka/consumergroup"
	"github.com/wvanbergen/kazoo-go"
)

const (
	waitMilliseconds = 1000
	consumerGroupName = "deepgreen"
)

func DoRead() error {
	var topic, zkString string

	rinfo := plugin.RInfo()
	ss := strings.Split(rinfo.Rpath, "/")
	//brokerList := ss[0]
	topic = ss[1]

	conf := rinfo.GetConf()
	for _, kv := range conf.GetKv() {
		if kv.GetKey() == "zookeeper" {
			zkString = kv.GetValue()
		}
	}

	plugin.FatalIf(topic == "" || zkString == "", "Kafka requires topic and zookeeper")

	var req xdrive.ReadRequest
        err := plugin.DelimRead(&req)
        if err != nil {
                plugin.DbgLogIfErr(err, "Delim read req failed.")
                return err
        }

        // Check/validate frag info.  Again, not necessary, as xdriver server should always
        // fill in good value.
        if req.FragCnt <= 0 || req.FragId < 0 || req.FragId >= req.FragCnt {
                plugin.DbgLog("Invalid read req %v", req)
                plugin.ReplyError(-3, fmt.Sprintf("Read request frag (%d, %d) is not valid.", req.FragId, req.FragCnt))
                return fmt.Errorf("Invalid read request")
        }

//	zkPeers := strings.Split(zkString, ",")

	config := consumergroup.NewConfig()
//	config.Offsets.ResetOffsets = true
//	config.Offsets.Initial = sarama.OffsetNewest

	var zkPeers []string
	zkPeers, config.Zookeeper.Chroot = kazoo.ParseConnectionString(zkString)

	consumer, consumerErr := consumergroup.JoinConsumerGroup(
		consumerGroupName,
		[]string{topic},
		zkPeers,
		config)

	if consumerErr != nil {
		plugin.DbgLog("Failed to join consumer group")
		plugin.ReplyError(-4, "join consumer group error")
		return consumerErr
	}

	defer func() {
		if closeErr := consumer.Close() ; closeErr != nil {
			plugin.DbgLogIfErr(closeErr, "consumer close failed")
		}
	}()

	tStart := time.Now()

	var js JsonReader
	js.Init(req.Filespec, req.Columndesc, req.Columnlist)

	var messages [][]byte
	running := true
	for ; running ; {
		select {
		case err := <- consumer.Errors():
			plugin.DbgLogIfErr(err, "consumer error")
			plugin.ReplyError(-20, "Consumer Error")
			return err
		case msg := <- consumer.Messages():
			tStart = time.Now()

			//plugin.DbgLog("message received...")
			//plugin.DbgLog(string(msg.Value))
			messages = append(messages, msg.Value)
			consumer.CommitUpto(msg)

			if len(messages) == 1000 {
				err = js.processAll(messages)
				if err != nil {
					plugin.DbgLogIfErr(err, "failed to write to deepgreen")
					plugin.ReplyError(-20, "Failed to write to deepgreen")
					return err
				}
				plugin.DbgLog("%d rows read", len(messages))
				messages = nil
				consumer.FlushOffsets()
			}

		default:
			elapsed := time.Since(tStart)
			if elapsed > waitMilliseconds*time.Millisecond {
				plugin.DbgLog("plugin timed out")
				running = false
			}
		}
	}

	if len(messages) > 0 {
		err = js.processAll(messages)
		if err != nil {
			plugin.DbgLogIfErr(err, "failed to write to deepgreen")
			plugin.ReplyError(-20, "Failed to write to deepgreen")
			return err
		}
		consumer.FlushOffsets()
		plugin.DbgLog("%d rows read", len(messages))
	}

	plugin.ReplyError(0, "")
	return nil
}
<commit_msg>without using zookeeper for offset<commit_after>package impl

import (
	"fmt"
	"strings"
	"time"
	"github.com/vitesse-ftian/dggo/vitessedata/proto/xdrive"
	"vitessedata/plugin"
	"github.com/Shopify/sarama"
	cluster "github.com/bsm/sarama-cluster"
	//"github.com/wvanbergen/kafka/consumergroup"
	//"github.com/wvanbergen/kazoo-go"
)

const (
	waitMilliseconds = 5000
	consumerGroupName = "deepgreen"
)

func DoRead() error {
	var topic, zkString string

	rinfo := plugin.RInfo()
	ss := strings.Split(rinfo.Rpath, "/")
	brokerList := ss[0]
	topic = ss[1]

	conf := rinfo.GetConf()
	for _, kv := range conf.GetKv() {
		if kv.GetKey() == "zookeeper" {
			zkString = kv.GetValue()
		}
	}

	plugin.FatalIf(topic == "" || zkString == "", "Kafka requires topic and zookeeper")

	var req xdrive.ReadRequest
        err := plugin.DelimRead(&req)
        if err != nil {
                plugin.DbgLogIfErr(err, "Delim read req failed.")
                return err
        }

        // Check/validate frag info.  Again, not necessary, as xdriver server should always
        // fill in good value.
        if req.FragCnt <= 0 || req.FragId < 0 || req.FragId >= req.FragCnt {
                plugin.DbgLog("Invalid read req %v", req)
                plugin.ReplyError(-3, fmt.Sprintf("Read request frag (%d, %d) is not valid.", req.FragId, req.FragCnt))
                return fmt.Errorf("Invalid read request")
        }

//	zkPeers := strings.Split(zkString, ",")

	config := cluster.NewConfig()
	config.Consumer.Return.Errors = true
	config.Group.Return.Notifications = true
	config.Consumer.Offsets.Initial = sarama.OffsetOldest

	plugin.DbgLog("Cluster consumer start...")
	consumer, err := cluster.NewConsumer(strings.Split(brokerList, ","),
		consumerGroupName,
		[]string{topic},
		config)

	if err != nil {
		plugin.DbgLogIfErr(err, "Failed to start consumer")
		plugin.ReplyError(-4, "Failed to start consumer")
		return err
	}

	defer consumer.Close()

	// consume errors
	go func() {
		for err := range consumer.Errors() {
			plugin.DbgLog("Error: %s\n", err.Error())
		}
	}()

	// consume notifications
	go func() {
		for ntf := range consumer.Notifications() {
			plugin.DbgLog("Rebalanced: %+v\n", ntf)
		}
	}()

	timeout := make(chan bool, 1)
	go func() {
		time.Sleep(waitMilliseconds * time.Millisecond)
		timeout <- true
	}()

	//tStart := time.Now()

	var js JsonReader
	js.Init(req.Filespec, req.Columndesc, req.Columnlist)

	var messages [][]byte
	firstmsg := true

	running := true
	for ; running ; {
		select {
		case msg, ok := <- consumer.Messages():
			if ok {
				//tStart = time.Now()
				if firstmsg {
					plugin.DbgLog("message received...")
					firstmsg = false
				}
				//plugin.DbgLog(string(msg.Value))
				messages = append(messages, msg.Value)
				consumer.MarkOffset(msg, "")

				if len(messages) == 1000 {
					plugin.DbgLog("%d messages write to db", len(messages))
					err = js.processAll(messages)
					if err != nil {
						plugin.DbgLogIfErr(err, "failed to write to deepgreen")
						plugin.ReplyError(-20, "Failed to write to deepgreen")
						return err
					}
					plugin.DbgLog("%d rows read", len(messages))
					messages = nil
					//consumer.FlushOffsets()
				}
			}

			case <- timeout:
			plugin.DbgLog("plugin timed out")
			running = false
		}
	}

	if len(messages) > 0 {
		plugin.DbgLog("%d messages write to db", len(messages))
		err = js.processAll(messages)
		if err != nil {
			plugin.DbgLogIfErr(err, "failed to write to deepgreen")
			plugin.ReplyError(-20, "Failed to write to deepgreen")
			return err
		}
//		consumer.FlushOffsets()
		plugin.DbgLog("%d rows read", len(messages))
	}

	plugin.ReplyError(0, "")
	return nil
}
<|endoftext|>


<commit_before>package isolated

import (
	"regexp"

	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("org-users command", func() {
	Describe("help", func() {
		When("--help flag is set", func() {
			It("displays command usage to output", func() {
				session := helpers.CF("org-users", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("org-users - Show org users by role"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say(regexp.QuoteMeta("cf org-users ORG")))
				Eventually(session).Should(Say("OPTIONS:"))
				Eventually(session).Should(Say(`--all-users, -a \s+List all users with roles in the org or in spaces within the org`))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("orgs, set-org-role"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the user is logged in", func() {
		var (
			orgName       string
			adminUsername string
		)

		BeforeEach(func() {
			adminUsername = helpers.LoginCF()
			orgName = helpers.NewOrgName()
			helpers.CreateOrg(orgName)
		})

		When("the target org has multiple users with different roles", func() {
			var (
				orgManagerUser     string
				billingManagerUser string
				orgAuditorUser     string
			)

			BeforeEach(func() {
				orgManagerUser, _ = helpers.CreateUserInOrgRole(orgName, "OrgManager")
				billingManagerUser, _ = helpers.CreateUserInOrgRole(orgName, "BillingManager")
				orgAuditorUser, _ = helpers.CreateUserInOrgRole(orgName, "OrgAuditor")
			})

			It("prints the users in the target org under their roles", func() {
				session := helpers.CF("org-users", orgName)
				Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
				Eventually(session).Should(Say("ORG MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, orgManagerUser))
				Eventually(session).Should(Say("BILLING MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, billingManagerUser))
				Eventually(session).Should(Say("ORG AUDITOR"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, orgAuditorUser))
				Eventually(session).Should(Exit(0))
			})

			When("the --all-users flag is given", func() {
				It("prints all users in the org, grouped under USERS", func() {
					session := helpers.CF("org-users", orgName, "--all-users")
					Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
					Eventually(session).Should(Say("USERS"))
					Eventually(session).Should(Exit(0))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, orgManagerUser))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, billingManagerUser))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, orgAuditorUser))
				})
			})
		})

		When("the target org has a client-credentials user", func() {
			var clientID string

			BeforeEach(func() {
				clientID, _ = helpers.SkipIfClientCredentialsNotSet()
				Eventually(helpers.CF("set-org-role", clientID, orgName, "OrgManager", "--client")).Should(Exit(0))
			})

			It("prints the client-credentials user", func() {
				session := helpers.CF("org-users", orgName)
				Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
				Eventually(session).Should(Say("ORG MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(client\)`, clientID))
				Eventually(session).Should(Say("BILLING MANAGER"))
				Eventually(session).Should(Say(`\s+No BILLING MANAGER found`))
				Eventually(session).Should(Say("ORG AUDITOR"))
				Eventually(session).Should(Say(`\s+No ORG AUDITOR found`))
				Eventually(session).Should(Exit(0))
			})
		})
	})
})
<commit_msg>Account for user role assignment in org users command integration tests<commit_after>package isolated

import (
	"regexp"

	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("org-users command", func() {
	Describe("help", func() {
		When("--help flag is set", func() {
			It("displays command usage to output", func() {
				session := helpers.CF("org-users", "--help")
				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("org-users - Show org users by role"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say(regexp.QuoteMeta("cf org-users ORG")))
				Eventually(session).Should(Say("OPTIONS:"))
				Eventually(session).Should(Say(`--all-users, -a \s+List all users with roles in the org or in spaces within the org`))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("orgs, set-org-role"))
				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the user is logged in", func() {
		var (
			orgName       string
			adminUsername string
		)

		BeforeEach(func() {
			adminUsername = helpers.LoginCF()
			orgName = helpers.NewOrgName()
			helpers.CreateOrg(orgName)
		})

		When("the target org has multiple users with different roles", func() {
			var (
				orgManagerUser     string
				billingManagerUser string
				orgAuditorUser     string
			)

			BeforeEach(func() {
				orgManagerUser, _ = helpers.CreateUserInOrgRole(orgName, "OrgManager")
				billingManagerUser, _ = helpers.CreateUserInOrgRole(orgName, "BillingManager")
				orgAuditorUser, _ = helpers.CreateUserInOrgRole(orgName, "OrgAuditor")
			})

			It("prints the users in the target org under their roles", func() {
				session := helpers.CF("org-users", orgName)
				Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
				Eventually(session).Should(Say("ORG MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, orgManagerUser))
				Eventually(session).Should(Say("BILLING MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, billingManagerUser))
				Eventually(session).Should(Say("ORG AUDITOR"))
				Eventually(session).Should(Say(`\s+%s \(uaa\)`, orgAuditorUser))
				Eventually(session).Should(Exit(0))
			})

			When("the --all-users flag is given", func() {
				It("prints all users in the org, grouped under USERS", func() {
					session := helpers.CF("org-users", orgName, "--all-users")
					Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
					Eventually(session).Should(Say("USERS"))
					Eventually(session).Should(Exit(0))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, orgManagerUser))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, billingManagerUser))
					Expect(session.Out.Contents()).To(MatchRegexp(`\s+%s \(uaa\)`, orgAuditorUser))
				})
			})
		})

		When("the target org has a client-credentials user", func() {
			var clientID string

			BeforeEach(func() {
				clientID, _ = helpers.SkipIfClientCredentialsNotSet()
				Eventually(helpers.CF("set-org-role", clientID, orgName, "OrgManager", "--client")).Should(Exit(0))
			})

			It("prints the client-credentials user", func() {
				session := helpers.CF("org-users", orgName)
				Eventually(session).Should(Say("Getting users in org %s as %s", orgName, adminUsername))
				Eventually(session).Should(Say("ORG MANAGER"))
				Eventually(session).Should(Say(`\s+%s \(client\)`, clientID))
				Eventually(session).Should(Say("BILLING MANAGER"))
				Eventually(session).Should(Say("ORG AUDITOR"))
				Eventually(session).Should(Exit(0))
			})
		})
	})
})
<|endoftext|>


<commit_before>package workloads

import (
	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

const (
	netstatCommand       = "echo stats | nc -w 1 127.0.0.1 11211"
	defaultMemcachedPath = "workloads/data_caching/memcached/memcached-1.4.25/build/memcached"
	swanPkg              = "github.com/intelsdi-x/swan"
)

// TestMemcachedWithExecutor is an integration test with local executor.
// See README for setup items.
func TestMemcachedWithExecutor(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	// Get optional custom Memcached path from MEMCACHED_PATH.
	memcachedPath := os.Getenv("MEMCACHED_BIN")

	if memcachedPath == "" {
		// If custom path does not exists use default path for built memcached.
		memcachedPath = path.Join(os.Getenv("GOPATH"), "src", swanPkg, defaultMemcachedPath)
	}

	Convey("While using Local Shell in Memcached launcher", t, func() {
		l := executor.NewLocal()
		memcachedLauncher := memcached.New(
			l, memcached.DefaultMemcachedConfig(memcachedPath))

		Convey("When memcached is launched", func() {
			// NOTE: It is needed for memcached to have default port available.
			taskHandle, err := memcachedLauncher.Launch()
			if taskHandle != nil {
				defer taskHandle.Stop()
				defer taskHandle.Clean()
				defer taskHandle.EraseOutput()
			}

			Convey("There should be no error", func() {
				stopErr := taskHandle.Stop()

				So(err, ShouldBeNil)
				So(stopErr, ShouldBeNil)
			})

			Convey("Wait 1 second for memcached to init", func() {
				isTerminated := taskHandle.Wait(1 * time.Second)

				Convey("Memcached should be still running", func() {
					stopErr := taskHandle.Stop()

					// NOTE: Here you will be failing if the memcached
					// can't start because it needs to have default port available.
					So(isTerminated, ShouldBeFalse)

					So(stopErr, ShouldBeNil)
				})

				Convey("When we check the memcached endpoint for stats after 1 second", func() {

					netstatTaskHandle, netstatErr := l.Execute(netstatCommand)
					if netstatTaskHandle != nil {
						defer netstatTaskHandle.Stop()
						defer netstatTaskHandle.Clean()
						defer netstatTaskHandle.EraseOutput()
					}
					Convey("There should be no error", func() {
						taskHandle.Stop()
						netstatTaskHandle.Stop()

						So(netstatErr, ShouldBeNil)

					})

					Convey("When we wait for netstat ", func() {
						netstatTaskHandle.Wait(0)

						Convey("The netstat task should be terminated, the task status should be 0"+
							" and output resultes with a STAT information", func() {

							netstatTaskState := netstatTaskHandle.Status()
							So(netstatTaskState, ShouldEqual, executor.TERMINATED)

							exitCode, err := netstatTaskHandle.ExitCode()
							So(err, ShouldBeNil)
							So(exitCode, ShouldEqual, 0)

							stdoutFile, stdoutErr := netstatTaskHandle.StdoutFile()

							So(stdoutErr, ShouldBeNil)
							So(stdoutFile, ShouldNotBeNil)

							data, readErr := ioutil.ReadAll(stdoutFile)
							So(readErr, ShouldBeNil)
							So(string(data[:]), ShouldStartWith, "STAT")
						})
					})
				})

				Convey("When we stop the memcached task", func() {
					err := taskHandle.Stop()

					Convey("There should be no error", func() {
						So(err, ShouldBeNil)
					})

					Convey("The task should be terminated and the task status "+
						"should be -1 or 0", func() {

						taskState := taskHandle.Status()
						So(taskState, ShouldEqual, executor.TERMINATED)

						exitCode, err := taskHandle.ExitCode()

						So(err, ShouldBeNil)
						// Memcached on CentOS returns 0 (successful code) after SIGTERM.
						So(exitCode, ShouldBeIn, -1, 0)
					})
				})
			})
		})
	})
}
<commit_msg>If task handle is nil, nested convey statements still gets executed (#118)<commit_after>package workloads

import (
	log "github.com/Sirupsen/logrus"
	"github.com/intelsdi-x/swan/pkg/executor"
	"github.com/intelsdi-x/swan/pkg/workloads/memcached"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"
)

const (
	netstatCommand       = "echo stats | nc -w 1 127.0.0.1 11211"
	defaultMemcachedPath = "workloads/data_caching/memcached/memcached-1.4.25/build/memcached"
	swanPkg              = "github.com/intelsdi-x/swan"
)

// TestMemcachedWithExecutor is an integration test with local executor.
// See README for setup items.
func TestMemcachedWithExecutor(t *testing.T) {
	log.SetLevel(log.ErrorLevel)

	// Get optional custom Memcached path from MEMCACHED_PATH.
	memcachedPath := os.Getenv("MEMCACHED_BIN")

	if memcachedPath == "" {
		// If custom path does not exists use default path for built memcached.
		memcachedPath = path.Join(os.Getenv("GOPATH"), "src", swanPkg, defaultMemcachedPath)
	}

	Convey("While using Local Shell in Memcached launcher", t, func() {
		l := executor.NewLocal()
		memcachedLauncher := memcached.New(
			l, memcached.DefaultMemcachedConfig(memcachedPath))

		Convey("When memcached is launched", func() {
			// NOTE: It is needed for memcached to have default port available.
			taskHandle, err := memcachedLauncher.Launch()
			So(taskHandle, ShouldNotBeNil)
			defer taskHandle.Stop()
			defer taskHandle.Clean()
			defer taskHandle.EraseOutput()

			Convey("There should be no error", func() {
				stopErr := taskHandle.Stop()

				So(err, ShouldBeNil)
				So(stopErr, ShouldBeNil)
			})

			Convey("Wait 1 second for memcached to init", func() {
				isTerminated := taskHandle.Wait(1 * time.Second)

				Convey("Memcached should be still running", func() {
					stopErr := taskHandle.Stop()

					// NOTE: Here you will be failing if the memcached
					// can't start because it needs to have default port available.
					So(isTerminated, ShouldBeFalse)

					So(stopErr, ShouldBeNil)
				})

				Convey("When we check the memcached endpoint for stats after 1 second", func() {

					netstatTaskHandle, netstatErr := l.Execute(netstatCommand)
					if netstatTaskHandle != nil {
						defer netstatTaskHandle.Stop()
						defer netstatTaskHandle.Clean()
						defer netstatTaskHandle.EraseOutput()
					}
					Convey("There should be no error", func() {
						taskHandle.Stop()
						netstatTaskHandle.Stop()

						So(netstatErr, ShouldBeNil)

					})

					Convey("When we wait for netstat ", func() {
						netstatTaskHandle.Wait(0)

						Convey("The netstat task should be terminated, the task status should be 0"+
							" and output resultes with a STAT information", func() {

							netstatTaskState := netstatTaskHandle.Status()
							So(netstatTaskState, ShouldEqual, executor.TERMINATED)

							exitCode, err := netstatTaskHandle.ExitCode()
							So(err, ShouldBeNil)
							So(exitCode, ShouldEqual, 0)

							stdoutFile, stdoutErr := netstatTaskHandle.StdoutFile()

							So(stdoutErr, ShouldBeNil)
							So(stdoutFile, ShouldNotBeNil)

							data, readErr := ioutil.ReadAll(stdoutFile)
							So(readErr, ShouldBeNil)
							So(string(data[:]), ShouldStartWith, "STAT")
						})
					})
				})

				Convey("When we stop the memcached task", func() {
					err := taskHandle.Stop()

					Convey("There should be no error", func() {
						So(err, ShouldBeNil)
					})

					Convey("The task should be terminated and the task status "+
						"should be -1 or 0", func() {

						taskState := taskHandle.Status()
						So(taskState, ShouldEqual, executor.TERMINATED)

						exitCode, err := taskHandle.ExitCode()

						So(err, ShouldBeNil)
						// Memcached on CentOS returns 0 (successful code) after SIGTERM.
						So(exitCode, ShouldBeIn, -1, 0)
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	textTempl "text/template"
	"strconv"
)

// type sourceConfig struct {
// 	name           string            "json:name"
// 	upstreamDriver string            "json:type"
// 	upstream       upstreamTarget "json:upstream"
// 	output         []outputType      "json:specFile"
// }

// type upstreamTarget interface {
// 	check() error
// 	version() string
// 	download() error
// }

type versionType struct {
	fmt  string   `json:"fmt"`
	cur  string   `json:"cur"`
	past []string `json:"past"`
}

func stringSubtraction(s string, chunks []string) ([]string, error) {
	var result []string
	remaining := strings.TrimLeft(chunks[0], s)

	for i := 1; i < len(chunks) ; i++{
		if !strings.Contains(remaining, chunks[i]) {
			return []string{}, errors.New(`chunk [%q] not found in [%q]
			original string [%q]
			cut chunks %#v
			parts gotten %#v`,
			chunks[i], remaining, s, chunks, result)
		}
		parts := strings.SplitN(remaining, chunks[i], 2)
		result = append(result, parts[0])
		remaining = parts[1]
	}

	if remaining != "" {
		result = append(result, remaining)
	}

	return result, nil
}

func stringsToInts(s []string) ([]int, error) {
	var results []int
	for _, s := range strings {
		i, err := strconv.Atoi(s)
		if err != nil {
			return []int{}, errors.New("failed - string [%q] is not an int - %v", s, err)
		}
		results = append(results, i)
	}
	return results
}

func (v versionType) nextPossible() (possible []string ) {
	fmtChunks := strings.Split(v.fmt, "%v")
	unprocessedPrefix := v.cur
	var processedSuffix []string

	for len(fmtChunks) > 0 {
		suffix := fmtChunks[len(fmtChunks)-1]
		prefix := strings.TrimSuffix(v.cur, fmtChunks[len(fmtChunks)-1])
		parts := string.Split(prefix, fmtChunks[len(fmtChunks)-2])
		versionNumber := parts[len(parts)-1]

	}
}

type outputType struct {
	active      bool   `json:"active"`
	gitRepo     string `json:"gitRepo"`
	location    string `json:"location"`
	outputID    int    `json:"outputID"`
	outputPrune int    `json:"outputPrune"`
}

type httpResponseTracker struct {
	version versionType `json:"version"`
	driver string `json:"driver"`
	url    string `json:"url"`
}

// func (upstream httpResponseTracker) check() error {

// 	req, _ := http.NewRequest("GET", upstream.Replace(upstream.url), nil)

// }<commit_msg>finished up stuff to format a string, and hopefully print out newer versions<commit_after>package main

import (
	"errors"
	"strconv"
	"strings"
)

// type sourceConfig struct {
// 	name           string            "json:name"
// 	upstreamDriver string            "json:type"
// 	upstream       upstreamTarget "json:upstream"
// 	output         []outputType      "json:specFile"
// }

// type upstreamTarget interface {
// 	check() error
// 	version() string
// 	download() error
// }

type versionType struct {
	fmt  string   `json:"fmt"`
	cur  string   `json:"cur"`
	past []string `json:"past"`
}

func stringSubtraction(s string, chunks []string) ([]string, error) {
	var result []string
	remaining := strings.TrimLeft(chunks[0], s)
	var i int
	if remaining != s {
		i = 1
	}

	for ; i < len(chunks); i++ {
		if !strings.Contains(remaining, chunks[i]) {
			return []string{}, errors.New(`chunk [%q] not found in [%q]
			original string [%q]
			cut chunks %#v
			parts gotten %#v`,
				chunks[i], remaining, s, chunks, result)
		}
		parts := strings.SplitN(remaining, chunks[i], 2)
		result = append(result, parts[0])
		remaining = parts[1]
	}

	if remaining != "" {
		result = append(result, remaining)
	}

	return result, nil
}

func stringsToInts(s []string) ([]int, error) {
	var results []int
	for _, s := range strings {
		i, err := strconv.Atoi(s)
		if err != nil {
			return []int{}, errors.New("failed - string [%q] is not an int - %v", s, err)
		}
		results = append(results, i)
	}
	return nil, results
}

func prefixArray(arr []string, prefix string) (out []string) {
	for _, s := range arr {
		out = append(out, prefix+s)
	}
}

func addFmt(numbers []int, format []string) []string {
	if len(numbers) > 0 {
		return []string{format[0]}
	}
	return prefixArray(addNum(numbers, format[1:]), format[0])
}

func addNum(numbers []int, format []string) (out []string) {
	// if there is no formatting below, simply return the current number
	if len(format) < 1 {
		return []string{
			"0",
			strconf.Itoa(numbers[0]),
		}
	}

	childParts := addFmt(numbers[1:], format)
	out := []string{"0" + childParts[0]}
	out = append(out, strconv.Itoa(numbers[0]+1)+childParts[0])
	out = append(out, prefixArray(childParts[1:], strconv.Itoa(numbers[0]))...)
	return out
}

func (v versionType) nextPossible() ([]string, error) {
	fmtChunks := strings.Split(v.fmt, "%v")

	vPartsString, err := stringSubtraction(v.cur, fmtChunks)
	if err != nil {
		return []string{}, err
	}
	vParts, err := stringsToInts(vPartsString)
	if err != nil {
		return []string{}, err
	}

	var possible []string
	if strings.HasPrefix(v.fmt, "%v") {
		possible = addFmt(vParts, fmtChunks)[1:]
	} else {
		possible = addNum(vParts, fmtChunks)[1:]
	}
	return possible,nil
}

type outputType struct {
	active      bool   `json:"active"`
	gitRepo     string `json:"gitRepo"`
	location    string `json:"location"`
	outputID    int    `json:"outputID"`
	outputPrune int    `json:"outputPrune"`
}

type httpResponseTracker struct {
	version versionType `json:"version"`
	driver  string      `json:"driver"`
	url     string      `json:"url"`
}

// func (upstream httpResponseTracker) check() error {

// 	req, _ := http.NewRequest("GET", upstream.Replace(upstream.url), nil)

// }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"image"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"time"
	"vip/fetch"
)

type UploadResponse struct {
	Url string `json:"url"`
}

type ErrorResponse struct {
	Msg string `json:"error"`
}

type verifyAuth func(http.ResponseWriter, *http.Request)

func (h verifyAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Enable cross-origin requests
	if domain := os.Getenv("ALLOWED_ORIGIN"); domain != "" {
		if origin := r.Header.Get("Origin"); origin == domain {
			w.Header().Set("Access-Control-Allow-Origin", origin)
			w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
			w.Header().Set("Access-Control-Allow-Headers",
				"Accept, Content-Type, Content-Length, Accept-Encoding, X-Vip-Token, Authorization")
		}
	} else {
		auth := r.Header.Get("X-Vip-Token")
		if auth != authToken {
			w.WriteHeader(http.StatusUnauthorized)
			return
		}
	}

	if r.Method == "OPTIONS" {
		return
	}

	h(w, r)
}

func fileKey(bucket string, width int, height int) string {
	seed := rand.New(rand.NewSource(time.Now().UnixNano()))
	key := fmt.Sprintf("%d-%s-%d", seed.Int63(), bucket, time.Now().UnixNano())

	hash := md5.New()
	io.WriteString(hash, key)
	return fmt.Sprintf("%x-%dx%d", hash.Sum(nil), width, height)
}

func handleImageRequest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
	}

	w.Header().Set("Cache-Control", "public, max-age=31536000")

	// Client is checking for a cached URI, assume it is valid
	// and return a 304
	if r.Header.Get("If-Modified-Since") != "" {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	gc := fetch.RequestContext(r)

	var data []byte
	err := cache.Get(gc, gc.CacheKey(), groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	w.Header().Set("Content-Type", http.DetectContentType(data))
	http.ServeContent(w, r, gc.ImageId, time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), bytes.NewReader(data))
}

func handleUpload(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	vars := mux.Vars(r)
	bucket := vars["bucket_id"]
	// Set a hard limit in MB on files
	var limit int64 = 5
	if r.ContentLength > limit<<20 {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusRequestEntityTooLarge)
		json.NewEncoder(w).Encode(ErrorResponse{
			Msg: fmt.Sprintf("The file size limit is %dMB", limit),
		})
		return
	} else if r.ContentLength == 0 {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusBadRequest)
		json.NewEncoder(w).Encode(ErrorResponse{
			Msg: fmt.Sprintf("File must have size greater than 0"),
		})
		return
	}

	raw, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	data := bytes.NewReader(raw)

	image, _, err := image.Decode(data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	width := image.Bounds().Size().X
	height := image.Bounds().Size().Y

	key := fileKey(bucket, width, height)

	data.Seek(0, 0)

	err = storage.PutReader(bucket, key, data,
		r.ContentLength, r.Header.Get("Content-Type"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	uri := r.URL

	if r.URL.Host == "" {
		uri.Host = os.Getenv("URI_HOSTNAME")
		if secure {
			uri.Scheme = "https"
		} else {
			uri.Scheme = "http"
		}
	}

	uri.Path = fmt.Sprintf("%s/%s", bucket, key)

	w.Header().Set("Content-Type", "application/json")

	w.WriteHeader(http.StatusCreated)
	json.NewEncoder(w).Encode(UploadResponse{
		Url: uri.String(),
	})
}

func handlePing(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "pong")
}
<commit_msg>Close the body in the right spot<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/golang/groupcache"
	"github.com/gorilla/mux"
	"image"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"time"
	"vip/fetch"
)

type UploadResponse struct {
	Url string `json:"url"`
}

type ErrorResponse struct {
	Msg string `json:"error"`
}

type verifyAuth func(http.ResponseWriter, *http.Request)

func (h verifyAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Enable cross-origin requests
	if domain := os.Getenv("ALLOWED_ORIGIN"); domain != "" {
		if origin := r.Header.Get("Origin"); origin == domain {
			w.Header().Set("Access-Control-Allow-Origin", origin)
			w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
			w.Header().Set("Access-Control-Allow-Headers",
				"Accept, Content-Type, Content-Length, Accept-Encoding, X-Vip-Token, Authorization")
		}
	} else {
		auth := r.Header.Get("X-Vip-Token")
		if auth != authToken {
			w.WriteHeader(http.StatusUnauthorized)
			return
		}
	}

	if r.Method == "OPTIONS" {
		return
	}

	h(w, r)
}

func fileKey(bucket string, width int, height int) string {
	seed := rand.New(rand.NewSource(time.Now().UnixNano()))
	key := fmt.Sprintf("%d-%s-%d", seed.Int63(), bucket, time.Now().UnixNano())

	hash := md5.New()
	io.WriteString(hash, key)
	return fmt.Sprintf("%x-%dx%d", hash.Sum(nil), width, height)
}

func handleImageRequest(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusMethodNotAllowed)
	}

	w.Header().Set("Cache-Control", "public, max-age=31536000")

	// Client is checking for a cached URI, assume it is valid
	// and return a 304
	if r.Header.Get("If-Modified-Since") != "" {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	gc := fetch.RequestContext(r)

	var data []byte
	err := cache.Get(gc, gc.CacheKey(), groupcache.AllocatingByteSliceSink(&data))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	w.Header().Set("Content-Type", http.DetectContentType(data))
	http.ServeContent(w, r, gc.ImageId, time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), bytes.NewReader(data))
}

func handleUpload(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		return
	}

	vars := mux.Vars(r)
	bucket := vars["bucket_id"]
	// Set a hard limit in MB on files
	var limit int64 = 5
	if r.ContentLength > limit<<20 {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusRequestEntityTooLarge)
		json.NewEncoder(w).Encode(ErrorResponse{
			Msg: fmt.Sprintf("The file size limit is %dMB", limit),
		})
		return
	} else if r.ContentLength == 0 {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusBadRequest)
		json.NewEncoder(w).Encode(ErrorResponse{
			Msg: fmt.Sprintf("File must have size greater than 0"),
		})
		return
	}

	raw, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	r.Body.Close()

	data := bytes.NewReader(raw)

	image, _, err := image.Decode(data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	width := image.Bounds().Size().X
	height := image.Bounds().Size().Y

	key := fileKey(bucket, width, height)

	data.Seek(0, 0)

	err = storage.PutReader(bucket, key, data,
		r.ContentLength, r.Header.Get("Content-Type"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	uri := r.URL

	if r.URL.Host == "" {
		uri.Host = os.Getenv("URI_HOSTNAME")
		if secure {
			uri.Scheme = "https"
		} else {
			uri.Scheme = "http"
		}
	}

	uri.Path = fmt.Sprintf("%s/%s", bucket, key)

	w.Header().Set("Content-Type", "application/json")

	w.WriteHeader(http.StatusCreated)
	json.NewEncoder(w).Encode(UploadResponse{
		Url: uri.String(),
	})
}

func handlePing(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "pong")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/joatmon08/docker-consul-handler/lib"
	"github.com/joatmon08/docker-consul-handler/runner"
)

func main() {
	ansible_runner, ok := os.LookupEnv("CONSUL_HANDLER_RUNNER")
	if !ok {
		panic("Environment variable CONSUL_HANDER_RUNNER is not set!")
	}

	scanner := bufio.NewScanner(os.Stdin)
	scanner.Split(bufio.ScanWords)

	runner_client := runner.Client{URL: ansible_runner}

	for scanner.Scan() {
		input := scanner.Bytes()
		networkID, err := lib.GetNewestNetwork(input)
		if err != nil {
			fmt.Println(err.Error())
			continue
		}
		fmt.Printf("Newest Network: %s \n", networkID)
		if len(networkID) > 0 {
			network, err := lib.GetNetworkDetails(networkID)
			if err != nil {
				fmt.Println(err.Error())
				continue
			}
			fmt.Printf("Network ID %s, Network Name: %s \n", networkID, network)
			play_body := []byte(fmt.Sprintf("{\"container_network\": \"%s\"}", network))
			if _, err = runner_client.Play(play_body); err != nil {
				fmt.Printf("Network %s not created", network)
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Ignore bridge network<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"

	"github.com/joatmon08/docker-consul-handler/lib"
	"github.com/joatmon08/docker-consul-handler/runner"
)

func main() {
	ansible_runner, ok := os.LookupEnv("CONSUL_HANDLER_RUNNER")
	if !ok {
		panic("Environment variable CONSUL_HANDER_RUNNER is not set!")
	}

	scanner := bufio.NewScanner(os.Stdin)
	scanner.Split(bufio.ScanWords)

	runner_client := runner.Client{URL: ansible_runner}

	for scanner.Scan() {
		input := scanner.Bytes()
		networkID, err := lib.GetNewestNetwork(input)
		if err != nil {
			fmt.Println(err.Error())
			continue
		}
		fmt.Printf("Newest Network: %s \n", networkID)
		if len(networkID) > 0 {
			network, err := lib.GetNetworkDetails(networkID)
			if err != nil {
				fmt.Println(err.Error())
				continue
			}
			if network == "bridge" {
				continue
			}
			fmt.Printf("Network ID %s, Network Name: %s \n", networkID, network)
			play_body := []byte(fmt.Sprintf("{\"container_network\": \"%s\"}", network))
			if _, err = runner_client.Play(play_body); err != nil {
				fmt.Printf("Network %s not created", network)
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package gost

import (
	"bufio"
	"crypto/tls"
	"net"
	"net/url"

	"github.com/ginuerzh/gosocks4"
	"github.com/ginuerzh/gosocks5"
	"github.com/go-log/log"
)

// Handler is a proxy server handler
type Handler interface {
	Handle(net.Conn)
}

// HandlerOptions describes the options for Handler.
type HandlerOptions struct {
	Addr      string
	Chain     *Chain
	Users     []*url.Userinfo
	TLSConfig *tls.Config
	Whitelist *Permissions
	Blacklist *Permissions
}

// HandlerOption allows a common way to set handler options.
type HandlerOption func(opts *HandlerOptions)

// AddrHandlerOption sets the Addr option of HandlerOptions.
func AddrHandlerOption(addr string) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Addr = addr
	}
}

// ChainHandlerOption sets the Chain option of HandlerOptions.
func ChainHandlerOption(chain *Chain) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Chain = chain
	}
}

// UsersHandlerOption sets the Users option of HandlerOptions.
func UsersHandlerOption(users ...*url.Userinfo) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Users = users
	}
}

// TLSConfigHandlerOption sets the TLSConfig option of HandlerOptions.
func TLSConfigHandlerOption(config *tls.Config) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.TLSConfig = config
	}
}

// WhitelistHandlerOption sets the Whitelist option of HandlerOptions.
func WhitelistHandlerOption(whitelist *Permissions) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Whitelist = whitelist
	}
}

// BlacklistHandlerOption sets the Blacklist option of HandlerOptions.
func BlacklistHandlerOption(blacklist *Permissions) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Blacklist = blacklist
	}
}

type autoHandler struct {
	options []HandlerOption
}

// AutoHandler creates a server Handler for auto proxy server.
func AutoHandler(opts ...HandlerOption) Handler {
	h := &autoHandler{
		options: opts,
	}
	return h
}

func (h *autoHandler) Handle(conn net.Conn) {
	br := bufio.NewReader(conn)
	b, err := br.Peek(1)
	if err != nil {
		log.Log(err)
		conn.Close()
		return
	}

	cc := &bufferdConn{Conn: conn, br: br}
	switch b[0] {
	case gosocks4.Ver4:
		cc.Close()
		return // SOCKS4(a) does not suppport authentication method, so we ignore it for security reason.
	case gosocks5.Ver5:
		SOCKS5Handler(h.options...).Handle(cc)
	default: // http
		HTTPHandler(h.options...).Handle(cc)
	}
}

type bufferdConn struct {
	net.Conn
	br *bufio.Reader
}

func (c *bufferdConn) Read(b []byte) (int, error) {
	return c.br.Read(b)
}
<commit_msg>fix #225<commit_after>package gost

import (
	"bufio"
	"crypto/tls"
	"net"
	"net/url"

	"github.com/ginuerzh/gosocks4"
	"github.com/ginuerzh/gosocks5"
	"github.com/go-log/log"
)

// Handler is a proxy server handler
type Handler interface {
	Handle(net.Conn)
}

// HandlerOptions describes the options for Handler.
type HandlerOptions struct {
	Addr      string
	Chain     *Chain
	Users     []*url.Userinfo
	TLSConfig *tls.Config
	Whitelist *Permissions
	Blacklist *Permissions
}

// HandlerOption allows a common way to set handler options.
type HandlerOption func(opts *HandlerOptions)

// AddrHandlerOption sets the Addr option of HandlerOptions.
func AddrHandlerOption(addr string) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Addr = addr
	}
}

// ChainHandlerOption sets the Chain option of HandlerOptions.
func ChainHandlerOption(chain *Chain) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Chain = chain
	}
}

// UsersHandlerOption sets the Users option of HandlerOptions.
func UsersHandlerOption(users ...*url.Userinfo) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Users = users
	}
}

// TLSConfigHandlerOption sets the TLSConfig option of HandlerOptions.
func TLSConfigHandlerOption(config *tls.Config) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.TLSConfig = config
	}
}

// WhitelistHandlerOption sets the Whitelist option of HandlerOptions.
func WhitelistHandlerOption(whitelist *Permissions) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Whitelist = whitelist
	}
}

// BlacklistHandlerOption sets the Blacklist option of HandlerOptions.
func BlacklistHandlerOption(blacklist *Permissions) HandlerOption {
	return func(opts *HandlerOptions) {
		opts.Blacklist = blacklist
	}
}

type autoHandler struct {
	options []HandlerOption
}

// AutoHandler creates a server Handler for auto proxy server.
func AutoHandler(opts ...HandlerOption) Handler {
	h := &autoHandler{
		options: opts,
	}
	return h
}

func (h *autoHandler) Handle(conn net.Conn) {
	br := bufio.NewReader(conn)
	b, err := br.Peek(1)
	if err != nil {
		log.Log(err)
		conn.Close()
		return
	}

	cc := &bufferdConn{Conn: conn, br: br}
	switch b[0] {
	case gosocks4.Ver4:
		options := &HandlerOptions{}
		for _, opt := range h.options {
			opt(options)
		}
		// SOCKS4(a) does not suppport authentication method,
		// so we ignore it when credentials are specified for security reason.
		if len(options.Users) > 0 {
			cc.Close()
			return
		}
		h := &socks4Handler{options}
		h.Handle(cc)
	case gosocks5.Ver5:
		SOCKS5Handler(h.options...).Handle(cc)
	default: // http
		HTTPHandler(h.options...).Handle(cc)
	}
}

type bufferdConn struct {
	net.Conn
	br *bufio.Reader
}

func (c *bufferdConn) Read(b []byte) (int, error) {
	return c.br.Read(b)
}
<|endoftext|>


<commit_before>package vmware

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"math/rand"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const BuilderId = "mitchellh.vmware"

type Builder struct {
	config config
	driver Driver
	runner multistep.Runner
}

type config struct {
	DiskName        string            `mapstructure:"vmdk_name"`
	DiskSize        uint              `mapstructure:"disk_size"`
	GuestOSType     string            `mapstructure:"guest_os_type"`
	ISOMD5          string            `mapstructure:"iso_md5"`
	ISOUrl          string            `mapstructure:"iso_url"`
	VMName          string            `mapstructure:"vm_name"`
	OutputDir       string            `mapstructure:"output_directory"`
	HTTPDir         string            `mapstructure:"http_directory"`
	HTTPPortMin     uint              `mapstructure:"http_port_min"`
	HTTPPortMax     uint              `mapstructure:"http_port_max"`
	BootCommand     []string          `mapstructure:"boot_command"`
	BootWait        time.Duration     ``
	ShutdownCommand string            `mapstructure:"shutdown_command"`
	ShutdownTimeout time.Duration     ``
	SSHUser         string            `mapstructure:"ssh_username"`
	SSHPassword     string            `mapstructure:"ssh_password"`
	SSHPort         uint              `mapstructure:"ssh_port"`
	SSHWaitTimeout  time.Duration     ``
	VMXData         map[string]string `mapstructure:"vmx_data"`
	VNCPortMin      uint              `mapstructure:"vnc_port_min"`
	VNCPortMax      uint              `mapstructure:"vnc_port_max"`

	PackerDebug bool `mapstructure:"packer_debug"`

	RawBootWait        string `mapstructure:"boot_wait"`
	RawShutdownTimeout string `mapstructure:"shutdown_timeout"`
	RawSSHWaitTimeout  string `mapstructure:"ssh_wait_timeout"`
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	if b.config.DiskName == "" {
		b.config.DiskName = "disk"
	}

	if b.config.DiskSize == 0 {
		b.config.DiskSize = 40000
	}

	if b.config.GuestOSType == "" {
		b.config.GuestOSType = "other"
	}

	if b.config.VMName == "" {
		b.config.VMName = "packer"
	}

	if b.config.HTTPPortMin == 0 {
		b.config.HTTPPortMin = 8000
	}

	if b.config.HTTPPortMax == 0 {
		b.config.HTTPPortMax = 9000
	}

	if b.config.VNCPortMin == 0 {
		b.config.VNCPortMin = 5900
	}

	if b.config.VNCPortMax == 0 {
		b.config.VNCPortMax = 6000
	}

	if b.config.OutputDir == "" {
		b.config.OutputDir = "vmware"
	}

	if b.config.SSHPort == 0 {
		b.config.SSHPort = 22
	}

	// Accumulate any errors
	var err error
	errs := make([]error, 0)

	if b.config.HTTPPortMin > b.config.HTTPPortMax {
		errs = append(errs, errors.New("http_port_min must be less than http_port_max"))
	}

	if b.config.ISOMD5 == "" {
		errs = append(errs, errors.New("Due to large file sizes, an iso_md5 is required"))
	} else {
		b.config.ISOMD5 = strings.ToLower(b.config.ISOMD5)
	}

	if b.config.ISOUrl == "" {
		errs = append(errs, errors.New("An iso_url must be specified."))
	} else {
		url, err := url.Parse(b.config.ISOUrl)
		if err != nil {
			errs = append(errs, fmt.Errorf("iso_url is not a valid URL: %s", err))
		} else {
			if url.Scheme == "" {
				url.Scheme = "file"
			}

			if url.Scheme == "file" {
				if _, err := os.Stat(b.config.ISOUrl); err != nil {
					errs = append(errs, fmt.Errorf("iso_url points to bad file: %s", err))
				}
			} else {
				supportedSchemes := []string{"file", "http", "https"}
				scheme := strings.ToLower(url.Scheme)

				found := false
				for _, supported := range supportedSchemes {
					if scheme == supported {
						found = true
						break
					}
				}

				if !found {
					errs = append(errs, fmt.Errorf("Unsupported URL scheme in iso_url: %s", scheme))
				}
			}
		}

		if len(errs) == 0 {
			// Put the URL back together since we may have modified it
			b.config.ISOUrl = url.String()
		}
	}

	if _, err := os.Stat(b.config.OutputDir); err == nil {
		errs = append(errs, errors.New("Output directory already exists. It must not exist."))
	}

	if b.config.SSHUser == "" {
		errs = append(errs, errors.New("An ssh_username must be specified."))
	}

	if b.config.RawBootWait != "" {
		b.config.BootWait, err = time.ParseDuration(b.config.RawBootWait)
		if err != nil {
			errs = append(errs, fmt.Errorf("Failed parsing boot_wait: %s", err))
		}
	}

	if b.config.RawShutdownTimeout == "" {
		b.config.RawShutdownTimeout = "5m"
	}

	b.config.ShutdownTimeout, err = time.ParseDuration(b.config.RawShutdownTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing shutdown_timeout: %s", err))
	}

	if b.config.RawSSHWaitTimeout == "" {
		b.config.RawSSHWaitTimeout = "20m"
	}

	b.config.SSHWaitTimeout, err = time.ParseDuration(b.config.RawSSHWaitTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err))
	}

	if b.config.VNCPortMin > b.config.VNCPortMax {
		errs = append(errs, fmt.Errorf("vnc_port_min must be less than vnc_port_max"))
	}

	b.driver, err = b.newDriver()
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed creating VMware driver: %s", err))
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	steps := []multistep.Step{
		&stepDownloadISO{},
		&stepPrepareOutputDir{},
		&stepCreateDisk{},
		&stepCreateVMX{},
		&stepHTTPServer{},
		&stepConfigureVNC{},
		&stepRun{},
		&stepTypeBootCommand{},
		&stepWaitForSSH{},
		&stepProvision{},
		&stepShutdown{},
	}

	// Setup the state bag
	state := make(map[string]interface{})
	state["cache"] = cache
	state["config"] = &b.config
	state["driver"] = b.driver
	state["hook"] = hook
	state["ui"] = ui

	// Run!
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	// If there was an error, return that
	if rawErr, ok := state["error"]; ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state[multistep.StateCancelled]; ok {
		return nil, errors.New("Build was cancelled.")
	}

	if _, ok := state[multistep.StateHalted]; ok {
		return nil, errors.New("Build was halted.")
	}

	// Compile the artifact list
	files := make([]string, 0, 10)
	visit := func(path string, info os.FileInfo, err error) error {
		files = append(files, path)
		return err
	}

	if err := filepath.Walk(b.config.OutputDir, visit); err != nil {
		return nil, err
	}

	return &Artifact{b.config.OutputDir, files}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}

func (b *Builder) newDriver() (Driver, error) {
	fusionAppPath := "/Applications/VMware Fusion.app"
	driver := &Fusion5Driver{fusionAppPath}
	if err := driver.Verify(); err != nil {
		return nil, err
	}

	return driver, nil
}
<commit_msg>builder/vmware: properly test file URLs<commit_after>package vmware

import (
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"math/rand"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const BuilderId = "mitchellh.vmware"

type Builder struct {
	config config
	driver Driver
	runner multistep.Runner
}

type config struct {
	DiskName        string            `mapstructure:"vmdk_name"`
	DiskSize        uint              `mapstructure:"disk_size"`
	GuestOSType     string            `mapstructure:"guest_os_type"`
	ISOMD5          string            `mapstructure:"iso_md5"`
	ISOUrl          string            `mapstructure:"iso_url"`
	VMName          string            `mapstructure:"vm_name"`
	OutputDir       string            `mapstructure:"output_directory"`
	HTTPDir         string            `mapstructure:"http_directory"`
	HTTPPortMin     uint              `mapstructure:"http_port_min"`
	HTTPPortMax     uint              `mapstructure:"http_port_max"`
	BootCommand     []string          `mapstructure:"boot_command"`
	BootWait        time.Duration     ``
	ShutdownCommand string            `mapstructure:"shutdown_command"`
	ShutdownTimeout time.Duration     ``
	SSHUser         string            `mapstructure:"ssh_username"`
	SSHPassword     string            `mapstructure:"ssh_password"`
	SSHPort         uint              `mapstructure:"ssh_port"`
	SSHWaitTimeout  time.Duration     ``
	VMXData         map[string]string `mapstructure:"vmx_data"`
	VNCPortMin      uint              `mapstructure:"vnc_port_min"`
	VNCPortMax      uint              `mapstructure:"vnc_port_max"`

	PackerDebug bool `mapstructure:"packer_debug"`

	RawBootWait        string `mapstructure:"boot_wait"`
	RawShutdownTimeout string `mapstructure:"shutdown_timeout"`
	RawSSHWaitTimeout  string `mapstructure:"ssh_wait_timeout"`
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	if b.config.DiskName == "" {
		b.config.DiskName = "disk"
	}

	if b.config.DiskSize == 0 {
		b.config.DiskSize = 40000
	}

	if b.config.GuestOSType == "" {
		b.config.GuestOSType = "other"
	}

	if b.config.VMName == "" {
		b.config.VMName = "packer"
	}

	if b.config.HTTPPortMin == 0 {
		b.config.HTTPPortMin = 8000
	}

	if b.config.HTTPPortMax == 0 {
		b.config.HTTPPortMax = 9000
	}

	if b.config.VNCPortMin == 0 {
		b.config.VNCPortMin = 5900
	}

	if b.config.VNCPortMax == 0 {
		b.config.VNCPortMax = 6000
	}

	if b.config.OutputDir == "" {
		b.config.OutputDir = "vmware"
	}

	if b.config.SSHPort == 0 {
		b.config.SSHPort = 22
	}

	// Accumulate any errors
	var err error
	errs := make([]error, 0)

	if b.config.HTTPPortMin > b.config.HTTPPortMax {
		errs = append(errs, errors.New("http_port_min must be less than http_port_max"))
	}

	if b.config.ISOMD5 == "" {
		errs = append(errs, errors.New("Due to large file sizes, an iso_md5 is required"))
	} else {
		b.config.ISOMD5 = strings.ToLower(b.config.ISOMD5)
	}

	if b.config.ISOUrl == "" {
		errs = append(errs, errors.New("An iso_url must be specified."))
	} else {
		url, err := url.Parse(b.config.ISOUrl)
		if err != nil {
			errs = append(errs, fmt.Errorf("iso_url is not a valid URL: %s", err))
		} else {
			if url.Scheme == "" {
				url.Scheme = "file"
			}

			if url.Scheme == "file" {
				if _, err := os.Stat(url.Path); err != nil {
					errs = append(errs, fmt.Errorf("iso_url points to bad file: %s", err))
				}
			} else {
				supportedSchemes := []string{"file", "http", "https"}
				scheme := strings.ToLower(url.Scheme)

				found := false
				for _, supported := range supportedSchemes {
					if scheme == supported {
						found = true
						break
					}
				}

				if !found {
					errs = append(errs, fmt.Errorf("Unsupported URL scheme in iso_url: %s", scheme))
				}
			}
		}

		if len(errs) == 0 {
			// Put the URL back together since we may have modified it
			b.config.ISOUrl = url.String()
		}
	}

	if _, err := os.Stat(b.config.OutputDir); err == nil {
		errs = append(errs, errors.New("Output directory already exists. It must not exist."))
	}

	if b.config.SSHUser == "" {
		errs = append(errs, errors.New("An ssh_username must be specified."))
	}

	if b.config.RawBootWait != "" {
		b.config.BootWait, err = time.ParseDuration(b.config.RawBootWait)
		if err != nil {
			errs = append(errs, fmt.Errorf("Failed parsing boot_wait: %s", err))
		}
	}

	if b.config.RawShutdownTimeout == "" {
		b.config.RawShutdownTimeout = "5m"
	}

	b.config.ShutdownTimeout, err = time.ParseDuration(b.config.RawShutdownTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing shutdown_timeout: %s", err))
	}

	if b.config.RawSSHWaitTimeout == "" {
		b.config.RawSSHWaitTimeout = "20m"
	}

	b.config.SSHWaitTimeout, err = time.ParseDuration(b.config.RawSSHWaitTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_wait_timeout: %s", err))
	}

	if b.config.VNCPortMin > b.config.VNCPortMax {
		errs = append(errs, fmt.Errorf("vnc_port_min must be less than vnc_port_max"))
	}

	b.driver, err = b.newDriver()
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed creating VMware driver: %s", err))
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	steps := []multistep.Step{
		&stepDownloadISO{},
		&stepPrepareOutputDir{},
		&stepCreateDisk{},
		&stepCreateVMX{},
		&stepHTTPServer{},
		&stepConfigureVNC{},
		&stepRun{},
		&stepTypeBootCommand{},
		&stepWaitForSSH{},
		&stepProvision{},
		&stepShutdown{},
	}

	// Setup the state bag
	state := make(map[string]interface{})
	state["cache"] = cache
	state["config"] = &b.config
	state["driver"] = b.driver
	state["hook"] = hook
	state["ui"] = ui

	// Run!
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	// If there was an error, return that
	if rawErr, ok := state["error"]; ok {
		return nil, rawErr.(error)
	}

	// If we were interrupted or cancelled, then just exit.
	if _, ok := state[multistep.StateCancelled]; ok {
		return nil, errors.New("Build was cancelled.")
	}

	if _, ok := state[multistep.StateHalted]; ok {
		return nil, errors.New("Build was halted.")
	}

	// Compile the artifact list
	files := make([]string, 0, 10)
	visit := func(path string, info os.FileInfo, err error) error {
		files = append(files, path)
		return err
	}

	if err := filepath.Walk(b.config.OutputDir, visit); err != nil {
		return nil, err
	}

	return &Artifact{b.config.OutputDir, files}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}

func (b *Builder) newDriver() (Driver, error) {
	fusionAppPath := "/Applications/VMware Fusion.app"
	driver := &Fusion5Driver{fusionAppPath}
	if err := driver.Verify(); err != nil {
		return nil, err
	}

	return driver, nil
}
<|endoftext|>


<commit_before>package term

import (
	"syscall"
	"unsafe"
)

const (
	getTermios = syscall.TCGETS
	setTermios = syscall.TCSETS
)

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd uintptr) (*State, error) {
	var oldState State
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, getTermios, uintptr(unsafe.Pointer(&oldState.termios))); err != 0 {
		return nil, err
	}

	newState := oldState.termios

	newState.Iflag &^= (syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON)
	newState.Oflag &^= syscall.OPOST
	newState.Lflag &^= (syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.EXTEN)
	newState.Cflag &^= (syscall.CSIZE | syscall.PARENB)
	newState.Cflag |= syscall.CS8

	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, setTermios, uintptr(unsafe.Pointer(&newState))); err != 0 {
		return nil, err
	}
	return &oldState, nil
}
<commit_msg>Fix syscall name.<commit_after>package term

import (
	"syscall"
	"unsafe"
)

const (
	getTermios = syscall.TCGETS
	setTermios = syscall.TCSETS
)

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

// MakeRaw put the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
func MakeRaw(fd uintptr) (*State, error) {
	var oldState State
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, getTermios, uintptr(unsafe.Pointer(&oldState.termios))); err != 0 {
		return nil, err
	}

	newState := oldState.termios

	newState.Iflag &^= (syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON)
	newState.Oflag &^= syscall.OPOST
	newState.Lflag &^= (syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN)
	newState.Cflag &^= (syscall.CSIZE | syscall.PARENB)
	newState.Cflag |= syscall.CS8

	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, setTermios, uintptr(unsafe.Pointer(&newState))); err != 0 {
		return nil, err
	}
	return &oldState, nil
}
<|endoftext|>


<commit_before>package gobreak

import "time"

type runFunc func() error
type fallbackFunc func(error) error

const (
	Success   = "success"
	ErrReject = "reject"
	ErrFail   = "fail"
)

// Do runs your function in a synchronous manner, blocking until either your function succeeds
// or an error is returned, including circuit errors
func Do(name string, run runFunc, fall fallbackFunc) error {
	c := getCircuit(name)

	done, err := c.Allow()
	if err != nil {
		requests.WithLabelValues(name, ErrReject).Inc()
		if fall != nil {
			err = fall(err)
		}
		return err
	}

	now := time.Now()

	err = run()

	elapsed := time.Now().Sub(now).Seconds()
	requestLatencyHistogram.WithLabelValues(name).Observe(elapsed)

	if err != nil {
		done(false)
		requests.WithLabelValues(name, ErrFail).Inc()
		if fall != nil {
			err = fall(err)
		}
		return err
	}

	done(true)
	requests.WithLabelValues(name, Success).Inc()
	return nil
}
<commit_msg>Add recovery and improve errType<commit_after>package gobreak

import "time"

type runFunc func() error
type fallbackFunc func(error) error

const (
	Success     = "success"
	ErrReject   = "reject"
	ErrFallBack = "fallback"
	ErrFail     = "fail"
	ErrPanic    = "panic"
)

// Do runs your function in a synchronous manner, blocking until either your function succeeds
// or an error is returned, including circuit errors
func Do(name string, run runFunc, fall fallbackFunc) error {
	errorType := Success
	// obtain circuit by name
	c := getCircuit(name)

	// ask circuit allow run or not
	done, err := c.Allow()
	if err != nil {
		errorType = ErrReject
		if fall != nil {
			errorType = ErrFallBack
			err = fall(err)
		}
		requests.WithLabelValues(name, errorType).Inc()
		return err
	}

	now := time.Now()

	// process run function
	err = run()
	// try recover when run function panics
	defer func() {
		e := recover()
		if e != nil {
			done(false)
			requests.WithLabelValues(name, ErrPanic).Inc()
			panic(e)
		}
	}()

	elapsed := time.Now().Sub(now).Seconds()
	requestLatencyHistogram.WithLabelValues(name).Observe(elapsed)

	// report run results to circuit
	done(err == nil)
	if err != nil {
		errorType = ErrFail
		if fall != nil {
			errorType = ErrFallBack
			err = fall(err)
		}
	}

	requests.WithLabelValues(name, errorType).Inc()
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package gobuild // import "github.com/caixw/gobuild"

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

// Build 执行热编译服务。
//
// logs 日志输出通道
// mainFiles 为 go build 最后的文件参数，可以为空，表示当前目录；
// outputName 指定可执行文件输出的文件路径，为空表示默认值；
// exts 指定监视的文件扩展名，为空表示不监视任何文件，* 表示监视所有文件；
// recursive 是否监视子目录；
// appArgs 传递给程序的参数；
// dir 表示需要监视的目录，至少指定一个目录，第一个目录被当作子目录，将编译其下的文件。
func Build(logs chan *Log, mainFiles, outputName, exts string, recursive bool, appArgs string, dir ...string) error {
	if len(dir) < 1 {
		return errors.New("参数 dir 至少指定一个")
	}
	wd, err := filepath.Abs(dir[0])
	if err != nil {
		return err
	}
	dir[0] = wd

	appName, err := getAppName(outputName, wd)
	if err != nil {
		return err
	}

	// 初始化 goCmd 的参数
	args := []string{"build", "-o", appName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	b := &builder{
		exts:      getExts(exts),
		appName:   appName,
		appArgs:   splitArgs(appArgs),
		goCmdArgs: args,
		logs:      logs,
	}

	// 输出提示信息
	logs <- &Log{
		Type:    LogTypeInfo,
		Message: fmt.Sprint("给程序传递了以下参数：", b.appArgs),
	}

	// 提示扩展名
	switch {
	case len(b.exts) == 0: // 允许不监视任意文件，但输出一信息来警告
		logs <- &Log{
			Type:    LogTypeWarn,
			Message: "将 ext 设置为空值，意味着不监视任何文件的改变！",
		}
	case len(b.exts) > 0:
		logs <- &Log{
			Type:    LogTypeInfo,
			Message: fmt.Sprint("系统将监视以下类型的文件:", b.exts),
		}
	}

	// 提示 appName
	logs <- &Log{
		Type:    LogTypeInfo,
		Message: fmt.Sprint("输出文件为:", b.appName),
	}

	paths, err := recursivePaths(recursive, dir)
	if err != nil {
		return err
	}
	w, err := b.initWatcher(paths)
	if err != nil {
		return err
	}
	defer w.Close()

	b.watch(w)
	go b.build()

	<-make(chan bool)
	return nil
}

func splitArgs(args string) []string {
	ret := make([]string, 0, 10)
	var state byte
	var start, index int

	for index = 0; index < len(args); index++ {
		b := args[index]
		if b == ' ' {
			if state != ' ' {
				ret = append(ret, args[start:index])
				state = ' '
			}
			start = index + 1
			continue
		}

		if b == '=' {
			if state != '=' {
				ret = append(ret, args[start:index])
				state = '='
			}
			start = index + 1
			continue
		}

		state = 0
	} // end for

	if start < len(args) {
		ret = append(ret, args[start:len(args)])
	}

	return ret
}

// 根据 recursive 值确定是否递归查找 paths 每个目录下的子目录。
func recursivePaths(recursive bool, paths []string) ([]string, error) {
	if !recursive {
		return paths, nil
	}

	ret := []string{}

	walk := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if fi.IsDir() && strings.Index(path, "/.") < 0 {
			ret = append(ret, path)
		}
		return nil
	}

	for _, path := range paths {
		if err := filepath.Walk(path, walk); err != nil {
			return nil, err
		}
	}

	return ret, nil
}

// 将 extString 分解成数组，并清理掉无用的内容，比如空字符串
func getExts(extString string) []string {
	exts := strings.Split(extString, ",")
	ret := make([]string, 0, len(exts))

	for _, ext := range exts {
		ext = strings.TrimSpace(ext)

		if len(ext) == 0 {
			continue
		}
		if ext[0] != '.' {
			ext = "." + ext
		}
		ret = append(ret, ext)
	}

	return ret
}

func getAppName(outputName, wd string) (string, error) {
	if outputName == "" {
		outputName = filepath.Base(wd)
	}
	if runtime.GOOS == "windows" && !strings.HasSuffix(outputName, ".exe") {
		outputName += ".exe"
	}

	// 没有分隔符，表示仅有一个文件名，需要加上 wd
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = filepath.Join(wd, outputName)
	}

	// 转成绝对路径
	outputName, err := filepath.Abs(outputName)
	if err != nil {
		return "", err
	}

	return outputName, nil
}
<commit_msg>采用 GOEXE 环境变量判断文件的后缀名<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package gobuild // import "github.com/caixw/gobuild"

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

// Build 执行热编译服务。
//
// logs 日志输出通道
// mainFiles 为 go build 最后的文件参数，可以为空，表示当前目录；
// outputName 指定可执行文件输出的文件路径，为空表示默认值；
// exts 指定监视的文件扩展名，为空表示不监视任何文件，* 表示监视所有文件；
// recursive 是否监视子目录；
// appArgs 传递给程序的参数；
// dir 表示需要监视的目录，至少指定一个目录，第一个目录被当作子目录，将编译其下的文件。
func Build(logs chan *Log, mainFiles, outputName, exts string, recursive bool, appArgs string, dir ...string) error {
	if len(dir) < 1 {
		return errors.New("参数 dir 至少指定一个")
	}
	wd, err := filepath.Abs(dir[0])
	if err != nil {
		return err
	}
	dir[0] = wd

	appName, err := getAppName(outputName, wd)
	if err != nil {
		return err
	}

	// 初始化 goCmd 的参数
	args := []string{"build", "-o", appName}
	if len(mainFiles) > 0 {
		args = append(args, mainFiles)
	}

	b := &builder{
		exts:      getExts(exts),
		appName:   appName,
		appArgs:   splitArgs(appArgs),
		goCmdArgs: args,
		logs:      logs,
	}

	// 输出提示信息
	logs <- &Log{
		Type:    LogTypeInfo,
		Message: fmt.Sprint("给程序传递了以下参数：", b.appArgs),
	}

	// 提示扩展名
	switch {
	case len(b.exts) == 0: // 允许不监视任意文件，但输出一信息来警告
		logs <- &Log{
			Type:    LogTypeWarn,
			Message: "将 ext 设置为空值，意味着不监视任何文件的改变！",
		}
	case len(b.exts) > 0:
		logs <- &Log{
			Type:    LogTypeInfo,
			Message: fmt.Sprint("系统将监视以下类型的文件:", b.exts),
		}
	}

	// 提示 appName
	logs <- &Log{
		Type:    LogTypeInfo,
		Message: fmt.Sprint("输出文件为:", b.appName),
	}

	paths, err := recursivePaths(recursive, dir)
	if err != nil {
		return err
	}
	w, err := b.initWatcher(paths)
	if err != nil {
		return err
	}
	defer w.Close()

	b.watch(w)
	go b.build()

	<-make(chan bool)
	return nil
}

func splitArgs(args string) []string {
	ret := make([]string, 0, 10)
	var state byte
	var start, index int

	for index = 0; index < len(args); index++ {
		b := args[index]
		if b == ' ' {
			if state != ' ' {
				ret = append(ret, args[start:index])
				state = ' '
			}
			start = index + 1
			continue
		}

		if b == '=' {
			if state != '=' {
				ret = append(ret, args[start:index])
				state = '='
			}
			start = index + 1
			continue
		}

		state = 0
	} // end for

	if start < len(args) {
		ret = append(ret, args[start:len(args)])
	}

	return ret
}

// 根据 recursive 值确定是否递归查找 paths 每个目录下的子目录。
func recursivePaths(recursive bool, paths []string) ([]string, error) {
	if !recursive {
		return paths, nil
	}

	ret := []string{}

	walk := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if fi.IsDir() && strings.Index(path, "/.") < 0 {
			ret = append(ret, path)
		}
		return nil
	}

	for _, path := range paths {
		if err := filepath.Walk(path, walk); err != nil {
			return nil, err
		}
	}

	return ret, nil
}

// 将 extString 分解成数组，并清理掉无用的内容，比如空字符串
func getExts(extString string) []string {
	exts := strings.Split(extString, ",")
	ret := make([]string, 0, len(exts))

	for _, ext := range exts {
		ext = strings.TrimSpace(ext)

		if len(ext) == 0 {
			continue
		}
		if ext[0] != '.' {
			ext = "." + ext
		}
		ret = append(ret, ext)
	}

	return ret
}

func getAppName(outputName, wd string) (string, error) {
	if outputName == "" {
		outputName = filepath.Base(wd)
	}

	goexe := os.Getenv("GOEXE")
	if goexe != "" && !strings.HasSuffix(outputName, goexe) {
		outputName += goexe
	}

	// 没有分隔符，表示仅有一个文件名，需要加上 wd
	if strings.IndexByte(outputName, '/') < 0 || strings.IndexByte(outputName, filepath.Separator) < 0 {
		outputName = filepath.Join(wd, outputName)
	}

	// 转成绝对路径
	outputName, err := filepath.Abs(outputName)
	if err != nil {
		return "", err
	}

	return outputName, nil
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package gofetch

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"fmt"
	"hash"
	"io"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
)

// ProgressReport represents the current download progress of a given file
type ProgressReport struct {
	// Total length in bytes of the file being downloaded
	Total int64
	// Written bytes to disk on a write by write basis. It does not accumulate.
	WrittenBytes int64
}

// Fetcher represents an instance of gofetch, holding global configuration options.
type Fetcher struct {
	destDir     string
	etag        bool
	concurrency int
	algorithm   string
	checksum    string
	httpClient  *http.Client
}

// Option as explained in http://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html
type Option func(*Fetcher)

// WithDestDir allows you to set the destination directory for the downloaded files.
// By default it is set to: ./
func WithDestDir(dir string) Option {
	return func(f *Fetcher) {
		f.destDir = dir
	}
}

// WithConcurrency allows you to set the number of goroutines used to download a specific
// file. By default it is set to 1.
func WithConcurrency(c int) Option {
	return func(f *Fetcher) {
		f.concurrency = c
	}
}

// WithETag enables ETag support, meaning that if an already downloaded file is currently on disk and matches the ETag value returned by the server,
// it will not be downloaded again. By default it is set to false. Be aware that different servers, serving the same file,
// are likely to return different ETag values, causing the file to be re-downloaded, even though it might already exist on disk.
func WithETag() Option {
	return func(f *Fetcher) {
		f.etag = true
	}
}

// WithTimeout allows to set a global timeout for the HTTP client. Defaults to 30 seconds. It is effectively a
// deadline as that's how Go's stdlib interprets it. It does not reset upon activity in the connection.
func WithTimeout(d time.Duration) Option {
	return func(f *Fetcher) {
		f.httpClient.Timeout = d
	}
}

// WithChecksum verifies the file once it is fully downloaded using the provided hash and expected value.
func WithChecksum(alg, value string) Option {
	return func(f *Fetcher) {
		f.algorithm = alg
		f.checksum = value
	}
}

var workDir string

func init() {
	dir, err := homedir.Dir()
	if err != nil {
		fmt.Printf(`Unable to get user home directory err=%s\n`, err)
	}
	workDir = filepath.Join(dir, ".gofetch")
}

// New creates a new instance of goFetch with the given options.
func New(opts ...Option) *Fetcher {
	// Creates instance and assigns defaults.
	gofetch := &Fetcher{
		concurrency: 1,
		destDir:     "./",
		httpClient:  &http.Client{Timeout: 0},
	}

	for _, opt := range opts {
		opt(gofetch)
	}
	return gofetch
}

// Fetch downloads content from the provided URL. It supports resuming and
// parallelizing downloads while being very memory efficient.
func (gf *Fetcher) Fetch(url string, progressCh chan<- ProgressReport) (*os.File, error) {
	if url == "" {
		return nil, errors.New("URL is required")
	}

	// We need to make a preflight request to get the size of the content and check if the server
	// supports requesting byte ranges.
	res, err := http.Head(url)
	if err != nil {
		return nil, err
	}

	if !strings.HasPrefix(res.Status, "2") {
		return nil, errors.New("HTTP requests returned a non 2xx status code")
	}

	if res.Header.Get("Accept-Ranges") != "bytes" {
		// Server does not support sending byte ranges, setting concurrency to 1
		gf.concurrency = 1
	}

	fileName := path.Base(url)
	destFilePath := filepath.Join(gf.destDir, fileName)

	var etag string
	if gf.etag {
		// Go's stdlib returns header value enclosed in double quotes.
		etag = strings.Trim(res.Header.Get("ETag"), `"`)

		if etag == "" {
			goto FETCH
		}

		// Create directory if it doesn't exist, we ignore errors if it already exists.
		cacheDir := filepath.Join(workDir, fileName)
		etagPath := filepath.Join(cacheDir, etag)
		os.MkdirAll(cacheDir, 0700)

		if _, err := os.Stat(etagPath); err == nil {
			// Our file has been already fully downloaded, return a file
			// descriptor to it and skip fetching altogether.
			fi, err := os.Stat(destFilePath)
			if err == nil && fi.Size() == res.ContentLength {
				if progressCh != nil {
					close(progressCh)
				}
				return os.Open(destFilePath)
			}
		} else {
			f, err := os.Create(etagPath)
			if err != nil {
				return nil, err
			}
			f.Close()
		}
	}

FETCH:
	f, err := gf.parallelFetch(url, destFilePath, res.ContentLength, progressCh)
	if err != nil {
		return nil, err
	}

	if gf.algorithm != "" {
		if err := gf.verify(f, gf.algorithm, gf.checksum); err != nil {
			return nil, errors.Wrap(err, "failed veryfing file integrity")
		}

		// We need to make sure we return the file descriptor ready to be read by the user again
		f.Seek(0, 0)
	}

	return f, nil
}

func (gf *Fetcher) verify(f *os.File, algorithm string, checksum string) error {
	var hasher hash.Hash
	switch algorithm {
	case "md5":
		hasher = md5.New()
	case "sha1":
		hasher = sha1.New()
	case "sha256":
		hasher = sha256.New()
	case "sha512":
		hasher = sha512.New()
	default:
		return fmt.Errorf("unsupported hashing algorithm: %s", algorithm)
	}

	// Makes sure file cursor is positioned at the beginning
	_, err := f.Seek(0, 0)
	if err != nil {
		return err
	}

	_, err = io.Copy(hasher, f)
	if err != nil {
		return err
	}

	result := fmt.Sprintf("%x", hasher.Sum(nil))

	if result != checksum {
		return fmt.Errorf("checksum does not match\n found: %s\n expected: %s", result, checksum)
	}

	return nil
}

// parallelFetch fetches using multiple goroutines, each piece is streamed down
// to disk which makes it very efficient in terms of memory usage.
func (gf *Fetcher) parallelFetch(url, destFilePath string, length int64, progressCh chan<- ProgressReport) (*os.File, error) {
	if progressCh != nil {
		defer close(progressCh)
	}

	var wg sync.WaitGroup

	report := ProgressReport{Total: length}
	concurrency := int64(gf.concurrency)
	chunkSize := length / concurrency
	remainingSize := length % concurrency
	chunksDir := filepath.Join(gf.destDir, path.Base(url)+".chunks")

	if err := os.MkdirAll(chunksDir, 0760); err != nil {
		return nil, err
	}

	var errs []error
	for i := int64(0); i < concurrency; i++ {
		min := chunkSize * i
		max := chunkSize * (i + 1)

		if i == (concurrency - 1) {
			// Add the remaining bytes in the last request
			max += remainingSize
		}

		wg.Add(1)
		go func(min, max int64, chunkNumber int) {
			defer wg.Done()
			chunkFile := filepath.Join(chunksDir, strconv.Itoa(chunkNumber))

			err := gf.fetch(url, chunkFile, min, max, report, progressCh)
			if err != nil {
				errs = append(errs, err)
			}
		}(min, max, int(i))
	}
	wg.Wait()

	if len(errs) > 0 {
		return nil, fmt.Errorf("errors: \n %s", errs)
	}

	file, err := gf.assembleChunks(destFilePath, chunksDir)
	if err != nil {
		return nil, err
	}

	os.RemoveAll(chunksDir)

	// Makes sure to return the file on the correct offset so it can be
	// consumed by users.
	_, err = file.Seek(0, 0)
	if err != nil {
		return nil, err
	}

	return file, err
}

// assembleChunks join all the data pieces together
func (gf *Fetcher) assembleChunks(destFile, chunksDir string) (*os.File, error) {
	file, err := os.Create(destFile)
	if err != nil {
		return nil, err
	}

	for i := 0; i < gf.concurrency; i++ {
		chunkFile, err := os.Open(filepath.Join(chunksDir, strconv.Itoa(i)))
		if err != nil {
			return nil, err
		}
		// Deferring within a loop is not ideal but we expect not too many chunks to exists.
		defer chunkFile.Close()

		if _, err := io.Copy(file, chunkFile); err != nil {
			return nil, err
		}
	}
	return file, nil
}

// fetch downloads files using one unbuffered HTTP connection and supports
// resuming downloads if interrupted.
func (gf *Fetcher) fetch(url, destFile string, min, max int64,
	report ProgressReport, progressCh chan<- ProgressReport) error {

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	// In order to resume previous interrupted downloads we need to open the file
	// in append mode.
	file, err := os.OpenFile(destFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660)
	if err != nil {
		return err
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		return err
	}

	currFileSize := fi.Size()
	currChunkSize := (max - min)

	// There is nothing to do if chunk data file was fully downloaded.
	if currFileSize > 0 && currFileSize == currChunkSize {
		return nil
	}

	// Report bytes written already into the file
	if progressCh != nil {
		report.WrittenBytes = currFileSize
		progressCh <- report
	}

	// Adjusts min to resume file download from where it was left off.
	if currFileSize > 0 {
		min = min + currFileSize
	}

	// Prepares writer to report download progress.
	writer := fetchWriter{
		Writer:         file,
		progressCh:     progressCh,
		progressReport: report,
	}

	brange := fmt.Sprintf("bytes=%d-%d", min, max-1)
	if max == -1 {
		brange = fmt.Sprintf("bytes=%d-", min)
	}

	req.Header.Add("Range", brange)
	//fmt.Printf("range %s\n", brange)
	res, err := gf.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if !strings.HasPrefix(res.Status, "2") {
		return errors.New("HTTP requests returned a non 2xx status code")
	}

	reader := res.Body.(io.Reader)
	if max > 0 {
		// Known content-length, so we only read from body the amount of bytes of requested chunk.
		reader = io.LimitReader(res.Body, currChunkSize)
	}
	_, err = io.Copy(&writer, reader)
	return err
}

// fetchWriter implements a custom io.Writer so we can send granular
// progress reports when streaming down content.
type fetchWriter struct {
	io.Writer
	//progressCh is the channel sent by the user to get download updates.
	progressCh chan<- ProgressReport
	// report is the structure sent through the progress channel.
	progressReport ProgressReport
}

func (fw *fetchWriter) Write(b []byte) (int, error) {
	n, err := fw.Writer.Write(b)

	if fw.progressCh != nil {
		fw.progressReport.WrittenBytes = int64(n)
		fw.progressCh <- fw.progressReport
	}

	return n, err
}
<commit_msg>Fixes progress reporting when resuming.<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, version 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package gofetch

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"fmt"
	"hash"
	"io"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/hooklift/httpclient"
	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
)

// ProgressReport represents the current download progress of a given file
type ProgressReport struct {
	// Total length in bytes of the file being downloaded
	Total int64
	// Written bytes to disk on a write by write basis. It does not accumulate.
	WrittenBytes int64
}

// Fetcher represents an instance of gofetch, holding global configuration options.
type Fetcher struct {
	destDir     string
	etag        bool
	concurrency int
	algorithm   string
	checksum    string
	httpClient  *http.Client
}

// Option as explained in http://commandcenter.blogspot.com/2014/01/self-referential-functions-and-design.html
type Option func(*Fetcher)

// WithDestDir allows you to set the destination directory for the downloaded files.
// By default it is set to: ./
func WithDestDir(dir string) Option {
	return func(f *Fetcher) {
		f.destDir = dir
	}
}

// WithConcurrency allows you to set the number of goroutines used to download a specific
// file. By default it is set to 1.
func WithConcurrency(c int) Option {
	return func(f *Fetcher) {
		f.concurrency = c
	}
}

// WithETag enables ETag support, meaning that if an already downloaded file is currently on disk and matches the ETag value returned by the server,
// it will not be downloaded again. By default it is set to false. Be aware that different servers, serving the same file,
// are likely to return different ETag values, causing the file to be re-downloaded, even though it might already exist on disk.
func WithETag() Option {
	return func(f *Fetcher) {
		f.etag = true
	}
}

// WithHTTPClient allows to provide a custom HTTP client. By default a HTTP client with support for read/write timeouts is used.
func WithHTTPClient(c *http.Client) Option {
	return func(f *Fetcher) {
		f.httpClient = c
	}
}

// WithChecksum verifies the file once it is fully downloaded using the provided hash and expected value.
func WithChecksum(alg, value string) Option {
	return func(f *Fetcher) {
		f.algorithm = alg
		f.checksum = value
	}
}

var workDir string

func init() {
	dir, err := homedir.Dir()
	if err != nil {
		fmt.Printf(`Unable to get user home directory err=%s\n`, err)
	}
	workDir = filepath.Join(dir, ".gofetch")
}

// New creates a new instance of goFetch with the given options.
func New(opts ...Option) *Fetcher {
	// Creates instance and assigns defaults.
	gofetch := &Fetcher{
		concurrency: 1,
		destDir:     "./",
		httpClient:  httpclient.Default(),
	}

	for _, opt := range opts {
		opt(gofetch)
	}

	return gofetch
}

// Fetch downloads content from the provided URL. It supports resuming and
// parallelizing downloads while being very memory efficient.
func (gf *Fetcher) Fetch(url string, progressCh chan<- ProgressReport) (*os.File, error) {
	if url == "" {
		return nil, errors.New("URL is required")
	}

	// We need to make a preflight request to get the size of the content and check if the server
	// supports requesting byte ranges.
	res, err := http.Head(url)
	if err != nil {
		return nil, err
	}

	if !strings.HasPrefix(res.Status, "2") {
		return nil, errors.New("HTTP requests returned a non 2xx status code")
	}

	if res.Header.Get("Accept-Ranges") != "bytes" {
		// Server does not support sending byte ranges, setting concurrency to 1
		gf.concurrency = 1
	}

	fileName := path.Base(url)
	destFilePath := filepath.Join(gf.destDir, fileName)

	var etag string
	if gf.etag {
		// Go's stdlib returns header value enclosed in double quotes.
		etag = strings.Trim(res.Header.Get("ETag"), `"`)

		if etag == "" {
			goto FETCH
		}

		// Create directory if it doesn't exist, we ignore errors if it already exists.
		cacheDir := filepath.Join(workDir, fileName)
		etagPath := filepath.Join(cacheDir, etag)
		os.MkdirAll(cacheDir, 0700)

		if _, err := os.Stat(etagPath); err == nil {
			// Our file has been already fully downloaded, return a file
			// descriptor to it and skip fetching altogether.
			fi, err := os.Stat(destFilePath)
			if err == nil && fi.Size() == res.ContentLength {
				if progressCh != nil {
					close(progressCh)
				}
				return os.Open(destFilePath)
			}
		} else {
			f, err := os.Create(etagPath)
			if err != nil {
				return nil, err
			}
			f.Close()
		}
	}

FETCH:
	f, err := gf.parallelFetch(url, destFilePath, res.ContentLength, progressCh)
	if err != nil {
		return nil, err
	}

	if gf.algorithm != "" {
		if err := gf.verify(f, gf.algorithm, gf.checksum); err != nil {
			return nil, errors.Wrap(err, "failed veryfing file integrity")
		}

		// We need to make sure we return the file descriptor ready to be read by the user again
		f.Seek(0, 0)
	}

	return f, nil
}

func (gf *Fetcher) verify(f *os.File, algorithm string, checksum string) error {
	var hasher hash.Hash
	switch algorithm {
	case "md5":
		hasher = md5.New()
	case "sha1":
		hasher = sha1.New()
	case "sha256":
		hasher = sha256.New()
	case "sha512":
		hasher = sha512.New()
	default:
		return fmt.Errorf("unsupported hashing algorithm: %s", algorithm)
	}

	// Makes sure file cursor is positioned at the beginning
	_, err := f.Seek(0, 0)
	if err != nil {
		return err
	}

	_, err = io.Copy(hasher, f)
	if err != nil {
		return err
	}

	result := fmt.Sprintf("%x", hasher.Sum(nil))

	if result != checksum {
		return fmt.Errorf("checksum does not match\n found: %s\n expected: %s", result, checksum)
	}

	return nil
}

// parallelFetch fetches using multiple goroutines, each piece is streamed down
// to disk which makes it very efficient in terms of memory usage.
func (gf *Fetcher) parallelFetch(url, destFilePath string, length int64, progressCh chan<- ProgressReport) (*os.File, error) {
	if progressCh != nil {
		defer close(progressCh)
	}

	var wg sync.WaitGroup

	report := ProgressReport{Total: length}
	concurrency := int64(gf.concurrency)
	chunkSize := length / concurrency
	remainingSize := length % concurrency
	chunksDir := filepath.Join(gf.destDir, path.Base(url)+".chunks")

	if err := os.MkdirAll(chunksDir, 0760); err != nil {
		return nil, err
	}

	var errs []error
	for i := int64(0); i < concurrency; i++ {
		min := chunkSize * i
		max := chunkSize * (i + 1)

		if i == (concurrency - 1) {
			// Add the remaining bytes in the last request
			max += remainingSize
		}

		wg.Add(1)
		go func(min, max int64, chunkNumber int) {
			defer wg.Done()
			chunkFile := filepath.Join(chunksDir, strconv.Itoa(chunkNumber))

			if err := gf.fetch(url, chunkFile, min, max, report, progressCh); err != nil {
				fmt.Printf("gofetch: error %#v\n", err)
				errs = append(errs, err)
			}
		}(min, max, int(i))
	}
	wg.Wait()

	if len(errs) > 0 {
		return nil, fmt.Errorf("errors: \n %s", errs)
	}

	file, err := gf.assembleChunks(destFilePath, chunksDir)
	if err != nil {
		return nil, err
	}

	os.RemoveAll(chunksDir)

	// Makes sure to return the file on the correct offset so it can be
	// consumed by users.
	_, err = file.Seek(0, 0)
	if err != nil {
		return nil, err
	}

	return file, err
}

// assembleChunks join all the data pieces together
func (gf *Fetcher) assembleChunks(destFile, chunksDir string) (*os.File, error) {
	file, err := os.Create(destFile)
	if err != nil {
		return nil, err
	}

	for i := 0; i < gf.concurrency; i++ {
		chunkFile, err := os.Open(filepath.Join(chunksDir, strconv.Itoa(i)))
		if err != nil {
			return nil, err
		}
		// Deferring within a loop is not ideal but we expect not too many chunks to exists.
		defer chunkFile.Close()

		if _, err := io.Copy(file, chunkFile); err != nil {
			return nil, err
		}
	}
	return file, nil
}

// fetch downloads files using one unbuffered HTTP connection and supports
// resuming downloads if interrupted.
func (gf *Fetcher) fetch(url, destFile string, min, max int64,
	report ProgressReport, progressCh chan<- ProgressReport) error {

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	// In order to resume previous interrupted downloads we need to open the file
	// in append mode.
	file, err := os.OpenFile(destFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0660)
	if err != nil {
		return err
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		return err
	}

	currFileSize := fi.Size()
	currChunkSize := (max - min)

	// Report bytes written already into the file
	if progressCh != nil {
		report.WrittenBytes = currFileSize
		progressCh <- report
	}

	// There is nothing to do if chunk data file was fully downloaded.
	if currFileSize > 0 && currFileSize == currChunkSize {
		return nil
	}

	// Adjusts min to resume file download from where it was left off.
	if currFileSize > 0 {
		min = min + currFileSize
	}

	// Prepares writer to report download progress.
	writer := fetchWriter{
		Writer:         file,
		progressCh:     progressCh,
		progressReport: report,
	}

	brange := fmt.Sprintf("bytes=%d-%d", min, max-1)
	if max == -1 {
		brange = fmt.Sprintf("bytes=%d-", min)
	}

	req.Header.Add("Range", brange)
	//fmt.Printf("range %s\n", brange)
	res, err := gf.httpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if !strings.HasPrefix(res.Status, "2") {
		return errors.New("HTTP requests returned a non 2xx status code")
	}

	reader := res.Body.(io.Reader)
	if max > 0 {
		// Known content-length, so we only read from body the amount of bytes of requested chunk.
		reader = io.LimitReader(res.Body, currChunkSize)
	}
	_, err = io.Copy(&writer, reader)
	return err
}

// fetchWriter implements a custom io.Writer so we can send granular
// progress reports when streaming down content.
type fetchWriter struct {
	io.Writer
	//progressCh is the channel sent by the user to get download updates.
	progressCh chan<- ProgressReport
	// report is the structure sent through the progress channel.
	progressReport ProgressReport
}

func (fw *fetchWriter) Write(b []byte) (int, error) {
	n, err := fw.Writer.Write(b)

	if fw.progressCh != nil {
		fw.progressReport.WrittenBytes = int64(n)
		fw.progressCh <- fw.progressReport
	}

	return n, err
}
<|endoftext|>