<commit_before>package main

import "regexp"

const (
	UserValid = iota
	UserInvalidNickname
	UserInvalidEmail
	UserWeakPassword
	UserPasswordMismatch
)

type RegisterResult struct {
	errors []int32
}

type RegisterFormValidator struct {
}

type RegexpsContainer struct {
	nicknameRegexp  *regexp.Regexp
	emailRegexp     *regexp.Regexp
	passwordRegexp1 *regexp.Regexp
	passwordRegexp2 *regexp.Regexp
}

var regexpsContainer RegexpsContainer

func (self *RegisterFormValidator) Check(user *SiteUser) RegisterResult {
	self.InitRegexpsContainer()
	result := RegisterResult{
		errors: []int32{},
	}
	if !self.IsNicknameValid(user.nickname) {
		result.errors = append(result.errors, UserInvalidNickname)
	}
	if !self.IsEmailValid(user.email) {
		result.errors = append(result.errors, UserInvalidEmail)
	}
	if !self.IsPasswordValid(user.password1) {
		result.errors = append(result.errors, UserWeakPassword)
	}
	if user.password1 != user.password2 {
		result.errors = append(result.errors, UserPasswordMismatch)
	}
	return result
}

func (self *RegisterFormValidator) InitRegexpsContainer() {
	regexpsContainer = RegexpsContainer{
		nicknameRegexp:  regexp.MustCompile("^[A-z0-9_]+$"),
		emailRegexp:     regexp.MustCompile("(?i)^[A-z0-9_]+@(?:gmail\\.com|yandex\\.ru|mail\\.ru)$"),
		passwordRegexp1: regexp.MustCompile("[A-z]+"),
		passwordRegexp2: regexp.MustCompile("\\d+"),
	}
}

func (self *RegisterFormValidator) IsNicknameValid(nickname string) bool {
	return regexpsContainer.nicknameRegexp.MatchString(nickname)
}

func (self *RegisterFormValidator) IsEmailValid(email string) bool {
	return regexpsContainer.emailRegexp.MatchString(email)
}

func (self *RegisterFormValidator) IsPasswordValid(password string) bool {
	result := regexpsContainer.passwordRegexp1.MatchString(password)
	result = result && regexpsContainer.passwordRegexp2.MatchString(password)
	return result
}
<commit_msg>regexpsContainer moved to RegisterFormValidator<commit_after>package main

import "regexp"

const (
	UserValid = iota
	UserInvalidNickname
	UserInvalidEmail
	UserWeakPassword
	UserPasswordMismatch
)

type RegisterResult struct {
	errors []int32
}

type RegisterFormValidator struct {
	regexpsContainer RegexpsContainer
}

type RegexpsContainer struct {
	nicknameRegexp  *regexp.Regexp
	emailRegexp     *regexp.Regexp
	passwordRegexp1 *regexp.Regexp
	passwordRegexp2 *regexp.Regexp
}

func (self *RegisterFormValidator) Check(user *SiteUser) RegisterResult {
	self.InitRegexpsContainer()
	result := RegisterResult{
		errors: []int32{},
	}
	if !self.IsNicknameValid(user.nickname) {
		result.errors = append(result.errors, UserInvalidNickname)
	}
	if !self.IsEmailValid(user.email) {
		result.errors = append(result.errors, UserInvalidEmail)
	}
	if !self.IsPasswordValid(user.password1) {
		result.errors = append(result.errors, UserWeakPassword)
	}
	if user.password1 != user.password2 {
		result.errors = append(result.errors, UserPasswordMismatch)
	}
	return result
}

func (self *RegisterFormValidator) InitRegexpsContainer() {
	self.regexpsContainer = RegexpsContainer{
		nicknameRegexp:  regexp.MustCompile("^[A-z0-9_]+$"),
		emailRegexp:     regexp.MustCompile("(?i)^[A-z0-9_]+@(?:gmail\\.com|yandex\\.ru|mail\\.ru)$"),
		passwordRegexp1: regexp.MustCompile("[A-z]+"),
		passwordRegexp2: regexp.MustCompile("\\d+"),
	}
}

func (self *RegisterFormValidator) IsNicknameValid(nickname string) bool {
	return self.regexpsContainer.nicknameRegexp.MatchString(nickname)
}

func (self *RegisterFormValidator) IsEmailValid(email string) bool {
	return self.regexpsContainer.emailRegexp.MatchString(email)
}

func (self *RegisterFormValidator) IsPasswordValid(password string) bool {
	result := self.regexpsContainer.passwordRegexp1.MatchString(password)
	result = result && self.regexpsContainer.passwordRegexp2.MatchString(password)
	return result
}
<|endoftext|>


<commit_before>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	fmt.Printf("in amazon.Reader()\n")
	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("keypair id: %v\n", string(rawCloudfrontKeyPairId))
			/*
					fmt.Printf("got cf keypaird id (%v)\n", string(rawCloudfrontKeyPairId))
					decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
					if err != nil {
						return nil, err
					}
					decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
					fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

				decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
				if err != nil {
					return nil, err
				}
				decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
				fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			*/
			//			block, _ := pem.Decode(bytes.TrimSpace(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(rawCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = signedURL
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		if connErr != nil {
			return nil, connErr
		}
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<commit_msg>Debug status code<commit_after>package obj

import (
	"bytes"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/cenkalti/backoff"
	"go.pedge.io/lion"
)

type amazonClient struct {
	bucket       string
	distribution string
	s3           *s3.S3
	uploader     *s3manager.Uploader
}

func newAmazonClient(bucket string, distribution string, id string, secret string, token string, region string) (*amazonClient, error) {
	session := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(id, secret, token),
		Region:      aws.String(region),
	})
	return &amazonClient{
		bucket:       bucket,
		distribution: strings.TrimSpace(distribution),
		s3:           s3.New(session),
		uploader:     s3manager.NewUploader(session),
	}, nil
}

func (c *amazonClient) Writer(name string) (io.WriteCloser, error) {
	return newBackoffWriteCloser(c, newWriter(c, name)), nil
}

func (c *amazonClient) Walk(name string, fn func(name string) error) error {
	var fnErr error
	if err := c.s3.ListObjectsPages(
		&s3.ListObjectsInput{
			Bucket: aws.String(c.bucket),
			Prefix: aws.String(name),
		},
		func(listObjectsOutput *s3.ListObjectsOutput, lastPage bool) bool {
			for _, object := range listObjectsOutput.Contents {
				if err := fn(*object.Key); err != nil {
					fnErr = err
					return false
				}
			}
			return true
		},
	); err != nil {
		return err
	}
	return fnErr
}

func (c *amazonClient) Reader(name string, offset uint64, size uint64) (io.ReadCloser, error) {
	byteRange := byteRange(offset, size)
	if byteRange != "" {
		byteRange = fmt.Sprintf("bytes=%s", byteRange)
	}
	fmt.Printf("in amazon.Reader()\n")
	var reader io.ReadCloser
	if c.distribution != "" {
		var resp *http.Response
		var connErr error
		url := fmt.Sprintf("http://%v.cloudfront.net/%v", c.distribution, name)

		fmt.Println("Checking for cloudfront private key")
		rawCloudfrontPrivateKey, err := ioutil.ReadFile("/amazon-secret/cloudfrontPrivateKey")
		if err == nil {
			// If cloudfront security credentials are present, use them
			fmt.Printf("got cf private key secret: (%v)\n", string(rawCloudfrontPrivateKey))

			rawCloudfrontKeyPairId, err := ioutil.ReadFile("/amazon-secret/cloudfrontKeyPairId")
			if err != nil {
				return nil, fmt.Errorf("cloudfront private key provided, but missing cloudfront key pair id")
			}
			fmt.Printf("keypair id: %v\n", string(rawCloudfrontKeyPairId))
			/*
					fmt.Printf("got cf keypaird id (%v)\n", string(rawCloudfrontKeyPairId))
					decodedCloudfrontKeyPairId, err := base64.StdEncoding.DecodeString(string(rawCloudfrontKeyPairId))
					if err != nil {
						return nil, err
					}
					decodedCloudfrontKeyPairId = bytes.TrimSpace(decodedCloudfrontKeyPairId)
					fmt.Printf("decoded keypair id (%v)\n", string(decodedCloudfrontKeyPairId))

				decodedCloudfrontPrivateKey, err := base64.StdEncoding.DecodeString(string(rawCloudfrontPrivateKey))
				if err != nil {
					return nil, err
				}
				decodedCloudfrontPrivateKey = bytes.TrimSpace(decodedCloudfrontPrivateKey)
				fmt.Printf("decoded private key (%v)\n", string(decodedCloudfrontPrivateKey))
			*/
			//			block, _ := pem.Decode(bytes.TrimSpace(decodedCloudfrontPrivateKey))
			block, _ := pem.Decode(bytes.TrimSpace(rawCloudfrontPrivateKey))
			if block == nil || block.Type != "RSA PRIVATE KEY" {
				return nil, fmt.Errorf("block undefined or wrong type: type is (%v) should be (RSA PRIVATE KEY)", block.Type)
			}

			cloudfrontPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
			if err != nil {
				return nil, err
			}
			signer := sign.NewURLSigner(string(rawCloudfrontKeyPairId), cloudfrontPrivateKey)
			signedURL, err := signer.Sign(url, time.Now().Add(1*time.Hour))
			fmt.Printf("orig url (%v), signed url (%v)\n", url, signedURL)
			if err != nil {
				return nil, err
			}
			url = signedURL
		}
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}
		req.Header.Add("Range", byteRange)

		backoff.RetryNotify(func() error {
			resp, connErr = http.DefaultClient.Do(req)
			if connErr != nil && isNetRetryable(connErr) {
				return connErr
			}
			return nil
		}, backoff.NewExponentialBackOff(), func(err error, d time.Duration) {
			lion.Infof("Error connecting to (%v); retrying in %s: %#v", url, d, err)
		})
		fmt.Printf("connErr (%v), resp (%v)\n", connErr, resp)
		if connErr != nil {
			return nil, connErr
		}
		fmt.Printf("resp status code %v %v\n", resp.StatusCode, resp.Status)
		if resp.StatusCode >= 300 {
			// Cloudfront returns 200s, and 206s as success codes
			return nil, fmt.Errorf("cloudfront returned HTTP error code %v for url %v", resp.Status, url)
		}
		reader = resp.Body
	} else {
		getObjectOutput, err := c.s3.GetObject(&s3.GetObjectInput{
			Bucket: aws.String(c.bucket),
			Key:    aws.String(name),
			Range:  aws.String(byteRange),
		})
		if err != nil {
			return nil, err
		}
		reader = getObjectOutput.Body
	}
	return newBackoffReadCloser(c, reader), nil
}

func (c *amazonClient) Delete(name string) error {
	_, err := c.s3.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err
}

func (c *amazonClient) Exists(name string) bool {
	_, err := c.s3.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(c.bucket),
		Key:    aws.String(name),
	})
	return err == nil
}

func (c *amazonClient) isRetryable(err error) (retVal bool) {
	if strings.Contains(err.Error(), "unexpected EOF") {
		return true
	}

	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	for _, c := range []string{
		storagegateway.ErrorCodeServiceUnavailable,
		storagegateway.ErrorCodeInternalError,
		storagegateway.ErrorCodeGatewayInternalError,
	} {
		if c == awsErr.Code() {
			return true
		}
	}
	return false
}

func (c *amazonClient) IsIgnorable(err error) bool {
	return false
}

func (c *amazonClient) IsNotExist(err error) bool {
	if c.distribution != "" {
		// cloudfront returns forbidden error for nonexisting data
		if strings.Contains(err.Error(), "error code 403") {
			return true
		}
	}
	awsErr, ok := err.(awserr.Error)
	if !ok {
		return false
	}
	if awsErr.Code() == "NoSuchKey" {
		return true
	}
	return false
}

type amazonWriter struct {
	errChan chan error
	pipe    *io.PipeWriter
}

func newWriter(client *amazonClient, name string) *amazonWriter {
	reader, writer := io.Pipe()
	w := &amazonWriter{
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.uploader.Upload(&s3manager.UploadInput{
			Body:            reader,
			Bucket:          aws.String(client.bucket),
			Key:             aws.String(name),
			ContentEncoding: aws.String("application/octet-stream"),
		})
		w.errChan <- err
	}()
	return w
}

func (w *amazonWriter) Write(p []byte) (int, error) {
	return w.pipe.Write(p)
}

func (w *amazonWriter) Close() error {
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}
<|endoftext|>


<commit_before>package worker

import (
	"sync"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"

	"golang.org/x/net/context"
)

type branchSet struct {
	Branches  []*pfs.BranchInfo
	NewBranch int //newBranch indicates which branch is new
	Err       error
}

type branchSetFactory interface {
	Chan() chan *branchSet
	Close()
}

type branchSetFactoryImpl struct {
	ch     chan *branchSet
	cancel context.CancelFunc
}

func (f *branchSetFactoryImpl) Close() {
	f.cancel()
}

func (f *branchSetFactoryImpl) Chan() chan *branchSet {
	return f.ch
}

func (a *APIServer) newBranchSetFactory(_ctx context.Context) (branchSetFactory, error) {
	ctx, cancel := context.WithCancel(_ctx)
	pfsClient := a.pachClient.PfsAPIClient

	rootInputs, err := a.rootInputs(ctx)
	if err != nil {
		return nil, err
	}
	directInputs := a.directInputs(ctx)

	commitSets := make([][]*pfs.CommitInfo, len(directInputs))
	var commitSetsMutex sync.Mutex

	ch := make(chan *branchSet)
	for i, input := range directInputs {
		i, input := i, input

		request := &pfs.SubscribeCommitRequest{
			Repo:   client.NewRepo(input.Repo),
			Branch: input.Branch,
		}
		if input.FromCommit != "" {
			request.From = client.NewCommit(input.Repo, input.FromCommit)
		}
		stream, err := pfsClient.SubscribeCommit(ctx, request)
		if err != nil {
			cancel()
			return nil, err
		}

		go func() {
			for {
				commitInfo, err := stream.Recv()
				if err != nil {
					select {
					case <-ctx.Done():
					case ch <- &branchSet{
						Err: err,
					}:
					}
					return
				}

				commitSetsMutex.Lock()
				commitSets[i] = append(commitSets[i], commitInfo)

				// Now we look for a set of commits that have provenance
				// commits from the root input repos.  The set is only valid
				// if there's precisely one provenance commit from each root
				// input repo.
				if set := findCommitSet(commitSets, i, func(set []*pfs.CommitInfo) bool {
					rootCommits := make(map[string]map[string]bool)
					setRootCommit := func(commit *pfs.Commit) {
						if rootCommits[commit.Repo.Name] == nil {
							rootCommits[commit.Repo.Name] = make(map[string]bool)
						}
						rootCommits[commit.Repo.Name][commit.ID] = true
					}
					for _, commitInfo := range set {
						setRootCommit(commitInfo.Commit)
						for _, provCommit := range commitInfo.Provenance {
							setRootCommit(provCommit)
						}
					}
					// ok tells us if it's ok to spawn a job with this
					// commit set.
					for _, rootInput := range rootInputs {
						if len(rootCommits[rootInput.Repo]) != 1 {
							return false
						}
					}
					return true
				}); set != nil {
					bs := &branchSet{
						NewBranch: i,
					}
					for _, commitInfo := range set {
						bs.Branches = append(bs.Branches, &pfs.BranchInfo{
							Name: input.Branch,
							Head: commitInfo.Commit,
						})
					}
					select {
					case <-ctx.Done():
						commitSetsMutex.Unlock()
						return
					case ch <- bs:
					}
				}
				commitSetsMutex.Unlock()
			}
		}()
	}

	f := &branchSetFactoryImpl{
		cancel: cancel,
		ch:     ch,
	}

	return f, nil
}

// findCommitSet runs a function on every commit set, starting with the
// most recent one.  If the function returns true, then findCommitSet
// removes the commit sets that are older than the current one and returns
// the current one.
// i is the index of the input from which we just got a new commit.  Since
// it's the "triggering" input, we know that we only have to consider commit
// sets that include the triggering commit since other commit sets must have
// already been considered in previous runs.
func findCommitSet(commitSets [][]*pfs.CommitInfo, i int, f func(commitSet []*pfs.CommitInfo) bool) []*pfs.CommitInfo {
	numCommitSets := 1
	for j, commits := range commitSets {
		if i != j {
			numCommitSets *= len(commits)
		}
	}
	for j := numCommitSets - 1; j >= 0; j-- {
		numCommitSets := j
		var commitSet []*pfs.CommitInfo
		var indexes []int
		for k, commits := range commitSets {
			var index int
			if k == i {
				index = len(commits) - 1
			} else {
				index = numCommitSets % len(commits)
				numCommitSets /= len(commits)
			}
			indexes = append(indexes, index)
			commitSet = append(commitSet, commits[index])
		}
		if f(commitSet) {
			// Remove older commit sets
			for k, index := range indexes {
				commitSets[k] = commitSets[k][index:]
			}
			return commitSet
		}
	}
	return nil
}

// directInputs returns the inputs that should trigger the pipeline.  Inputs
// from the same repo/branch are de-duplicated.
func (a *APIServer) directInputs(ctx context.Context) []*pps.AtomInput {
	repoSet := make(map[string]bool)
	var atomInputs []*pps.AtomInput
	pps.VisitInput(a.pipelineInfo.Input, func(input *pps.Input) {
		if input.Atom != nil {
			if repoSet[input.Atom.Repo] {
				return
			}
			repoSet[input.Atom.Repo] = true
			atomInputs = append(atomInputs, input.Atom)
		}
	})
	return atomInputs
}

// rootInputs returns the root provenance of direct inputs.
func (a *APIServer) rootInputs(ctx context.Context) ([]*pps.AtomInput, error) {
	atomInputs := a.directInputs(ctx)
	return a._rootInputs(ctx, atomInputs)
}

func (a *APIServer) _rootInputs(ctx context.Context, atomInputs []*pps.AtomInput) ([]*pps.AtomInput, error) {
	pfsClient := a.pachClient.PfsAPIClient
	ppsClient := a.pachClient.PpsAPIClient
	// map from repo.Name + branch to *pps.AtomInput so that we don't
	// repeat ourselves.
	resultMap := make(map[string]*pps.AtomInput)
	for _, atomInput := range atomInputs {
		repoInfo, err := pfsClient.InspectRepo(ctx, &pfs.InspectRepoRequest{client.NewRepo(atomInput.Repo)})
		if err != nil {
			return nil, err
		}
		if len(repoInfo.Provenance) == 0 {
			resultMap[atomInput.Repo+atomInput.Branch] = atomInput
			continue
		}
		// if the repo has nonzero provenance we know that it's a pipeline
		pipelineInfo, err := ppsClient.InspectPipeline(ctx, &pps.InspectPipelineRequest{client.NewPipeline(atomInput.Repo)})
		if err != nil {
			return nil, err
		}
		// TODO we should be propagating `from_commit` here
		var visitErr error
		pps.VisitInput(pipelineInfo.Input, func(input *pps.Input) {
			if input.Atom != nil {
				subResults, err := a._rootInputs(ctx, []*pps.AtomInput{input.Atom})
				if err != nil && visitErr == nil {
					visitErr = err
				}
				for _, atomInput := range subResults {
					resultMap[atomInput.Repo+atomInput.Branch] = atomInput
				}
			}
		})
		if visitErr != nil {
			return nil, visitErr
		}
	}
	var result []*pps.AtomInput
	for _, atomInput := range resultMap {
		result = append(result, atomInput)
	}
	return result, nil
}
<commit_msg>Fix a bug where branches in branchSet are assigned the wrong branch name<commit_after>package worker

import (
	"sync"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pps"

	"golang.org/x/net/context"
)

type branchSet struct {
	Branches  []*pfs.BranchInfo
	NewBranch int //newBranch indicates which branch is new
	Err       error
}

type branchSetFactory interface {
	Chan() chan *branchSet
	Close()
}

type branchSetFactoryImpl struct {
	ch     chan *branchSet
	cancel context.CancelFunc
}

func (f *branchSetFactoryImpl) Close() {
	f.cancel()
}

func (f *branchSetFactoryImpl) Chan() chan *branchSet {
	return f.ch
}

func (a *APIServer) newBranchSetFactory(_ctx context.Context) (branchSetFactory, error) {
	ctx, cancel := context.WithCancel(_ctx)
	pfsClient := a.pachClient.PfsAPIClient

	rootInputs, err := a.rootInputs(ctx)
	if err != nil {
		return nil, err
	}
	directInputs := a.directInputs(ctx)

	commitSets := make([][]*pfs.CommitInfo, len(directInputs))
	var commitSetsMutex sync.Mutex

	ch := make(chan *branchSet)
	for i, input := range directInputs {
		i, input := i, input

		request := &pfs.SubscribeCommitRequest{
			Repo:   client.NewRepo(input.Repo),
			Branch: input.Branch,
		}
		if input.FromCommit != "" {
			request.From = client.NewCommit(input.Repo, input.FromCommit)
		}
		stream, err := pfsClient.SubscribeCommit(ctx, request)
		if err != nil {
			cancel()
			return nil, err
		}

		go func() {
			for {
				commitInfo, err := stream.Recv()
				if err != nil {
					select {
					case <-ctx.Done():
					case ch <- &branchSet{
						Err: err,
					}:
					}
					return
				}

				commitSetsMutex.Lock()
				commitSets[i] = append(commitSets[i], commitInfo)

				// Now we look for a set of commits that have provenance
				// commits from the root input repos.  The set is only valid
				// if there's precisely one provenance commit from each root
				// input repo.
				if set := findCommitSet(commitSets, i, func(set []*pfs.CommitInfo) bool {
					rootCommits := make(map[string]map[string]bool)
					setRootCommit := func(commit *pfs.Commit) {
						if rootCommits[commit.Repo.Name] == nil {
							rootCommits[commit.Repo.Name] = make(map[string]bool)
						}
						rootCommits[commit.Repo.Name][commit.ID] = true
					}
					for _, commitInfo := range set {
						setRootCommit(commitInfo.Commit)
						for _, provCommit := range commitInfo.Provenance {
							setRootCommit(provCommit)
						}
					}
					// ok tells us if it's ok to spawn a job with this
					// commit set.
					for _, rootInput := range rootInputs {
						if len(rootCommits[rootInput.Repo]) != 1 {
							return false
						}
					}
					return true
				}); set != nil {
					bs := &branchSet{
						NewBranch: i,
					}
					for i, commitInfo := range set {
						bs.Branches = append(bs.Branches, &pfs.BranchInfo{
							Name: directInputs[i].Branch,
							Head: commitInfo.Commit,
						})
					}
					select {
					case <-ctx.Done():
						commitSetsMutex.Unlock()
						return
					case ch <- bs:
					}
				}
				commitSetsMutex.Unlock()
			}
		}()
	}

	f := &branchSetFactoryImpl{
		cancel: cancel,
		ch:     ch,
	}

	return f, nil
}

// findCommitSet runs a function on every commit set, starting with the
// most recent one.  If the function returns true, then findCommitSet
// removes the commit sets that are older than the current one and returns
// the current one.
// i is the index of the input from which we just got a new commit.  Since
// it's the "triggering" input, we know that we only have to consider commit
// sets that include the triggering commit since other commit sets must have
// already been considered in previous runs.
func findCommitSet(commitSets [][]*pfs.CommitInfo, i int, f func(commitSet []*pfs.CommitInfo) bool) []*pfs.CommitInfo {
	numCommitSets := 1
	for j, commits := range commitSets {
		if i != j {
			numCommitSets *= len(commits)
		}
	}
	for j := numCommitSets - 1; j >= 0; j-- {
		numCommitSets := j
		var commitSet []*pfs.CommitInfo
		var indexes []int
		for k, commits := range commitSets {
			var index int
			if k == i {
				index = len(commits) - 1
			} else {
				index = numCommitSets % len(commits)
				numCommitSets /= len(commits)
			}
			indexes = append(indexes, index)
			commitSet = append(commitSet, commits[index])
		}
		if f(commitSet) {
			// Remove older commit sets
			for k, index := range indexes {
				commitSets[k] = commitSets[k][index:]
			}
			return commitSet
		}
	}
	return nil
}

// directInputs returns the inputs that should trigger the pipeline.  Inputs
// from the same repo/branch are de-duplicated.
func (a *APIServer) directInputs(ctx context.Context) []*pps.AtomInput {
	repoSet := make(map[string]bool)
	var atomInputs []*pps.AtomInput
	pps.VisitInput(a.pipelineInfo.Input, func(input *pps.Input) {
		if input.Atom != nil {
			if repoSet[input.Atom.Repo] {
				return
			}
			repoSet[input.Atom.Repo] = true
			atomInputs = append(atomInputs, input.Atom)
		}
	})
	return atomInputs
}

// rootInputs returns the root provenance of direct inputs.
func (a *APIServer) rootInputs(ctx context.Context) ([]*pps.AtomInput, error) {
	atomInputs := a.directInputs(ctx)
	return a._rootInputs(ctx, atomInputs)
}

func (a *APIServer) _rootInputs(ctx context.Context, atomInputs []*pps.AtomInput) ([]*pps.AtomInput, error) {
	pfsClient := a.pachClient.PfsAPIClient
	ppsClient := a.pachClient.PpsAPIClient
	// map from repo.Name + branch to *pps.AtomInput so that we don't
	// repeat ourselves.
	resultMap := make(map[string]*pps.AtomInput)
	for _, atomInput := range atomInputs {
		repoInfo, err := pfsClient.InspectRepo(ctx, &pfs.InspectRepoRequest{client.NewRepo(atomInput.Repo)})
		if err != nil {
			return nil, err
		}
		if len(repoInfo.Provenance) == 0 {
			resultMap[atomInput.Repo+atomInput.Branch] = atomInput
			continue
		}
		// if the repo has nonzero provenance we know that it's a pipeline
		pipelineInfo, err := ppsClient.InspectPipeline(ctx, &pps.InspectPipelineRequest{client.NewPipeline(atomInput.Repo)})
		if err != nil {
			return nil, err
		}
		// TODO we should be propagating `from_commit` here
		var visitErr error
		pps.VisitInput(pipelineInfo.Input, func(input *pps.Input) {
			if input.Atom != nil {
				subResults, err := a._rootInputs(ctx, []*pps.AtomInput{input.Atom})
				if err != nil && visitErr == nil {
					visitErr = err
				}
				for _, atomInput := range subResults {
					resultMap[atomInput.Repo+atomInput.Branch] = atomInput
				}
			}
		})
		if visitErr != nil {
			return nil, visitErr
		}
	}
	var result []*pps.AtomInput
	for _, atomInput := range resultMap {
		result = append(result, atomInput)
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Package dnsdisco is a DNS service discovery library with health check and
// load balancer features.
//
// The library is very flexible and uses interfaces everywhere to make it
// possible for the library user to replace any part with a custom algorithm. A
// basic use would be:
//
//    package main
//
//    import (
//      "fmt"
//      "github.com/rafaeljusto/dnsdisco"
//    )
//
//    func main() {
//      target, port, err := dnsdisco.Discover("jabber", "tcp", "registro.br")
//      if err != nil {
//        fmt.Println(err)
//        return
//      }
//
//      fmt.Printf("Target: %s\nPort: %d\n", target, port)
//    }
package dnsdisco

import (
	"fmt"
	"net"
)

// Discover is the fastest way to find a target using all the default
// parameters. It will send a SRV query in _service._proto.name format and
// return the first target (address and port) that passed on the health check
// (simple connection check).
//
// proto must be "udp" or "tcp", otherwise an UnknownNetworkError error will be
// returned. The library will use the local resolver to send the DNS package.
func Discover(service, proto, name string) (target string, port uint16, err error) {
	discovery := NewDiscovery(service, proto, name)
	if err = discovery.Refresh(); err != nil {
		return
	}

	target, port = discovery.Choose()
	return
}

// Discovery stores all the necessary information to discover the services,
// check if it still works and choose the best one.
type Discovery struct {
	// Service is the name of the application that the library is looking for.
	Service string

	// Proto is the protocol used by the application. Could be "udp" or "tcp".
	Proto string

	// Name is the domain name where the library will look for the SRV records.
	Name string

	// Retriever is responsible for sending the SRV requests. It is possible to
	// implement this interface to change the retrieve behaviour, that by default
	// queries the local resolver.
	Retriever retriever

	// HealthChecker is responsible for verifying if the target is still on, if
	// not the library can move to the next target. By default the health check
	// only tries a simple connection to the target.
	HealthChecker healthChecker

	// Balancer is responsible for choosing the target that will be used. It has
	// the healthChecker as parameter to make it possible to choose only an online
	// target. By default the library choose the first online target from the
	// list, as it is already ordered by priority and weight.
	Balancer balancer

	// services stores the retrieved services to avoid DNS requests all the time.
	services []*net.SRV
}

// NewDiscovery builds a Discovery type with all default values. To retrieve the
// services it will use the net.LookupSRV (local resolver), for health check
// will only perform a simple connection, and the chosen target will be the
// first online one.
func NewDiscovery(service, proto, name string) Discovery {
	return Discovery{
		Service: service,
		Name:    name,
		Proto:   proto,

		Retriever: Retrieve(func(service, proto, name string) (services []*net.SRV, err error) {
			_, services, err = net.LookupSRV(service, proto, name)
			return
		}),

		HealthChecker: HealthCheck(func(target string, port uint16, proto string) (ok bool, err error) {
			address := fmt.Sprintf("%s:%d", target, port)
			if proto != "tcp" && proto != "udp" {
				return false, net.UnknownNetworkError(proto)
			}

			conn, err := net.Dial(proto, address)
			if err != nil {
				return false, err
			}
			conn.Close()
			return true, nil
		}),

		Balancer: Balance(func(services []*net.SRV, healthCheck healthChecker, proto string) (index int) {
			for i, service := range services {
				ok, err := healthCheck.Check(service.Target, service.Port, proto)
				if err != nil || !ok {
					continue
				}

				return i
			}

			return -1
		}),
	}
}

// Refresh retrieves the services using the DNS SRV solution. It is possible to
// change the default behaviour (local resolver with default timeouts) replacing
// the Retriever attribute from the Discovery type.
func (d *Discovery) Refresh() (err error) {
	d.services, err = d.Retriever.Retrieve(d.Service, d.Proto, d.Name)
	return
}

// Choose will return the best target to use based on a defined balancer. By
// default the library choose the first online target with best priority and
// weight. It is possible to change the balancer behaviour replacing the
// Balancer attribute from the Discovery type.
func (d Discovery) Choose() (target string, port uint16) {
	if i := d.Balancer.Balance(d.services, d.HealthChecker, d.Proto); i >= 0 && i < len(d.services) {
		return d.services[i].Target, d.services[i].Port
	}
	return
}

// retriever allows the library user to define a custom DNS retrieve algorithm.
type retriever interface {
	// Retrieve will send the DNS request and return all SRV records retrieved
	// from the response.
	Retrieve(service, proto, name string) ([]*net.SRV, error)
}

// Retrieve is an easy-to-use implementation of the interface that is
// responsible for sending the DNS SRV requests.
type Retrieve func(service, proto, name string) ([]*net.SRV, error)

// Retrieve will send the DNS request and return all SRV records retrieved from
// the response.
func (r Retrieve) Retrieve(service, proto, name string) ([]*net.SRV, error) {
	return r(service, proto, name)
}

// healthChecker allows the library user to define a custom health check
// algorithm.
type healthChecker interface {
	// Check will analyze the target port/proto to check if it is still capable of
	// receiving requests.
	Check(target string, port uint16, proto string) (ok bool, err error)
}

// HealthCheck is an easy-to-use implementation of the interface that is
// responsible for checking if a target is still alive.
type HealthCheck func(target string, port uint16, proto string) (ok bool, err error)

// Check will analyze the target port/proto to check if it is still capable of
// receiving requests.
func (h HealthCheck) Check(target string, port uint16, proto string) (ok bool, err error) {
	return h(target, port, proto)
}

// balancer allows the library user to define a custom balance algorithm.
type balancer interface {
	// Balance will choose the best target.
	Balance(services []*net.SRV, healthCheck healthChecker, proto string) (index int)
}

// Balance is an easy-to-use implementation of the interface that is responsible
// for choosing the best target.
type Balance func(services []*net.SRV, healthCheck healthChecker, proto string) (index int)

// Balance will choose the best target.
func (b Balance) Balance(services []*net.SRV, healthCheck healthChecker, proto string) (index int) {
	return b(services, healthCheck, proto)
}
<commit_msg>Rename types to keep the same naming style as http package (HandlerFunc)<commit_after>// Package dnsdisco is a DNS service discovery library with health check and
// load balancer features.
//
// The library is very flexible and uses interfaces everywhere to make it
// possible for the library user to replace any part with a custom algorithm. A
// basic use would be:
//
//    package main
//
//    import (
//      "fmt"
//      "github.com/rafaeljusto/dnsdisco"
//    )
//
//    func main() {
//      target, port, err := dnsdisco.Discover("jabber", "tcp", "registro.br")
//      if err != nil {
//        fmt.Println(err)
//        return
//      }
//
//      fmt.Printf("Target: %s\nPort: %d\n", target, port)
//    }
package dnsdisco

import (
	"fmt"
	"net"
)

// Discover is the fastest way to find a target using all the default
// parameters. It will send a SRV query in _service._proto.name format and
// return the first target (address and port) that passed on the health check
// (simple connection check).
//
// proto must be "udp" or "tcp", otherwise an UnknownNetworkError error will be
// returned. The library will use the local resolver to send the DNS package.
func Discover(service, proto, name string) (target string, port uint16, err error) {
	discovery := NewDiscovery(service, proto, name)
	if err = discovery.Refresh(); err != nil {
		return
	}

	target, port = discovery.Choose()
	return
}

// Discovery stores all the necessary information to discover the services,
// check if it still works and choose the best one.
type Discovery struct {
	// Service is the name of the application that the library is looking for.
	Service string

	// Proto is the protocol used by the application. Could be "udp" or "tcp".
	Proto string

	// Name is the domain name where the library will look for the SRV records.
	Name string

	// Retriever is responsible for sending the SRV requests. It is possible to
	// implement this interface to change the retrieve behaviour, that by default
	// queries the local resolver.
	Retriever retriever

	// HealthChecker is responsible for verifying if the target is still on, if
	// not the library can move to the next target. By default the health check
	// only tries a simple connection to the target.
	HealthChecker healthChecker

	// Balancer is responsible for choosing the target that will be used. It has
	// the healthChecker as parameter to make it possible to choose only an online
	// target. By default the library choose the first online target from the
	// list, as it is already ordered by priority and weight.
	Balancer balancer

	// services stores the retrieved services to avoid DNS requests all the time.
	services []*net.SRV
}

// NewDiscovery builds a Discovery type with all default values. To retrieve the
// services it will use the net.LookupSRV (local resolver), for health check
// will only perform a simple connection, and the chosen target will be the
// first online one.
func NewDiscovery(service, proto, name string) Discovery {
	return Discovery{
		Service: service,
		Name:    name,
		Proto:   proto,

		Retriever: RetrieverFunc(func(service, proto, name string) (services []*net.SRV, err error) {
			_, services, err = net.LookupSRV(service, proto, name)
			return
		}),

		HealthChecker: HealthCheckerFunc(func(target string, port uint16, proto string) (ok bool, err error) {
			address := fmt.Sprintf("%s:%d", target, port)
			if proto != "tcp" && proto != "udp" {
				return false, net.UnknownNetworkError(proto)
			}

			conn, err := net.Dial(proto, address)
			if err != nil {
				return false, err
			}
			conn.Close()
			return true, nil
		}),

		Balancer: BalancerFunc(func(services []*net.SRV, healthCheck healthChecker, proto string) (index int) {
			for i, service := range services {
				ok, err := healthCheck.HealthCheck(service.Target, service.Port, proto)
				if err != nil || !ok {
					continue
				}

				return i
			}

			return -1
		}),
	}
}

// Refresh retrieves the services using the DNS SRV solution. It is possible to
// change the default behaviour (local resolver with default timeouts) replacing
// the Retriever attribute from the Discovery type.
func (d *Discovery) Refresh() (err error) {
	d.services, err = d.Retriever.Retrieve(d.Service, d.Proto, d.Name)
	return
}

// Choose will return the best target to use based on a defined balancer. By
// default the library choose the first online target with best priority and
// weight. It is possible to change the balancer behaviour replacing the
// Balancer attribute from the Discovery type.
func (d Discovery) Choose() (target string, port uint16) {
	if i := d.Balancer.Balance(d.services, d.HealthChecker, d.Proto); i >= 0 && i < len(d.services) {
		return d.services[i].Target, d.services[i].Port
	}
	return
}

// retriever allows the library user to define a custom DNS retrieve algorithm.
type retriever interface {
	// Retrieve will send the DNS request and return all SRV records retrieved
	// from the response.
	Retrieve(service, proto, name string) ([]*net.SRV, error)
}

// RetrieverFunc is an easy-to-use implementation of the interface that is
// responsible for sending the DNS SRV requests.
type RetrieverFunc func(service, proto, name string) ([]*net.SRV, error)

// Retrieve will send the DNS request and return all SRV records retrieved from
// the response.
func (r RetrieverFunc) Retrieve(service, proto, name string) ([]*net.SRV, error) {
	return r(service, proto, name)
}

// healthChecker allows the library user to define a custom health check
// algorithm.
type healthChecker interface {
	// HealthCheck will analyze the target port/proto to check if it is still
	// capable of receiving requests.
	HealthCheck(target string, port uint16, proto string) (ok bool, err error)
}

// HealthCheckerFunc is an easy-to-use implementation of the interface that is
// responsible for checking if a target is still alive.
type HealthCheckerFunc func(target string, port uint16, proto string) (ok bool, err error)

// HealthCheck will analyze the target port/proto to check if it is still
// capable of receiving requests.
func (h HealthCheckerFunc) HealthCheck(target string, port uint16, proto string) (ok bool, err error) {
	return h(target, port, proto)
}

// balancer allows the library user to define a custom balance algorithm.
type balancer interface {
	// Balance will choose the best target.
	Balance(services []*net.SRV, healthCheck healthChecker, proto string) (index int)
}

// BalancerFunc is an easy-to-use implementation of the interface that is
// responsible for choosing the best target.
type BalancerFunc func(services []*net.SRV, healthCheck healthChecker, proto string) (index int)

// Balance will choose the best target.
func (b BalancerFunc) Balance(services []*net.SRV, healthCheck healthChecker, proto string) (index int) {
	return b(services, healthCheck, proto)
}
<|endoftext|>


<commit_before>package finalize

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"bytes"

	"github.com/cloudfoundry/libbuildpack"
)

type Staticfile struct {
	RootDir               string `yaml:"root"`
	HostDotFiles          bool   `yaml:"host_dot_files"`
	LocationInclude       string `yaml:"location_include"`
	DirectoryIndex        bool   `yaml:"directory"`
	SSI                   bool   `yaml:"ssi"`
	PushState             bool   `yaml:"pushstate"`
	HSTS                  bool   `yaml:"http_strict_transport_security"`
	HSTSIncludeSubDomains bool   `yaml:"http_strict_transport_security_include_subdomains"`
	HSTSPreload           bool   `yaml:"http_strict_transport_security_preload"`
	ForceHTTPS            bool   `yaml:"force_https"`
	BasicAuth             bool
}

type YAML interface {
	Load(string, interface{}) error
}

type Finalizer struct {
	BuildDir string
	DepDir   string
	Log      *libbuildpack.Logger
	Config   Staticfile
	YAML     YAML
}

var skipCopyFile = map[string]bool{
	"Staticfile":      true,
	"Staticfile.auth": true,
	"manifest.yml":    true,
	".profile":        true,
	".profile.d":      true,
	"stackato.yml":    true,
	".cloudfoundry":   true,
}

func Run(sf *Finalizer) error {
	var err error

	err = sf.LoadStaticfile()
	if err != nil {
		sf.Log.Error("Unable to load Staticfile: %s", err.Error())
		return err
	}

	appRootDir, err := sf.GetAppRootDir()
	if err != nil {
		sf.Log.Error("Invalid root directory: %s", err.Error())
		return err
	}

	sf.Warnings()

	err = sf.CopyFilesToPublic(appRootDir)
	if err != nil {
		sf.Log.Error("Unable to copy project files: %s", err.Error())
		return err
	}

	err = sf.ConfigureNginx()
	if err != nil {
		sf.Log.Error("Unable to configure nginx: %s", err.Error())
		return err
	}

	err = sf.WriteStartupFiles()
	if err != nil {
		sf.Log.Error("Unable to write startup file: %s", err.Error())
		return err
	}
	return nil
}

func (sf *Finalizer) WriteStartupFiles() error {
	profiledDir := filepath.Join(sf.DepDir, "profile.d")
	err := os.MkdirAll(profiledDir, 0755)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(profiledDir, "staticfile.sh"), []byte(initScript), 0755)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(sf.BuildDir, "start_logging.sh"), []byte(startLoggingScript), 0755)
	if err != nil {
		return err
	}

	bootScript := filepath.Join(sf.BuildDir, "boot.sh")
	return ioutil.WriteFile(bootScript, []byte(startCommand), 0755)
}

func (sf *Finalizer) LoadStaticfile() error {
	var hash = make(map[string]string)
	conf := &sf.Config

	err := sf.YAML.Load(filepath.Join(sf.BuildDir, "Staticfile"), &hash)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for key, value := range hash {
		isEnabled := (value == "enabled" || value == "true")
		switch key {
		case "root":
			conf.RootDir = value
		case "host_dot_files":
			if isEnabled {
				sf.Log.BeginStep("Enabling hosting of dotfiles")
				conf.HostDotFiles = true
			}
		case "location_include":
			conf.LocationInclude = value
			if conf.LocationInclude != "" {
				sf.Log.BeginStep("Enabling location include file %s", conf.LocationInclude)
			}
		case "directory":
			if value != "" {
				sf.Log.BeginStep("Enabling directory index for folders without index.html files")
				conf.DirectoryIndex = true
			}
		case "ssi":
			if isEnabled {
				sf.Log.BeginStep("Enabling SSI")
				conf.SSI = true
			}
		case "pushstate":
			if isEnabled {
				sf.Log.BeginStep("Enabling pushstate")
				conf.PushState = true
			}
		case "http_strict_transport_security":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS")
				conf.HSTS = true
			}
		case "http_strict_transport_security_include_subdomains":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS and HSTS includeSubDomains")
				conf.HSTS = true
				conf.HSTSIncludeSubDomains = true
			}
		case "http_strict_transport_security_preload":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS and HSTS Preload")
				conf.HSTS = true
				conf.HSTSIncludeSubDomains = true
				conf.HSTSPreload = true
			}
		case "force_https":
			if isEnabled {
				sf.Log.BeginStep("Enabling HTTPS redirect")
				conf.ForceHTTPS = true
			}
		}
	}

	authFile := filepath.Join(sf.BuildDir, "Staticfile.auth")
	_, err = os.Stat(authFile)
	if err == nil {
		conf.BasicAuth = true
		sf.Log.BeginStep("Enabling basic authentication using Staticfile.auth")
		sf.Log.Protip("Learn about basic authentication", "http://docs.cloudfoundry.org/buildpacks/staticfile/index.html#authentication")
	}

	return nil
}

func (sf *Finalizer) GetAppRootDir() (string, error) {
	var rootDirRelative string

	if sf.Config.RootDir != "" {
		rootDirRelative = sf.Config.RootDir
	} else {
		rootDirRelative = "."
	}

	rootDirAbs, err := filepath.Abs(filepath.Join(sf.BuildDir, rootDirRelative))
	if err != nil {
		return "", err
	}

	sf.Log.BeginStep("Root folder %s", rootDirAbs)

	dirInfo, err := os.Stat(rootDirAbs)
	if err != nil {
		return "", fmt.Errorf("the application Staticfile specifies a root directory %s that does not exist", rootDirRelative)
	}

	if !dirInfo.IsDir() {
		return "", fmt.Errorf("the application Staticfile specifies a root directory %s that is a plain file, but was expected to be a directory", rootDirRelative)
	}

	return rootDirAbs, nil
}

func (sf *Finalizer) CopyFilesToPublic(appRootDir string) error {
	sf.Log.BeginStep("Copying project files into public")

	publicDir := filepath.Join(sf.BuildDir, "public")

	if publicDir == appRootDir {
		return nil
	}

	tmpDir, err := ioutil.TempDir("", "staticfile-buildpack.approot.")
	if err != nil {
		return err
	}

	files, err := ioutil.ReadDir(appRootDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if skipCopyFile[file.Name()] {
			continue
		}

		if strings.HasPrefix(file.Name(), ".") && !sf.Config.HostDotFiles {
			continue
		}

		err = os.Rename(filepath.Join(appRootDir, file.Name()), filepath.Join(tmpDir, file.Name()))
		if err != nil {
			return err
		}
	}

	if err := os.RemoveAll(publicDir); err != nil {
		return err
	}

	if err := os.Rename(tmpDir, publicDir); err != nil {
		return err
	}

	return nil
}

func (sf *Finalizer) Warnings() {
	if len(sf.Config.LocationInclude) > 0 && len(sf.Config.RootDir) == 0 {
		sf.Log.Warning("The location_include directive only works in conjunction with root.\nPlease specify root to use location_include")
	}

	if len(sf.Config.RootDir) == 0 {
		found, _ := libbuildpack.FileExists(filepath.Join(sf.BuildDir, "nginx", "conf"))
		if found {
			sf.Log.Info("\n\n\n")
			sf.Log.Warning("You have an nginx/conf directory, but have not set *root*.\nIf you are using the nginx/conf directory for nginx configuration, you probably need to also set the *root* directive.")
			sf.Log.Info("\n\n\n")
		}
	}
}

func (sf *Finalizer) ConfigureNginx() error {
	var err error

	sf.Log.BeginStep("Configuring nginx")

	nginxConf, err := sf.generateNginxConf()
	if err != nil {
		sf.Log.Error("Unable to generate nginx.conf: %s", err.Error())
		return err
	}

	confDir := filepath.Join(sf.BuildDir, "nginx", "conf")
	if err := os.MkdirAll(confDir, 0755); err != nil {
		return err
	}

	logsDir := filepath.Join(sf.BuildDir, "nginx", "logs")
	if err := os.MkdirAll(logsDir, 0755); err != nil {
		return err
	}

	confFiles := map[string]string{
		"nginx.conf": nginxConf,
		"mime.types": MimeTypes}

	for file, contents := range confFiles {
		confDest := filepath.Join(confDir, file)
		customConfFile := filepath.Join(sf.BuildDir, "public", file)

		_, err = os.Stat(customConfFile)
		if err == nil {
			err = libbuildpack.CopyFile(customConfFile, confDest)
		} else {
			err = ioutil.WriteFile(confDest, []byte(contents), 0644)
		}

		if err != nil {
			return err
		}
	}

	if sf.Config.BasicAuth {
		authFile := filepath.Join(sf.BuildDir, "Staticfile.auth")
		err = libbuildpack.CopyFile(authFile, filepath.Join(confDir, ".htpasswd"))
		if err != nil {
			return err
		}
	}

	return nil
}

func (sf *Finalizer) generateNginxConf() (string, error) {
	buffer := new(bytes.Buffer)

	t := template.Must(template.New("nginx.conf").Parse(nginxConfTemplate))

	err := t.Execute(buffer, sf.Config)
	if err != nil {
		return "", err
	}
	return buffer.String(), nil
}
<commit_msg>Be honest in the logs about what's happening<commit_after>package finalize

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"bytes"

	"github.com/cloudfoundry/libbuildpack"
)

type Staticfile struct {
	RootDir               string `yaml:"root"`
	HostDotFiles          bool   `yaml:"host_dot_files"`
	LocationInclude       string `yaml:"location_include"`
	DirectoryIndex        bool   `yaml:"directory"`
	SSI                   bool   `yaml:"ssi"`
	PushState             bool   `yaml:"pushstate"`
	HSTS                  bool   `yaml:"http_strict_transport_security"`
	HSTSIncludeSubDomains bool   `yaml:"http_strict_transport_security_include_subdomains"`
	HSTSPreload           bool   `yaml:"http_strict_transport_security_preload"`
	ForceHTTPS            bool   `yaml:"force_https"`
	BasicAuth             bool
}

type YAML interface {
	Load(string, interface{}) error
}

type Finalizer struct {
	BuildDir string
	DepDir   string
	Log      *libbuildpack.Logger
	Config   Staticfile
	YAML     YAML
}

var skipCopyFile = map[string]bool{
	"Staticfile":      true,
	"Staticfile.auth": true,
	"manifest.yml":    true,
	".profile":        true,
	".profile.d":      true,
	"stackato.yml":    true,
	".cloudfoundry":   true,
}

func Run(sf *Finalizer) error {
	var err error

	err = sf.LoadStaticfile()
	if err != nil {
		sf.Log.Error("Unable to load Staticfile: %s", err.Error())
		return err
	}

	appRootDir, err := sf.GetAppRootDir()
	if err != nil {
		sf.Log.Error("Invalid root directory: %s", err.Error())
		return err
	}

	sf.Warnings()

	err = sf.CopyFilesToPublic(appRootDir)
	if err != nil {
		sf.Log.Error("Unable to copy project files: %s", err.Error())
		return err
	}

	err = sf.ConfigureNginx()
	if err != nil {
		sf.Log.Error("Unable to configure nginx: %s", err.Error())
		return err
	}

	err = sf.WriteStartupFiles()
	if err != nil {
		sf.Log.Error("Unable to write startup file: %s", err.Error())
		return err
	}
	return nil
}

func (sf *Finalizer) WriteStartupFiles() error {
	profiledDir := filepath.Join(sf.DepDir, "profile.d")
	err := os.MkdirAll(profiledDir, 0755)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(profiledDir, "staticfile.sh"), []byte(initScript), 0755)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(filepath.Join(sf.BuildDir, "start_logging.sh"), []byte(startLoggingScript), 0755)
	if err != nil {
		return err
	}

	bootScript := filepath.Join(sf.BuildDir, "boot.sh")
	return ioutil.WriteFile(bootScript, []byte(startCommand), 0755)
}

func (sf *Finalizer) LoadStaticfile() error {
	var hash = make(map[string]string)
	conf := &sf.Config

	err := sf.YAML.Load(filepath.Join(sf.BuildDir, "Staticfile"), &hash)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for key, value := range hash {
		isEnabled := (value == "enabled" || value == "true")
		switch key {
		case "root":
			conf.RootDir = value
		case "host_dot_files":
			if isEnabled {
				sf.Log.BeginStep("Enabling hosting of dotfiles")
				conf.HostDotFiles = true
			}
		case "location_include":
			conf.LocationInclude = value
			if conf.LocationInclude != "" {
				sf.Log.BeginStep("Enabling location include file %s", conf.LocationInclude)
			}
		case "directory":
			if value != "" {
				sf.Log.BeginStep("Enabling directory index for folders without index.html files")
				conf.DirectoryIndex = true
			}
		case "ssi":
			if isEnabled {
				sf.Log.BeginStep("Enabling SSI")
				conf.SSI = true
			}
		case "pushstate":
			if isEnabled {
				sf.Log.BeginStep("Enabling pushstate")
				conf.PushState = true
			}
		case "http_strict_transport_security":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS")
				conf.HSTS = true
			}
		case "http_strict_transport_security_include_subdomains":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS and HSTS includeSubDomains")
				conf.HSTS = true
				conf.HSTSIncludeSubDomains = true
			}
		case "http_strict_transport_security_preload":
			if isEnabled {
				sf.Log.BeginStep("Enabling HSTS, HSTS includeSubDomains, and HSTS Preload")
				conf.HSTS = true
				conf.HSTSIncludeSubDomains = true
				conf.HSTSPreload = true
			}
		case "force_https":
			if isEnabled {
				sf.Log.BeginStep("Enabling HTTPS redirect")
				conf.ForceHTTPS = true
			}
		}
	}

	authFile := filepath.Join(sf.BuildDir, "Staticfile.auth")
	_, err = os.Stat(authFile)
	if err == nil {
		conf.BasicAuth = true
		sf.Log.BeginStep("Enabling basic authentication using Staticfile.auth")
		sf.Log.Protip("Learn about basic authentication", "http://docs.cloudfoundry.org/buildpacks/staticfile/index.html#authentication")
	}

	return nil
}

func (sf *Finalizer) GetAppRootDir() (string, error) {
	var rootDirRelative string

	if sf.Config.RootDir != "" {
		rootDirRelative = sf.Config.RootDir
	} else {
		rootDirRelative = "."
	}

	rootDirAbs, err := filepath.Abs(filepath.Join(sf.BuildDir, rootDirRelative))
	if err != nil {
		return "", err
	}

	sf.Log.BeginStep("Root folder %s", rootDirAbs)

	dirInfo, err := os.Stat(rootDirAbs)
	if err != nil {
		return "", fmt.Errorf("the application Staticfile specifies a root directory %s that does not exist", rootDirRelative)
	}

	if !dirInfo.IsDir() {
		return "", fmt.Errorf("the application Staticfile specifies a root directory %s that is a plain file, but was expected to be a directory", rootDirRelative)
	}

	return rootDirAbs, nil
}

func (sf *Finalizer) CopyFilesToPublic(appRootDir string) error {
	sf.Log.BeginStep("Copying project files into public")

	publicDir := filepath.Join(sf.BuildDir, "public")

	if publicDir == appRootDir {
		return nil
	}

	tmpDir, err := ioutil.TempDir("", "staticfile-buildpack.approot.")
	if err != nil {
		return err
	}

	files, err := ioutil.ReadDir(appRootDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if skipCopyFile[file.Name()] {
			continue
		}

		if strings.HasPrefix(file.Name(), ".") && !sf.Config.HostDotFiles {
			continue
		}

		err = os.Rename(filepath.Join(appRootDir, file.Name()), filepath.Join(tmpDir, file.Name()))
		if err != nil {
			return err
		}
	}

	if err := os.RemoveAll(publicDir); err != nil {
		return err
	}

	if err := os.Rename(tmpDir, publicDir); err != nil {
		return err
	}

	return nil
}

func (sf *Finalizer) Warnings() {
	if len(sf.Config.LocationInclude) > 0 && len(sf.Config.RootDir) == 0 {
		sf.Log.Warning("The location_include directive only works in conjunction with root.\nPlease specify root to use location_include")
	}

	if len(sf.Config.RootDir) == 0 {
		found, _ := libbuildpack.FileExists(filepath.Join(sf.BuildDir, "nginx", "conf"))
		if found {
			sf.Log.Info("\n\n\n")
			sf.Log.Warning("You have an nginx/conf directory, but have not set *root*.\nIf you are using the nginx/conf directory for nginx configuration, you probably need to also set the *root* directive.")
			sf.Log.Info("\n\n\n")
		}
	}
}

func (sf *Finalizer) ConfigureNginx() error {
	var err error

	sf.Log.BeginStep("Configuring nginx")

	nginxConf, err := sf.generateNginxConf()
	if err != nil {
		sf.Log.Error("Unable to generate nginx.conf: %s", err.Error())
		return err
	}

	confDir := filepath.Join(sf.BuildDir, "nginx", "conf")
	if err := os.MkdirAll(confDir, 0755); err != nil {
		return err
	}

	logsDir := filepath.Join(sf.BuildDir, "nginx", "logs")
	if err := os.MkdirAll(logsDir, 0755); err != nil {
		return err
	}

	confFiles := map[string]string{
		"nginx.conf": nginxConf,
		"mime.types": MimeTypes}

	for file, contents := range confFiles {
		confDest := filepath.Join(confDir, file)
		customConfFile := filepath.Join(sf.BuildDir, "public", file)

		_, err = os.Stat(customConfFile)
		if err == nil {
			err = libbuildpack.CopyFile(customConfFile, confDest)
		} else {
			err = ioutil.WriteFile(confDest, []byte(contents), 0644)
		}

		if err != nil {
			return err
		}
	}

	if sf.Config.BasicAuth {
		authFile := filepath.Join(sf.BuildDir, "Staticfile.auth")
		err = libbuildpack.CopyFile(authFile, filepath.Join(confDir, ".htpasswd"))
		if err != nil {
			return err
		}
	}

	return nil
}

func (sf *Finalizer) generateNginxConf() (string, error) {
	buffer := new(bytes.Buffer)

	t := template.Must(template.New("nginx.conf").Parse(nginxConfTemplate))

	err := t.Execute(buffer, sf.Config)
	if err != nil {
		return "", err
	}
	return buffer.String(), nil
}
<|endoftext|>


<commit_before>package eth

import (
	"container/list"
	"fmt"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethdb"
	"github.com/ethereum/eth-go/ethrpc"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/eth-go/ethwire"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	quit         chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db ethutil.Database
	// State manager for processing new blocks and managing the over all states
	stateManager *ethchain.StateManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	txPool *ethchain.TxPool
	// The canonical chain
	blockChain *ethchain.BlockChain
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr
	Port string

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps

	nat NAT

	// Specifies the desired amount of maximum peers
	MaxPeers int

	Mining bool

	listening bool

	reactor *ethutil.ReactorEngine

	RpcServer *ethrpc.JsonRpcServer
}

func New(caps Caps, usePnp bool) (*Ethereum, error) {
	db, err := ethdb.NewLDBDatabase("database")
	//db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	var nat NAT
	if usePnp {
		nat, err = Discover()
		if err != nil {
			ethutil.Config.Log.Debugln("UPnP failed", err)
		}
	}

	ethutil.Config.Db = db

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		quit:         make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
		nat:          nat,
	}
	ethereum.reactor = ethutil.NewReactorEngine()

	ethereum.txPool = ethchain.NewTxPool(ethereum)
	ethereum.blockChain = ethchain.NewBlockChain(ethereum)
	ethereum.stateManager = ethchain.NewStateManager(ethereum)

	// Start the tx pool
	ethereum.txPool.Start()

	return ethereum, nil
}

func (s *Ethereum) Reactor() *ethutil.ReactorEngine {
	return s.reactor
}

func (s *Ethereum) BlockChain() *ethchain.BlockChain {
	return s.blockChain
}

func (s *Ethereum) StateManager() *ethchain.StateManager {
	return s.stateManager
}

func (s *Ethereum) TxPool() *ethchain.TxPool {
	return s.txPool
}

func (s *Ethereum) ServerCaps() Caps {
	return s.serverCaps
}
func (s *Ethereum) IsMining() bool {
	return s.Mining
}
func (s *Ethereum) PeerCount() int {
	return s.peers.Len()
}
func (s *Ethereum) IsUpToDate() bool {
	upToDate := true
	eachPeer(s.peers, func(peer *Peer, e *list.Element) {
		if atomic.LoadInt32(&peer.connected) == 1 {
			if peer.catchingUp == true {
				upToDate = false
			}
		}
	})
	return upToDate
}
func (s *Ethereum) PushPeer(peer *Peer) {
	s.peers.PushBack(peer)
}
func (s *Ethereum) IsListening() bool {
	return s.listening
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil {
		if s.peers.Len() < s.MaxPeers {
			peer.Start()
		} else {
			ethutil.Config.Log.Debugf("[SERV] Max connected peers reached. Not adding incoming peer.")
		}
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	if s.peers.Len() < s.MaxPeers {
		var alreadyConnected bool

		ahost, _, _ := net.SplitHostPort(addr)
		var chost string

		ips, err := net.LookupIP(ahost)

		if err != nil {
			return err
		} else {
			// If more then one ip is available try stripping away the ipv6 ones
			if len(ips) > 1 {
				var ipsv4 []net.IP
				// For now remove the ipv6 addresses
				for _, ip := range ips {
					if strings.Contains(ip.String(), "::") {
						continue
					} else {
						ipsv4 = append(ipsv4, ip)
					}
				}
				if len(ipsv4) == 0 {
					return fmt.Errorf("[SERV] No IPV4 addresses available for hostname")
				}

				// Pick a random ipv4 address, simulating round-robin DNS.
				rand.Seed(time.Now().UTC().UnixNano())
				i := rand.Intn(len(ipsv4))
				chost = ipsv4[i].String()
			} else {
				if len(ips) == 0 {
					return fmt.Errorf("[SERV] No IPs resolved for the given hostname")
					return nil
				}
				chost = ips[0].String()
			}
		}

		eachPeer(s.peers, func(p *Peer, v *list.Element) {
			if p.conn == nil {
				return
			}
			phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())

			if phost == chost {
				alreadyConnected = true
				//ethutil.Config.Log.Debugf("[SERV] Peer %s already added.\n", chost)
				return
			}
		})

		if alreadyConnected {
			return nil
		}

		NewOutboundPeer(addr, s, s.serverCaps)
	}

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.removePeerElement(e)
		}
	})
}

func (s *Ethereum) removePeerElement(e *list.Element) {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	s.peers.Remove(e)

	s.reactor.Post("peerList", s.peers)
}

func (s *Ethereum) RemovePeer(p *Peer) {
	eachPeer(s.peers, func(peer *Peer, e *list.Element) {
		if peer == p {
			s.removePeerElement(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// Start the ethereum
func (s *Ethereum) Start(seed bool) {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":"+s.Port)
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
		s.listening = false
	} else {
		s.listening = true
		// Starting accepting connections
		ethutil.Config.Log.Infoln("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	if s.nat != nil {
		go s.upnpUpdateThread()
	}

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	if seed {
		s.Seed()
	}
}

func (s *Ethereum) Seed() {
	ethutil.Config.Log.Debugln("[SERV] Retrieving seed nodes")

	// Eth-Go Bootstrapping
	ips, er := net.LookupIP("seed.bysh.me")
	if er == nil {
		peers := []string{}
		for _, ip := range ips {
			node := fmt.Sprintf("%s:%d", ip.String(), 30303)
			ethutil.Config.Log.Debugln("[SERV] Found DNS Go Peer:", node)
			peers = append(peers, node)
		}
		s.ProcessPeerList(peers)
	}

	// Official DNS Bootstrapping
	_, nodes, err := net.LookupSRV("eth", "tcp", "ethereum.org")
	if err == nil {
		peers := []string{}
		// Iterate SRV nodes
		for _, n := range nodes {
			target := n.Target
			port := strconv.Itoa(int(n.Port))
			// Resolve target to ip (Go returns list, so may resolve to multiple ips?)
			addr, err := net.LookupHost(target)
			if err == nil {
				for _, a := range addr {
					// Build string out of SRV port and Resolved IP
					peer := net.JoinHostPort(a, port)
					ethutil.Config.Log.Debugln("[SERV] Found DNS Bootstrap Peer:", peer)
					peers = append(peers, peer)
				}
			} else {
				ethutil.Config.Log.Debugln("[SERV} Couldn't resolve :", target)
			}
		}
		// Connect to Peer list
		s.ProcessPeerList(peers)
	} else {
		// Fallback to servers.poc3.txt
		resp, err := http.Get("http://www.ethereum.org/servers.poc3.txt")
		if err != nil {
			log.Println("Fetching seed failed:", err)
			return
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Println("Reading seed failed:", err)
			return
		}

		s.ConnectToPeer(string(body))
	}
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			ethutil.Config.Log.Debugln(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	close(s.quit)

	if s.RpcServer != nil {
		s.RpcServer.Stop()
	}
	s.txPool.Stop()
	s.stateManager.Stop()

	close(s.shutdownChan)
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}

func (s *Ethereum) upnpUpdateThread() {
	// Go off immediately to prevent code duplication, thereafter we renew
	// lease every 15 minutes.
	timer := time.NewTimer(5 * time.Minute)
	lport, _ := strconv.ParseInt(s.Port, 10, 16)
	first := true
out:
	for {
		select {
		case <-timer.C:
			var err error
			_, err = s.nat.AddPortMapping("TCP", int(lport), int(lport), "eth listen port", 20*60)
			if err != nil {
				ethutil.Config.Log.Debugln("can't add UPnP port mapping:", err)
				break out
			}
			if first && err == nil {
				_, err = s.nat.GetExternalAddress()
				if err != nil {
					ethutil.Config.Log.Debugln("UPnP can't get external address:", err)
					continue out
				}
				first = false
			}
			timer.Reset(time.Minute * 15)
		case <-s.quit:
			break out
		}
	}

	timer.Stop()

	if err := s.nat.DeletePortMapping("TCP", int(lport), int(lport)); err != nil {
		ethutil.Config.Log.Debugln("unable to remove UPnP port mapping:", err)
	} else {
		ethutil.Config.Log.Debugln("succesfully disestablished UPnP port mapping")
	}
}
<commit_msg>Added Block do which replays the given block or error<commit_after>package eth

import (
	"container/list"
	"fmt"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethdb"
	"github.com/ethereum/eth-go/ethrpc"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/eth-go/ethwire"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	quit         chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db ethutil.Database
	// State manager for processing new blocks and managing the over all states
	stateManager *ethchain.StateManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	txPool *ethchain.TxPool
	// The canonical chain
	blockChain *ethchain.BlockChain
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr
	Port string

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps

	nat NAT

	// Specifies the desired amount of maximum peers
	MaxPeers int

	Mining bool

	listening bool

	reactor *ethutil.ReactorEngine

	RpcServer *ethrpc.JsonRpcServer
}

func New(caps Caps, usePnp bool) (*Ethereum, error) {
	db, err := ethdb.NewLDBDatabase("database")
	//db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	var nat NAT
	if usePnp {
		nat, err = Discover()
		if err != nil {
			ethutil.Config.Log.Debugln("UPnP failed", err)
		}
	}

	ethutil.Config.Db = db

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		quit:         make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
		nat:          nat,
	}
	ethereum.reactor = ethutil.NewReactorEngine()

	ethereum.txPool = ethchain.NewTxPool(ethereum)
	ethereum.blockChain = ethchain.NewBlockChain(ethereum)
	ethereum.stateManager = ethchain.NewStateManager(ethereum)

	// Start the tx pool
	ethereum.txPool.Start()

	return ethereum, nil
}

// Replay block
func (self *Ethereum) BlockDo(hash []byte) error {
	block := self.blockChain.GetBlock(hash)
	if block == nil {
		return fmt.Errorf("unknown block %x", hash)
	}

	parent := self.blockChain.GetBlock(block.PrevHash)

	_, err := self.stateManager.ApplyDiff(parent.State(), parent, block)
	if err != nil {
		return err
	}

	return nil

}

func (s *Ethereum) Reactor() *ethutil.ReactorEngine {
	return s.reactor
}

func (s *Ethereum) BlockChain() *ethchain.BlockChain {
	return s.blockChain
}

func (s *Ethereum) StateManager() *ethchain.StateManager {
	return s.stateManager
}

func (s *Ethereum) TxPool() *ethchain.TxPool {
	return s.txPool
}

func (s *Ethereum) ServerCaps() Caps {
	return s.serverCaps
}
func (s *Ethereum) IsMining() bool {
	return s.Mining
}
func (s *Ethereum) PeerCount() int {
	return s.peers.Len()
}
func (s *Ethereum) IsUpToDate() bool {
	upToDate := true
	eachPeer(s.peers, func(peer *Peer, e *list.Element) {
		if atomic.LoadInt32(&peer.connected) == 1 {
			if peer.catchingUp == true {
				upToDate = false
			}
		}
	})
	return upToDate
}
func (s *Ethereum) PushPeer(peer *Peer) {
	s.peers.PushBack(peer)
}
func (s *Ethereum) IsListening() bool {
	return s.listening
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil {
		if s.peers.Len() < s.MaxPeers {
			peer.Start()
		} else {
			ethutil.Config.Log.Debugf("[SERV] Max connected peers reached. Not adding incoming peer.")
		}
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	if s.peers.Len() < s.MaxPeers {
		var alreadyConnected bool

		ahost, _, _ := net.SplitHostPort(addr)
		var chost string

		ips, err := net.LookupIP(ahost)

		if err != nil {
			return err
		} else {
			// If more then one ip is available try stripping away the ipv6 ones
			if len(ips) > 1 {
				var ipsv4 []net.IP
				// For now remove the ipv6 addresses
				for _, ip := range ips {
					if strings.Contains(ip.String(), "::") {
						continue
					} else {
						ipsv4 = append(ipsv4, ip)
					}
				}
				if len(ipsv4) == 0 {
					return fmt.Errorf("[SERV] No IPV4 addresses available for hostname")
				}

				// Pick a random ipv4 address, simulating round-robin DNS.
				rand.Seed(time.Now().UTC().UnixNano())
				i := rand.Intn(len(ipsv4))
				chost = ipsv4[i].String()
			} else {
				if len(ips) == 0 {
					return fmt.Errorf("[SERV] No IPs resolved for the given hostname")
					return nil
				}
				chost = ips[0].String()
			}
		}

		eachPeer(s.peers, func(p *Peer, v *list.Element) {
			if p.conn == nil {
				return
			}
			phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())

			if phost == chost {
				alreadyConnected = true
				//ethutil.Config.Log.Debugf("[SERV] Peer %s already added.\n", chost)
				return
			}
		})

		if alreadyConnected {
			return nil
		}

		NewOutboundPeer(addr, s, s.serverCaps)
	}

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.removePeerElement(e)
		}
	})
}

func (s *Ethereum) removePeerElement(e *list.Element) {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	s.peers.Remove(e)

	s.reactor.Post("peerList", s.peers)
}

func (s *Ethereum) RemovePeer(p *Peer) {
	eachPeer(s.peers, func(peer *Peer, e *list.Element) {
		if peer == p {
			s.removePeerElement(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// Start the ethereum
func (s *Ethereum) Start(seed bool) {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":"+s.Port)
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
		s.listening = false
	} else {
		s.listening = true
		// Starting accepting connections
		ethutil.Config.Log.Infoln("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	if s.nat != nil {
		go s.upnpUpdateThread()
	}

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	if seed {
		s.Seed()
	}
}

func (s *Ethereum) Seed() {
	ethutil.Config.Log.Debugln("[SERV] Retrieving seed nodes")

	// Eth-Go Bootstrapping
	ips, er := net.LookupIP("seed.bysh.me")
	if er == nil {
		peers := []string{}
		for _, ip := range ips {
			node := fmt.Sprintf("%s:%d", ip.String(), 30303)
			ethutil.Config.Log.Debugln("[SERV] Found DNS Go Peer:", node)
			peers = append(peers, node)
		}
		s.ProcessPeerList(peers)
	}

	// Official DNS Bootstrapping
	_, nodes, err := net.LookupSRV("eth", "tcp", "ethereum.org")
	if err == nil {
		peers := []string{}
		// Iterate SRV nodes
		for _, n := range nodes {
			target := n.Target
			port := strconv.Itoa(int(n.Port))
			// Resolve target to ip (Go returns list, so may resolve to multiple ips?)
			addr, err := net.LookupHost(target)
			if err == nil {
				for _, a := range addr {
					// Build string out of SRV port and Resolved IP
					peer := net.JoinHostPort(a, port)
					ethutil.Config.Log.Debugln("[SERV] Found DNS Bootstrap Peer:", peer)
					peers = append(peers, peer)
				}
			} else {
				ethutil.Config.Log.Debugln("[SERV} Couldn't resolve :", target)
			}
		}
		// Connect to Peer list
		s.ProcessPeerList(peers)
	} else {
		// Fallback to servers.poc3.txt
		resp, err := http.Get("http://www.ethereum.org/servers.poc3.txt")
		if err != nil {
			log.Println("Fetching seed failed:", err)
			return
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Println("Reading seed failed:", err)
			return
		}

		s.ConnectToPeer(string(body))
	}
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			ethutil.Config.Log.Debugln(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	close(s.quit)

	if s.RpcServer != nil {
		s.RpcServer.Stop()
	}
	s.txPool.Stop()
	s.stateManager.Stop()

	close(s.shutdownChan)
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}

func (s *Ethereum) upnpUpdateThread() {
	// Go off immediately to prevent code duplication, thereafter we renew
	// lease every 15 minutes.
	timer := time.NewTimer(5 * time.Minute)
	lport, _ := strconv.ParseInt(s.Port, 10, 16)
	first := true
out:
	for {
		select {
		case <-timer.C:
			var err error
			_, err = s.nat.AddPortMapping("TCP", int(lport), int(lport), "eth listen port", 20*60)
			if err != nil {
				ethutil.Config.Log.Debugln("can't add UPnP port mapping:", err)
				break out
			}
			if first && err == nil {
				_, err = s.nat.GetExternalAddress()
				if err != nil {
					ethutil.Config.Log.Debugln("UPnP can't get external address:", err)
					continue out
				}
				first = false
			}
			timer.Reset(time.Minute * 15)
		case <-s.quit:
			break out
		}
	}

	timer.Stop()

	if err := s.nat.DeletePortMapping("TCP", int(lport), int(lport)); err != nil {
		ethutil.Config.Log.Debugln("unable to remove UPnP port mapping:", err)
	} else {
		ethutil.Config.Log.Debugln("succesfully disestablished UPnP port mapping")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/urfave/cli"
)

type EthernetState struct {
	Enabled bool      `json:"enabled"`
	Configg *Ethernet `json:"config"`
}

func EthernetCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableEthernet(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableEthernet(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveEthernet(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configEthernetCMD(ctx)
	}
	return nil
}

// Enable ethernet enables ethernet network in the host machine. This relies on
// systemd to be the init system of the host machine.
//
// If the config flag is set, ethernet will be configured before  enabling it.
// Ommit the config flag if  ethernet is already configured using this tool.
func EnableEthernet(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configEthernetCMD(ctx)
		if err != nil {
			return err
		}
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "link", "set", "up", e.Configg.Interface).Output()
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	e.Enabled = true
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, i), data)
}

// gives the current state of the ethernet configuration. This will return an
// error if the system hast been configured yet.
//
// Configuration state files are written in $FCONF_CONFIGDIR directory.
func ethernetState(i string) (*EthernetState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultEthernetConfig, i)))
	if err != nil {
		return nil, err
	}
	e := &EthernetState{}
	err = json.Unmarshal(b, e)
	if err != nil {
		return nil, err
	}
	if e.Configg == nil {
		return nil, ErrWrongStateFile
	}
	if e.Configg.Interface == "" {
		e.Configg.Interface = "eth0"
	}
	return e, nil
}

func configEthernetCMD(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := Ethernet{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "eth0"
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	ctx.GlobalSet("interface", e.Interface)
	fmt.Printf("successful written ethernet configuration to %s \n", filename)
	state := &EthernetState{Configg: &e}
	b, _ = json.Marshal(state)
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, e.Interface), b)
}

func keepState(filename string, src []byte) error {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	err := checkDir(dir)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(dir, filename), src, 0644)
}

//DisableEthernet disables ethernet temporaly.
func DisableEthernet(ctx *cli.Context) error {
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	ctx.Set("interface", i)
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "link", "set", "down", "dev", e.Configg.Interface).Output()
	if err != nil {
		return err
	}
	fmt.Println("successfully disabled ethernet")
	//e.Enabled = false
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, i), data)
}

//RemoveEthernet removes ethernet service.
func RemoveEthernet(ctx *cli.Context) error {
	err := DisableEthernet(ctx)
	if err != nil {
		return err
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	// removestate file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultEthernetConfig, i))
	err = removeFile(stateFile)
	if err != nil {
		return err
	}
	// remove systemd file
	unit := filepath.Join(networkBase, ethernetService)
	err = removeFile(unit)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}

	// Flush settings for the interface
	err = FlushInterface(e.Configg.Interface)
	if err != nil {
		return err
	}

	// reload systemd-networkd
	return restartService("systemd-networkd")
}

func removeFile(name string) error {
	fmt.Printf("removing %s ...", name)
	err := os.Remove(name)
	if err != nil {
		fmt.Println(" error")
		return err
	}
	fmt.Println(" done without error")
	return nil
}

func stateDir() string {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	return dir
}
<commit_msg>ethernet: refactor to be like 4g<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/urfave/cli"
)

type EthernetState struct {
	Enabled bool      `json:"enabled"`
	Configg *Ethernet `json:"config"`
}

func EthernetCMD(ctx *cli.Context) error {
	if ctx.IsSet(enableFlag) {
		return EnableEthernet(ctx)
	}
	if ctx.IsSet(disableFlag) {
		return DisableEthernet(ctx)
	}
	if ctx.IsSet(removeFlag) {
		return RemoveEthernet(ctx)
	}
	if ctx.IsSet(configFlag) {
		return configEthernetCMD(ctx)
	}
	return nil
}

// Enable ethernet enables ethernet network in the host machine. This relies on
// systemd to be the init system of the host machine.
//
// If the config flag is set, ethernet will be configured before  enabling it.
// Ommit the config flag if  ethernet is already configured using this tool.
func EnableEthernet(ctx *cli.Context) error {
	if ctx.IsSet(configFlag) {
		err := configEthernetCMD(ctx)
		if err != nil {
			return err
		}
	}
	var i string
	if ctx.IsSet("interface") {
		i = ctx.String("interface")
	} else {
		i = ctx.Args().First()
	}
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(ethernetService, e.Configg.Interface))
	_, err = os.Stat(unit)
	if os.IsNotExist(err) {
		err = CreateSystemdFile(e.Configg, unit, 0644)
		if err != nil {
			return err
		}
	}
	_, err = exec.Command("ip", "link", "set", "up", e.Configg.Interface).Output()
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	e.Enabled = true
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, i), data)
}

// gives the current state of the ethernet configuration. This will return an
// error if the system hast been configured yet.
//
// Configuration state files are written in $FCONF_CONFIGDIR directory.
func ethernetState(i string) (*EthernetState, error) {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	b, err := ioutil.ReadFile(filepath.Join(dir,
		fmt.Sprintf(defaultEthernetConfig, i)))
	if err != nil {
		return nil, err
	}
	e := &EthernetState{}
	err = json.Unmarshal(b, e)
	if err != nil {
		return nil, err
	}
	if e.Configg == nil {
		return nil, ErrWrongStateFile
	}
	if e.Configg.Interface == "" {
		e.Configg.Interface = "eth0"
	}
	return e, nil
}

func configEthernetCMD(ctx *cli.Context) error {
	base := ctx.String("dir")
	name := ctx.String("name")
	src := ctx.String("config")
	if src == "" {
		return errors.New("fconf: missing configuration source file")
	}
	var b []byte
	var err error
	if src == "stdin" {
		b, err = ReadFromStdin()
		if err != nil {
			return err
		}
	} else {
		b, err = ioutil.ReadFile(src)
		if err != nil {
			return err
		}
	}
	e := Ethernet{}
	err = json.Unmarshal(b, &e)
	if err != nil {
		return err
	}
	err = checkDir(base)
	if err != nil {
		return err
	}
	if e.Interface == "" {
		e.Interface = "eth0"
	}
	if strings.Contains(name, "%s") {
		name = fmt.Sprintf(name, e.Interface)
	}
	filename := filepath.Join(base, name)
	err = CreateSystemdFile(e, filename, 0644)
	if err != nil {
		return err
	}
	ctx.GlobalSet("interface", e.Interface)
	fmt.Printf("successful written ethernet configuration to %s \n", filename)
	state := &EthernetState{Configg: &e}
	b, _ = json.Marshal(state)
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, e.Interface), b)
}

func keepState(filename string, src []byte) error {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	err := checkDir(dir)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(dir, filename), src, 0644)
}

//DisableEthernet disables ethernet temporaly.
func DisableEthernet(ctx *cli.Context) error {
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	_, err = exec.Command("ip", "addr", "flush", "dev", e.Configg.Interface).Output()
	if err != nil {
		return fmt.Errorf("ERROR: running ip addr flush dev %s %v",
			e.Configg.Interface, err,
		)
	}
	unit := filepath.Join(networkBase,
		fmt.Sprintf(ethernetService, e.Configg.Interface))
	err = removeFile(unit)
	if err != nil {
		return err
	}
	err = restartService("systemd-networkd")
	if err != nil {
		return err
	}
	fmt.Println("successfully disabled ethernet")
	//e.Enabled = false
	data, err := json.Marshal(e)
	if err != nil {
		return err
	}
	return keepState(
		fmt.Sprintf(defaultEthernetConfig, i), data)
}

//RemoveEthernet removes ethernet service.
func RemoveEthernet(ctx *cli.Context) error {
	err := DisableEthernet(ctx)
	if err != nil {
		return err
	}
	i := getInterface(ctx)
	if i == "" {
		return errors.New("missing interface, you must specify interface")
	}
	e, err := ethernetState(i)
	if err != nil {
		return err
	}
	// removestate file
	stateFile := filepath.Join(stateDir(),
		fmt.Sprintf(defaultEthernetConfig, i))
	err = removeFile(stateFile)
	if err != nil {
		return err
	}
	// remove systemd file
	unit := filepath.Join(networkBase, ethernetService)
	err = removeFile(unit)
	if err != nil {
		return err
	}
	if err != nil {
		return err
	}

	// Flush settings for the interface
	err = FlushInterface(e.Configg.Interface)
	if err != nil {
		return err
	}

	// reload systemd-networkd
	return restartService("systemd-networkd")
}

func removeFile(name string) error {
	fmt.Printf("removing %s ...", name)
	err := os.Remove(name)
	if err != nil {
		fmt.Println(" error")
		return err
	}
	fmt.Println(" done without error")
	return nil
}

func stateDir() string {
	dir := os.Getenv("FCONF_CONFIGDIR")
	if dir == "" {
		dir = fconfConfigDir
	}
	return dir
}
<|endoftext|>