<commit_before>package git_history_based_cleanup

import (
	"context"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"

	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/config"
)

type ReferenceToScan struct {
	*plumbing.Reference
	CreatedAt  time.Time
	HeadCommit *object.Commit
	referenceScanOptions
}

type referenceScanOptions struct {
	scanDepthLimit          int
	imagesCleanupKeepPolicy config.MetaCleanupKeepPolicyImagesPerReference
}

func (r *ReferenceToScan) String() string {
	imagesCleanupKeepPolicy := r.imagesCleanupKeepPolicy.String()
	if imagesCleanupKeepPolicy != "" {
		imagesCleanupKeepPolicy = fmt.Sprintf(" (%s)", imagesCleanupKeepPolicy)
	}

	return fmt.Sprintf("%s%s", r.Name().Short(), imagesCleanupKeepPolicy)
}

func ReferencesToScan(ctx context.Context, gitRepository *git.Repository, keepPolicies []*config.MetaCleanupKeepPolicy) ([]*ReferenceToScan, error) {
	rs, err := gitRepository.References()
	if err != nil {
		return nil, fmt.Errorf("get repository references failed: %s", err)
	}

	var refs []*ReferenceToScan
	if err := rs.ForEach(func(reference *plumbing.Reference) error {
		n := reference.Name()

		// Get all remote branches and tags
		if !(n.IsRemote() || n.IsTag()) {
			return nil
		}

		// Use only origin upstream
		if n.IsRemote() && !strings.HasPrefix(n.Short(), "origin/") {
			return nil
		}

		var scanDepthLimit int
		var modifiedAt time.Time
		var refCommit *object.Commit
		if !n.IsTag() {
			scanDepthLimit = -1 // unlimited

			refHash := reference.Hash()
			if refHash == plumbing.ZeroHash {
				return nil
			}

			refCommit, err = gitRepository.CommitObject(refHash)
			if err != nil {
				return fmt.Errorf("reference %s: commit object %s failed: %s", n.Short(), refHash.String(), err)
			}

			modifiedAt = refCommit.Committer.When
		} else {
			scanDepthLimit = 1

			revHash, err := gitRepository.ResolveRevision(plumbing.Revision(reference.Hash().String()))
			if err != nil {
				return fmt.Errorf("resolve revision %s failed: %s", n.Short(), err)
			}

			refCommit, err = gitRepository.CommitObject(*revHash)
			if err != nil {
				return fmt.Errorf("reference %s: commit object %s failed: %s", n.Short(), revHash.String(), err)
			}

			tagObject, err := gitRepository.TagObject(reference.Hash())
			if err == nil {
				modifiedAt = tagObject.Tagger.When
			} else if err == plumbing.ErrObjectNotFound { // lightweight tag
				modifiedAt = refCommit.Committer.When
			} else {
				return fmt.Errorf("tag object %s failed: %s", reference.Hash(), err)
			}
		}

		refs = append(refs, &ReferenceToScan{
			Reference:  reference,
			CreatedAt:  modifiedAt,
			HeadCommit: refCommit,
			referenceScanOptions: referenceScanOptions{
				scanDepthLimit: scanDepthLimit,
			},
		})

		return nil
	}); err != nil {
		return nil, err
	}

	// Split branches and tags references
	var branchesRefs, tagsRefs []*ReferenceToScan
	for _, ref := range refs {
		if ref.Name().IsTag() {
			tagsRefs = append(tagsRefs, ref)
		} else {
			branchesRefs = append(branchesRefs, ref)
		}
	}

	// Apply user or default policies
	if len(keepPolicies) == 0 {
		tagLast := 10
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				TagRegexp: regexp.MustCompile(".*"),
				Limit: &config.MetaCleanupKeepPolicyLimit{
					Last: &tagLast,
				},
			},
		})

		branchLast := 10
		branchIn := time.Hour * 24 * 7
		branchImagesPerReferenceLast := 2
		branchImagesPerReferenceIn := time.Hour * 24 * 7
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				BranchRegexp: regexp.MustCompile(".*"),
				Limit: &config.MetaCleanupKeepPolicyLimit{
					Last:     &branchLast,
					In:       &branchIn,
					Operator: &config.AndOperator,
				},
			},
			ImagesPerReference: config.MetaCleanupKeepPolicyImagesPerReference{
				MetaCleanupKeepPolicyLimit: config.MetaCleanupKeepPolicyLimit{
					Last:     &branchImagesPerReferenceLast,
					In:       &branchImagesPerReferenceIn,
					Operator: &config.AndOperator,
				},
			},
		})

		mainBranchImagesPerReferenceLast := 10
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				BranchRegexp: regexp.MustCompile("^(master|staging|production)$"),
			},
			ImagesPerReference: config.MetaCleanupKeepPolicyImagesPerReference{
				MetaCleanupKeepPolicyLimit: config.MetaCleanupKeepPolicyLimit{
					Last: &mainBranchImagesPerReferenceLast,
				},
			},
		})
	}

	var resultTagsRefs, resultBranchesRefs []*ReferenceToScan
	for _, policy := range keepPolicies {
		var policyRefs []*ReferenceToScan

		if policy.References.BranchRegexp != nil {
			policyRefs = selectBranchReferencesByRegexp(branchesRefs, policy.References.BranchRegexp)
			policyRefs = applyCleanupKeepPolicy(policyRefs, policy)
			resultBranchesRefs = mergeReferences(resultBranchesRefs, policyRefs)
		} else if policy.References.TagRegexp != nil {
			policyRefs = selectTagReferencesByRegexp(tagsRefs, policy.References.TagRegexp)
			policyRefs = applyCleanupKeepPolicy(policyRefs, policy)
			resultTagsRefs = mergeReferences(resultTagsRefs, policyRefs)
		}

		logboek.Context(ctx).Default().LogBlock(policy.String()).Do(func() {
			for _, ref := range policyRefs {
				logboek.Context(ctx).Default().LogLnDetails(ref.Name().Short())
			}
		})
	}

	// Sort by ModifiedAt
	sort.Slice(resultBranchesRefs, func(i, j int) bool {
		return resultBranchesRefs[i].CreatedAt.After(resultBranchesRefs[j].CreatedAt)
	})
	sort.Slice(resultTagsRefs, func(i, j int) bool {
		return resultTagsRefs[i].CreatedAt.After(resultTagsRefs[j].CreatedAt)
	})

	// Unite tags and branches references
	result := append(resultBranchesRefs, resultTagsRefs...)

	return result, nil
}

func selectBranchReferencesByRegexp(branchesRefs []*ReferenceToScan, regexp *regexp.Regexp) []*ReferenceToScan {
	var result []*ReferenceToScan

	for _, branchRef := range branchesRefs {
		refShortNameWithoutRemote := strings.SplitN(branchRef.Name().Short(), "/", 2)[1]
		if regexp.MatchString(refShortNameWithoutRemote) {
			result = append(result, branchRef)
		}
	}

	return result
}

func selectTagReferencesByRegexp(tagsRefs []*ReferenceToScan, regexp *regexp.Regexp) []*ReferenceToScan {
	var result []*ReferenceToScan

	for _, tagRef := range tagsRefs {
		if regexp.MatchString(tagRef.Name().Short()) {
			result = append(result, tagRef)
		}
	}

	return result
}

func applyCleanupKeepPolicy(refs []*ReferenceToScan, policy *config.MetaCleanupKeepPolicy) []*ReferenceToScan {
	refs = applyReferencesLimit(refs, policy.References.Limit)
	applyImagesPerReference(refs, policy.ImagesPerReference)

	return refs
}

func applyReferencesLimit(refs []*ReferenceToScan, limit *config.MetaCleanupKeepPolicyLimit) []*ReferenceToScan {
	if limit == nil {
		return refs
	}

	var policyInRefs []*ReferenceToScan
	if limit.In != nil {
		policyInRefs = filterReferencesByIn(refs, *limit.In)
	}

	var policyLastRefs []*ReferenceToScan
	if limit.Last != nil {
		policyLastRefs = filterReferencesByLast(refs, *limit.Last)
	}

	if limit.In == nil {
		return policyLastRefs
	} else if limit.Last == nil {
		return policyInRefs
	}

	var policyRefs []*ReferenceToScan
	if limit.Operator != nil && *limit.Operator == config.OrOperator {
		policyRefs = referencesOr(policyInRefs, policyLastRefs)
	} else {
		policyRefs = referencesAnd(policyInRefs, policyLastRefs)
	}

	return policyRefs
}

func applyImagesPerReference(policyBranchesRefs []*ReferenceToScan, imagesPerReference config.MetaCleanupKeepPolicyImagesPerReference) {
	for _, ref := range policyBranchesRefs {
		ref.imagesCleanupKeepPolicy = imagesPerReference
	}
}

func referencesOr(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	return mergeReferences(refs1, refs2)
}

func referencesAnd(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	var result []*ReferenceToScan

outerLoop:
	for _, ref1 := range refs1 {
		for _, ref2 := range refs2 {
			if ref1 == ref2 {
				result = append(result, ref1)
				continue outerLoop
			}
		}
	}

	return result
}

func filterReferencesByIn(refs []*ReferenceToScan, in time.Duration) (result []*ReferenceToScan) {
	for _, ref := range refs {
		if ref.CreatedAt.After(time.Now().Add(-in)) {
			result = append(result, ref)
		}
	}

	return
}

func filterReferencesByLast(refs []*ReferenceToScan, last int) []*ReferenceToScan {
	if last == -1 {
		return refs
	}

	sort.Slice(refs, func(i, j int) bool {
		return refs[i].CreatedAt.After(refs[j].CreatedAt)
	})

	if len(refs) < last {
		return refs
	}

	return refs[:last]
}

func mergeReferences(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	result := refs2[:]

outerLoop:
	for _, ref1 := range refs1 {
		for _, ref2 := range refs2 {
			if ref1 == ref2 {
				continue outerLoop
			}
		}

		result = append(result, ref1)
	}

	return result
}
<commit_msg>fix(cleanup): nested git tag cannot be processed<commit_after>package git_history_based_cleanup

import (
	"context"
	"fmt"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"

	"github.com/werf/logboek"

	"github.com/werf/werf/pkg/config"
)

type ReferenceToScan struct {
	*plumbing.Reference
	CreatedAt  time.Time
	HeadCommit *object.Commit
	referenceScanOptions
}

type referenceScanOptions struct {
	scanDepthLimit          int
	imagesCleanupKeepPolicy config.MetaCleanupKeepPolicyImagesPerReference
}

func (r *ReferenceToScan) String() string {
	imagesCleanupKeepPolicy := r.imagesCleanupKeepPolicy.String()
	if imagesCleanupKeepPolicy != "" {
		imagesCleanupKeepPolicy = fmt.Sprintf(" (%s)", imagesCleanupKeepPolicy)
	}

	return fmt.Sprintf("%s%s", r.Name().Short(), imagesCleanupKeepPolicy)
}

func ReferencesToScan(ctx context.Context, gitRepository *git.Repository, keepPolicies []*config.MetaCleanupKeepPolicy) ([]*ReferenceToScan, error) {
	rs, err := gitRepository.References()
	if err != nil {
		return nil, fmt.Errorf("get repository references failed: %s", err)
	}

	var refs []*ReferenceToScan
	if err := rs.ForEach(func(reference *plumbing.Reference) error {
		n := reference.Name()

		// Get all remote branches and tags
		if !(n.IsRemote() || n.IsTag()) {
			return nil
		}

		// Use only origin upstream
		if n.IsRemote() && !strings.HasPrefix(n.Short(), "origin/") {
			return nil
		}

		var scanDepthLimit int
		var modifiedAt time.Time
		var refCommit *object.Commit
		if !n.IsTag() {
			scanDepthLimit = -1 // unlimited

			refHash := reference.Hash()
			if refHash == plumbing.ZeroHash {
				return nil
			}

			refCommit, err = gitRepository.CommitObject(refHash)
			if err != nil {
				return fmt.Errorf("reference %s: commit object %s failed: %s", n.Short(), refHash.String(), err)
			}

			modifiedAt = refCommit.Committer.When
		} else {
			scanDepthLimit = 1

			refHash, err := getCommitHashForReference(gitRepository, reference.Name().String())
			if err != nil {
				return fmt.Errorf("unable to get commit hash for reference %q: %s", reference.Name(), err)
			}

			refCommit, err = gitRepository.CommitObject(refHash)
			if err != nil {
				return fmt.Errorf("reference %s: commit object %s failed: %s", n.Short(), refHash.String(), err)
			}

			tagObject, err := gitRepository.TagObject(reference.Hash())
			if err == nil {
				modifiedAt = tagObject.Tagger.When
			} else if err == plumbing.ErrObjectNotFound { // lightweight tag
				modifiedAt = refCommit.Committer.When
			} else {
				return fmt.Errorf("tag object %s failed: %s", reference.Hash(), err)
			}
		}

		refs = append(refs, &ReferenceToScan{
			Reference:  reference,
			CreatedAt:  modifiedAt,
			HeadCommit: refCommit,
			referenceScanOptions: referenceScanOptions{
				scanDepthLimit: scanDepthLimit,
			},
		})

		return nil
	}); err != nil {
		return nil, err
	}

	// Split branches and tags references
	var branchesRefs, tagsRefs []*ReferenceToScan
	for _, ref := range refs {
		if ref.Name().IsTag() {
			tagsRefs = append(tagsRefs, ref)
		} else {
			branchesRefs = append(branchesRefs, ref)
		}
	}

	// Apply user or default policies
	if len(keepPolicies) == 0 {
		tagLast := 10
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				TagRegexp: regexp.MustCompile(".*"),
				Limit: &config.MetaCleanupKeepPolicyLimit{
					Last: &tagLast,
				},
			},
		})

		branchLast := 10
		branchIn := time.Hour * 24 * 7
		branchImagesPerReferenceLast := 2
		branchImagesPerReferenceIn := time.Hour * 24 * 7
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				BranchRegexp: regexp.MustCompile(".*"),
				Limit: &config.MetaCleanupKeepPolicyLimit{
					Last:     &branchLast,
					In:       &branchIn,
					Operator: &config.AndOperator,
				},
			},
			ImagesPerReference: config.MetaCleanupKeepPolicyImagesPerReference{
				MetaCleanupKeepPolicyLimit: config.MetaCleanupKeepPolicyLimit{
					Last:     &branchImagesPerReferenceLast,
					In:       &branchImagesPerReferenceIn,
					Operator: &config.AndOperator,
				},
			},
		})

		mainBranchImagesPerReferenceLast := 10
		keepPolicies = append(keepPolicies, &config.MetaCleanupKeepPolicy{
			References: config.MetaCleanupKeepPolicyReferences{
				BranchRegexp: regexp.MustCompile("^(master|staging|production)$"),
			},
			ImagesPerReference: config.MetaCleanupKeepPolicyImagesPerReference{
				MetaCleanupKeepPolicyLimit: config.MetaCleanupKeepPolicyLimit{
					Last: &mainBranchImagesPerReferenceLast,
				},
			},
		})
	}

	var resultTagsRefs, resultBranchesRefs []*ReferenceToScan
	for _, policy := range keepPolicies {
		var policyRefs []*ReferenceToScan

		if policy.References.BranchRegexp != nil {
			policyRefs = selectBranchReferencesByRegexp(branchesRefs, policy.References.BranchRegexp)
			policyRefs = applyCleanupKeepPolicy(policyRefs, policy)
			resultBranchesRefs = mergeReferences(resultBranchesRefs, policyRefs)
		} else if policy.References.TagRegexp != nil {
			policyRefs = selectTagReferencesByRegexp(tagsRefs, policy.References.TagRegexp)
			policyRefs = applyCleanupKeepPolicy(policyRefs, policy)
			resultTagsRefs = mergeReferences(resultTagsRefs, policyRefs)
		}

		logboek.Context(ctx).Default().LogBlock(policy.String()).Do(func() {
			for _, ref := range policyRefs {
				logboek.Context(ctx).Default().LogLnDetails(ref.Name().Short())
			}
		})
	}

	// Sort by ModifiedAt
	sort.Slice(resultBranchesRefs, func(i, j int) bool {
		return resultBranchesRefs[i].CreatedAt.After(resultBranchesRefs[j].CreatedAt)
	})
	sort.Slice(resultTagsRefs, func(i, j int) bool {
		return resultTagsRefs[i].CreatedAt.After(resultTagsRefs[j].CreatedAt)
	})

	// Unite tags and branches references
	result := append(resultBranchesRefs, resultTagsRefs...)

	return result, nil
}

func getCommitHashForReference(gitRepository *git.Repository, reference string) (plumbing.Hash, error) {
	ref, err := gitRepository.Reference(plumbing.ReferenceName(reference), true)
	if err != nil {
		return plumbing.ZeroHash, err
	}

	obj, err := gitRepository.Object(plumbing.AnyObject, ref.Hash())
	if err != nil {
		return plumbing.ZeroHash, err
	}

	for {
		switch objType := obj.(type) {
		case *object.Tag:
			if objType.TargetType != plumbing.CommitObject {
				obj, err = objType.Object()
				if err != nil {
					return plumbing.ZeroHash, err
				}

				continue
			}

			return objType.Target, nil
		case *object.Commit:
			return objType.Hash, nil
		default:
			return plumbing.ZeroHash, fmt.Errorf("unsupported tag target %q", objType.Type())
		}
	}
}

func selectBranchReferencesByRegexp(branchesRefs []*ReferenceToScan, regexp *regexp.Regexp) []*ReferenceToScan {
	var result []*ReferenceToScan

	for _, branchRef := range branchesRefs {
		refShortNameWithoutRemote := strings.SplitN(branchRef.Name().Short(), "/", 2)[1]
		if regexp.MatchString(refShortNameWithoutRemote) {
			result = append(result, branchRef)
		}
	}

	return result
}

func selectTagReferencesByRegexp(tagsRefs []*ReferenceToScan, regexp *regexp.Regexp) []*ReferenceToScan {
	var result []*ReferenceToScan

	for _, tagRef := range tagsRefs {
		if regexp.MatchString(tagRef.Name().Short()) {
			result = append(result, tagRef)
		}
	}

	return result
}

func applyCleanupKeepPolicy(refs []*ReferenceToScan, policy *config.MetaCleanupKeepPolicy) []*ReferenceToScan {
	refs = applyReferencesLimit(refs, policy.References.Limit)
	applyImagesPerReference(refs, policy.ImagesPerReference)

	return refs
}

func applyReferencesLimit(refs []*ReferenceToScan, limit *config.MetaCleanupKeepPolicyLimit) []*ReferenceToScan {
	if limit == nil {
		return refs
	}

	var policyInRefs []*ReferenceToScan
	if limit.In != nil {
		policyInRefs = filterReferencesByIn(refs, *limit.In)
	}

	var policyLastRefs []*ReferenceToScan
	if limit.Last != nil {
		policyLastRefs = filterReferencesByLast(refs, *limit.Last)
	}

	if limit.In == nil {
		return policyLastRefs
	} else if limit.Last == nil {
		return policyInRefs
	}

	var policyRefs []*ReferenceToScan
	if limit.Operator != nil && *limit.Operator == config.OrOperator {
		policyRefs = referencesOr(policyInRefs, policyLastRefs)
	} else {
		policyRefs = referencesAnd(policyInRefs, policyLastRefs)
	}

	return policyRefs
}

func applyImagesPerReference(policyBranchesRefs []*ReferenceToScan, imagesPerReference config.MetaCleanupKeepPolicyImagesPerReference) {
	for _, ref := range policyBranchesRefs {
		ref.imagesCleanupKeepPolicy = imagesPerReference
	}
}

func referencesOr(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	return mergeReferences(refs1, refs2)
}

func referencesAnd(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	var result []*ReferenceToScan

outerLoop:
	for _, ref1 := range refs1 {
		for _, ref2 := range refs2 {
			if ref1 == ref2 {
				result = append(result, ref1)
				continue outerLoop
			}
		}
	}

	return result
}

func filterReferencesByIn(refs []*ReferenceToScan, in time.Duration) (result []*ReferenceToScan) {
	for _, ref := range refs {
		if ref.CreatedAt.After(time.Now().Add(-in)) {
			result = append(result, ref)
		}
	}

	return
}

func filterReferencesByLast(refs []*ReferenceToScan, last int) []*ReferenceToScan {
	if last == -1 {
		return refs
	}

	sort.Slice(refs, func(i, j int) bool {
		return refs[i].CreatedAt.After(refs[j].CreatedAt)
	})

	if len(refs) < last {
		return refs
	}

	return refs[:last]
}

func mergeReferences(refs1 []*ReferenceToScan, refs2 []*ReferenceToScan) []*ReferenceToScan {
	result := refs2[:]

outerLoop:
	for _, ref1 := range refs1 {
		for _, ref2 := range refs2 {
			if ref1 == ref2 {
				continue outerLoop
			}
		}

		result = append(result, ref1)
	}

	return result
}
<|endoftext|>


<commit_before>package chart_extender

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/werf/werf/pkg/deploy/helm/command_helpers"

	uuid "github.com/satori/go.uuid"
	"github.com/werf/lockgate"

	"helm.sh/helm/v3/pkg/cli"

	"github.com/pkg/errors"
	"github.com/werf/werf/pkg/werf"
	"helm.sh/helm/v3/pkg/chart"
	"sigs.k8s.io/yaml"

	"github.com/werf/logboek"

	"helm.sh/helm/v3/pkg/chart/loader"
)

func GetChartDependenciesCacheDir(lockDigest string) string {
	return filepath.Join(werf.GetLocalCacheDir(), "helm_chart_dependencies", lockDigest)
}

func LoadMetadata(files []*chart.ChartExtenderBufferedFile) (*chart.Metadata, error) {
	var metadata *chart.Metadata

	for _, f := range files {
		if f.Name == "Chart.yaml" {
			metadata = new(chart.Metadata)

			if err := yaml.Unmarshal(f.Data, metadata); err != nil {
				return nil, errors.Wrap(err, "cannot load Chart.yaml")
			}

			if metadata.APIVersion == "" {
				metadata.APIVersion = chart.APIVersionV1
			}

			break
		}
	}

	for _, f := range files {
		if f.Name == "requirements.yaml" {
			if err := yaml.Unmarshal(f.Data, metadata); err != nil {
				return nil, errors.Wrap(err, "cannot load requirements.yaml")
			}
			break
		}
	}

	return metadata, nil
}

func LoadLock(files []*chart.ChartExtenderBufferedFile) (*chart.Lock, *chart.ChartExtenderBufferedFile, error) {
	var lock *chart.Lock
	var lockFile *chart.ChartExtenderBufferedFile

loop:
	for _, f := range files {
		switch {
		case f.Name == "Chart.lock":
			lock = new(chart.Lock)
			if err := yaml.Unmarshal(f.Data, &lock); err != nil {
				return nil, nil, errors.Wrap(err, "cannot load Chart.lock")
			}
			lockFile = f
			break loop
		case f.Name == "requirements.lock":
			lock = new(chart.Lock)
			if err := yaml.Unmarshal(f.Data, &lock); err != nil {
				return nil, nil, errors.Wrap(err, "cannot load requirements.lock")
			}
			lockFile = f
			break loop
		}
	}

	return lock, lockFile, nil
}

func GetPreparedChartDependenciesDir(ctx context.Context, lockDigest string, lockFileData []byte, chartFileData []byte, helmEnvSettings *cli.EnvSettings, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) (string, error) {
	depsDir := GetChartDependenciesCacheDir(lockDigest)

	if _, err := os.Stat(depsDir); os.IsNotExist(err) {
		if err := logboek.Context(ctx).Default().LogProcess("Building chart dependencies").DoError(func() error {
			logboek.Context(ctx).Default().LogF("Using chart dependencies directory: %s\n", depsDir)
			if _, lock, err := werf.AcquireHostLock(ctx, depsDir, lockgate.AcquireOptions{}); err != nil {
				return fmt.Errorf("error acquiring lock for %q: %s", depsDir, err)
			} else {
				defer werf.ReleaseHostLock(lock)
			}

			tmpDepsDir := fmt.Sprintf("%s.tmp.%s", depsDir, uuid.NewV4().String())

			buildChartDependenciesOpts.LoadOptions = &loader.LoadOptions{
				ChartExtender:               NewWerfChartStub(ctx),
				SubchartExtenderFactoryFunc: nil,
			}

			if err := command_helpers.BuildChartDependenciesInDir(ctx, lockFileData, chartFileData, tmpDepsDir, helmEnvSettings, buildChartDependenciesOpts); err != nil {
				return fmt.Errorf("error building chart dependencies: %s", err)
			}

			if err := os.Rename(tmpDepsDir, depsDir); err != nil {
				return fmt.Errorf("error renaming %q to %q: %s", tmpDepsDir, depsDir, err)
			}

			return nil
		}); err != nil {
			return "", err
		}
	} else if err != nil {
		return "", fmt.Errorf("error accessing %q: %s", depsDir, err)
	} else {
		logboek.Context(ctx).Default().LogF("Using cached chart dependencies directory: %s\n", depsDir)
	}

	return depsDir, nil
}

func LoadChartDependencies(ctx context.Context, loadedFiles []*chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) ([]*chart.ChartExtenderBufferedFile, error) {
	var chartFile *chart.ChartExtenderBufferedFile
	for _, f := range loadedFiles {
		if f.Name == "Chart.yaml" {
			chartFile = f
			break
		}
	}

	res := loadedFiles

	if chartFile != nil {
		if lock, lockFile, err := LoadLock(loadedFiles); err != nil {
			return nil, fmt.Errorf("error loading chart lock file: %s", err)
		} else if lock == nil {
			if metadata, err := LoadMetadata(loadedFiles); err != nil {
				return nil, fmt.Errorf("error loading chart metadata file: %s", err)
			} else if len(metadata.Dependencies) > 0 {
				logboek.Context(ctx).Error().LogLn("Cannot build chart dependencies and preload charts without lock file (.helm/Chart.lock or .helm/requirements.lock)")
				logboek.Context(ctx).Error().LogLn("It is recommended to add Chart.lock file to your project repository or remove chart dependencies.")
				logboek.Context(ctx).Error().LogLn()
				logboek.Context(ctx).Error().LogLn("To generate a lock file run 'werf helm dependency update .helm' and commit resulting .helm/Chart.lock (it is not required to commit whole .helm/charts directory).")
				logboek.Context(ctx).Error().LogLn()
			}
		} else {
			if depsDir, err := GetPreparedChartDependenciesDir(ctx, lock.Digest, lockFile.Data, chartFile.Data, helmEnvSettings, buildChartDependenciesOpts); err != nil {
				return nil, fmt.Errorf("")
			} else {
				localFiles, err := loader.GetFilesFromLocalFilesystem(depsDir)
				if err != nil {
					return nil, err
				}

				for _, f := range localFiles {
					if strings.HasPrefix(f.Name, "charts/") {
						f1 := new(chart.ChartExtenderBufferedFile)
						*f1 = chart.ChartExtenderBufferedFile(*f)
						res = append(res, f1)
						logboek.Context(ctx).Debug().LogF("-- GiterministicFilesLoader: loading subchart %q from dependencies dir %s\n", f.Name, depsDir)
					}
				}
			}
		}
	}

	return res, nil
}
<commit_msg>[helm][giterminism] Cache chart dependencies by hashsum of the whole .helm/Chart.lock, not digest<commit_after>package chart_extender

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/werf/werf/pkg/util"

	"github.com/werf/werf/pkg/deploy/helm/command_helpers"

	uuid "github.com/satori/go.uuid"
	"github.com/werf/lockgate"

	"helm.sh/helm/v3/pkg/cli"

	"github.com/pkg/errors"
	"github.com/werf/werf/pkg/werf"
	"helm.sh/helm/v3/pkg/chart"
	"sigs.k8s.io/yaml"

	"github.com/werf/logboek"

	"helm.sh/helm/v3/pkg/chart/loader"
)

func GetChartDependenciesCacheDir(lockChecksum string) string {
	return filepath.Join(werf.GetLocalCacheDir(), "helm_chart_dependencies", "1", lockChecksum)
}

func LoadMetadata(files []*chart.ChartExtenderBufferedFile) (*chart.Metadata, error) {
	var metadata *chart.Metadata

	for _, f := range files {
		if f.Name == "Chart.yaml" {
			metadata = new(chart.Metadata)

			if err := yaml.Unmarshal(f.Data, metadata); err != nil {
				return nil, errors.Wrap(err, "cannot load Chart.yaml")
			}

			if metadata.APIVersion == "" {
				metadata.APIVersion = chart.APIVersionV1
			}

			break
		}
	}

	for _, f := range files {
		if f.Name == "requirements.yaml" {
			if err := yaml.Unmarshal(f.Data, metadata); err != nil {
				return nil, errors.Wrap(err, "cannot load requirements.yaml")
			}
			break
		}
	}

	return metadata, nil
}

func LoadLock(files []*chart.ChartExtenderBufferedFile) (*chart.Lock, *chart.ChartExtenderBufferedFile, error) {
	var lock *chart.Lock
	var lockFile *chart.ChartExtenderBufferedFile

loop:
	for _, f := range files {
		switch {
		case f.Name == "Chart.lock":
			lock = new(chart.Lock)
			if err := yaml.Unmarshal(f.Data, &lock); err != nil {
				return nil, nil, errors.Wrap(err, "cannot load Chart.lock")
			}
			lockFile = f
			break loop
		case f.Name == "requirements.lock":
			lock = new(chart.Lock)
			if err := yaml.Unmarshal(f.Data, &lock); err != nil {
				return nil, nil, errors.Wrap(err, "cannot load requirements.lock")
			}
			lockFile = f
			break loop
		}
	}

	return lock, lockFile, nil
}

func GetPreparedChartDependenciesDir(ctx context.Context, lockDigest string, lockFileData []byte, chartFileData []byte, helmEnvSettings *cli.EnvSettings, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) (string, error) {
	depsDir := GetChartDependenciesCacheDir(util.Sha256Hash(string(lockFileData)))

	if _, err := os.Stat(depsDir); os.IsNotExist(err) {
		if err := logboek.Context(ctx).Default().LogProcess("Building chart dependencies").DoError(func() error {
			logboek.Context(ctx).Default().LogF("Using chart dependencies directory: %s\n", depsDir)
			if _, lock, err := werf.AcquireHostLock(ctx, depsDir, lockgate.AcquireOptions{}); err != nil {
				return fmt.Errorf("error acquiring lock for %q: %s", depsDir, err)
			} else {
				defer werf.ReleaseHostLock(lock)
			}

			tmpDepsDir := fmt.Sprintf("%s.tmp.%s", depsDir, uuid.NewV4().String())

			buildChartDependenciesOpts.LoadOptions = &loader.LoadOptions{
				ChartExtender:               NewWerfChartStub(ctx),
				SubchartExtenderFactoryFunc: nil,
			}

			if err := command_helpers.BuildChartDependenciesInDir(ctx, lockFileData, chartFileData, tmpDepsDir, helmEnvSettings, buildChartDependenciesOpts); err != nil {
				return fmt.Errorf("error building chart dependencies: %s", err)
			}

			if err := os.Rename(tmpDepsDir, depsDir); err != nil {
				return fmt.Errorf("error renaming %q to %q: %s", tmpDepsDir, depsDir, err)
			}

			return nil
		}); err != nil {
			return "", err
		}
	} else if err != nil {
		return "", fmt.Errorf("error accessing %q: %s", depsDir, err)
	} else {
		logboek.Context(ctx).Default().LogF("Using cached chart dependencies directory: %s\n", depsDir)
	}

	return depsDir, nil
}

func LoadChartDependencies(ctx context.Context, loadedFiles []*chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) ([]*chart.ChartExtenderBufferedFile, error) {
	var chartFile *chart.ChartExtenderBufferedFile
	for _, f := range loadedFiles {
		if f.Name == "Chart.yaml" {
			chartFile = f
			break
		}
	}

	res := loadedFiles

	if chartFile != nil {
		if lock, lockFile, err := LoadLock(loadedFiles); err != nil {
			return nil, fmt.Errorf("error loading chart lock file: %s", err)
		} else if lock == nil {
			if metadata, err := LoadMetadata(loadedFiles); err != nil {
				return nil, fmt.Errorf("error loading chart metadata file: %s", err)
			} else if len(metadata.Dependencies) > 0 {
				logboek.Context(ctx).Error().LogLn("Cannot build chart dependencies and preload charts without lock file (.helm/Chart.lock or .helm/requirements.lock)")
				logboek.Context(ctx).Error().LogLn("It is recommended to add Chart.lock file to your project repository or remove chart dependencies.")
				logboek.Context(ctx).Error().LogLn()
				logboek.Context(ctx).Error().LogLn("To generate a lock file run 'werf helm dependency update .helm' and commit resulting .helm/Chart.lock (it is not required to commit whole .helm/charts directory).")
				logboek.Context(ctx).Error().LogLn()
			}
		} else {
			if depsDir, err := GetPreparedChartDependenciesDir(ctx, lock.Digest, lockFile.Data, chartFile.Data, helmEnvSettings, buildChartDependenciesOpts); err != nil {
				return nil, fmt.Errorf("")
			} else {
				localFiles, err := loader.GetFilesFromLocalFilesystem(depsDir)
				if err != nil {
					return nil, err
				}

				for _, f := range localFiles {
					if strings.HasPrefix(f.Name, "charts/") {
						f1 := new(chart.ChartExtenderBufferedFile)
						*f1 = chart.ChartExtenderBufferedFile(*f)
						res = append(res, f1)
						logboek.Context(ctx).Debug().LogF("-- GiterministicFilesLoader: loading subchart %q from dependencies dir %s\n", f.Name, depsDir)
					}
				}
			}
		}
	}

	return res, nil
}
<|endoftext|>


<commit_before>/*
    norx.go
    ------

    This file is part of the Go reference implementation of NORX.

    :copyright: (c) 2014 Philipp Jovanovic <philipp@jovanovic.io>
    :license: BSD (3-Clause), see LICENSE
*/

package aead

const (
    NORX_W = 64              // wordsize
    NORX_R = 4               // number of rounds
    NORX_D = 1               // parallelism degree
    NORX_A = NORX_W * 4      // tag size
    NORX_N = NORX_W * 2      // nonce size
    NORX_K = NORX_W * 4      // key size
    NORX_B = NORX_W * 16     // state size
    NORX_C = NORX_W * 6      // capacity size
    RATE = NORX_B - NORX_C   // rate size
    HEADER_TAG  = 1 << 0
    PAYLOAD_TAG = 1 << 1
    TRAILER_TAG = 1 << 2
    FINAL_TAG   = 1 << 3
    BRANCH_TAG  = 1 << 4
    MERGE_TAG   = 1 << 5
    R0, R1, R2, R3 = 8, 19, 40, 63 // rotation offsets
    u0, u1 = 0x243F6A8885A308D3, 0x13198A2E03707344
    u2, u3 = 0xA4093822299F31D0, 0x082EFA98EC4E6C89
    u4, u5 = 0xAE8858DC339325A1, 0x670A134EE52D7FA6
    u6, u7 = 0xC4316D80CD967541, 0xD21DFBF8B630B762
    u8, u9 = 0x375A18D261E7F892, 0x343D1F187D92285B
)

type state_t struct {
    s [16]uint64
}

func ROTR(x,c uint64) uint64 {
    return (x >> c | x << (NORX_W - c))
}

func H(x,y uint64) uint64 {
    return (x ^ y) ^ ((x & y) << 1)
}

func G(a,b,c,d uint64) (uint64,uint64,uint64,uint64) {

    a = H(a,b)
    d = ROTR(a ^ d, R0)
    c = H(c,d)
    b = ROTR(b ^ c, R1)
    a = H(a,b)
    d = ROTR(a ^ d, R2)
    c = H(c,d)
    b = ROTR(b ^ c, R3)
    return a,b,c,d
}

func F(s []uint64) {
    // Column step
    s[ 0], s[ 4], s[ 8], s[12] = G(s[ 0], s[ 4], s[ 8], s[12])
    s[ 1], s[ 5], s[ 9], s[13] = G(s[ 1], s[ 5], s[ 9], s[13])
    s[ 2], s[ 6], s[10], s[14] = G(s[ 2], s[ 6], s[10], s[14])
    s[ 3], s[ 7], s[11], s[15] = G(s[ 3], s[ 7], s[11], s[15])
    // Diagonal step
    s[ 0], s[ 5], s[10], s[15] = G(s[ 0], s[ 5], s[10], s[15])
    s[ 1], s[ 6], s[11], s[12] = G(s[ 1], s[ 6], s[11], s[12])
    s[ 2], s[ 7], s[ 8], s[13] = G(s[ 2], s[ 7], s[ 8], s[13])
    s[ 3], s[ 4], s[ 9], s[14] = G(s[ 3], s[ 4], s[ 9], s[14])
}

func permute(state *state_t) {
    s := state.s[:]
    for i:=0; i<NORX_R; i++ { F(s) }
}

func setup(state *state_t, k []uint8, n []uint8) {

    s := state.s[:]

    s[ 0] = u0
    s[ 1] = LOAD64(n[ 0: 8])
    s[ 2] = LOAD64(n[ 8:16])
    s[ 3] = u1

    s[ 4] = LOAD64(k[ 0: 8])
    s[ 5] = LOAD64(k[ 8:16])
    s[ 6] = LOAD64(k[16:24])
    s[ 7] = LOAD64(k[24:32])

    s[ 8] = u2
    s[ 9] = u3
    s[10] = u4
    s[11] = u5

    s[12] = u6
    s[13] = u7
    s[14] = u8
    s[15] = u9

    s[14] ^= (NORX_R << 26) | (NORX_D << 18) | (NORX_W << 10) | NORX_A
    permute(state)
}

func process_header(state *state_t, in []uint8, inlen uint64) {

    if inlen > 0 {
        lastblock := make([]uint8, BYTES64(RATE))
        i := uint64(0)
        n := BYTES64(RATE)
        for inlen >= n {
            absorb_block( state, in[n*i:n*(i+1)], HEADER_TAG )
            inlen -= n
            i++
        }
        pad(lastblock[:], in[n*i:], inlen)
        absorb_block(state, lastblock[:], HEADER_TAG)
        BURN8(lastblock[:], BYTES64(RATE))
    }
}

func encrypt_msg(state *state_t, out []uint8, in []uint8, inlen uint64) {

    if inlen > 0 {
        lastblock := make([]uint8, BYTES64(RATE))
        n := BYTES64(RATE)
        i := uint64(0)
        for inlen >= n {
            encrypt_block(state, out[n*i:n*(i+1)], in[n*i:n*(i+1)])
            inlen -= n
            i++
        }
        pad(lastblock[:], in[n*i:], inlen)
        encrypt_block(state, lastblock[:], lastblock[:])
        copy(out[n*i:n*i+inlen], lastblock[:])
        BURN8(lastblock[:], BYTES64(RATE))
    }
}

func decrypt_msg(state *state_t, out []uint8, in []uint8, inlen uint64) {

    if inlen > 0 {
        n := BYTES64(RATE)
        i := uint64(0)
        for inlen >= n {
            decrypt_block(state, out[n*i:n*(i+1)], in[n*i:n*(i+1)])
            inlen -= n
            i++
        }
        decrypt_lastblock(state, out[n*i:n*i+inlen], in[n*i:n*i+inlen], inlen)
    }
}

func process_trailer(state *state_t, in []uint8, inlen uint64) {

    if inlen > 0 {
        lastblock := make([]uint8, BYTES64(RATE))
        i := uint64(0)
        n := BYTES64(RATE)
        for inlen >= n {
            absorb_block(state, in[n*i:n*(i+1)], TRAILER_TAG)
            inlen -= n
            i++
        }
        pad(lastblock[:], in[n*i:], inlen)
        absorb_block(state, lastblock[:], HEADER_TAG)
        BURN8(lastblock[:], BYTES64(RATE))
    }
}

func output_tag(state *state_t, tag []uint8) {

    inject_tag(state, FINAL_TAG)
    permute(state)
    permute(state)

    s := state.s[:]
    lastblock := make([]uint8, BYTES64(RATE))
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        STORE64(lastblock[b*i:b*(i+1)], s[i])
    }
    copy(tag[:], lastblock[:])
    BURN8(lastblock[:], BYTES64(RATE))
}

func verify_tag(tag1 []uint8, tag2 []uint8) int {

    acc := 0
    for i := uint64(0); i < BYTES64(NORX_A); i++ {
        acc |= int(tag1[i] ^ tag2[i])
    }
    return (((acc - 1) >> 8) & 1) - 1
}

func pad(out []uint8, in []uint8, inlen uint64) {

    copy(out[:],in[:inlen])
    out[inlen] = 0x01
    out[BYTES64(RATE) - 1] |= 0x80
}

func inject_tag(state *state_t, tag uint64) {
    s := state.s[:]
    s[15] ^= tag
}

func absorb_block(state *state_t, in []uint8, tag uint64) {

    inject_tag(state, tag)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        s[i] ^= LOAD64(in[b*i:b*(i+1)])
    }
}

func encrypt_block(state *state_t, out []uint8, in []uint8) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        s[i] ^= LOAD64(in[b*i:b*(i+1)])
        STORE64(out[b*i:b*(i+1)], s[i])
    }
}

func decrypt_block(state *state_t, out []uint8, in []uint8) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        c := LOAD64(in[b*i:b*(i+1)])
        STORE64(out[b*i:b*(i+1)], s[i] ^ c)
        s[i] = c
    }
}

func decrypt_lastblock(state *state_t, out []uint8, in []uint8, inlen uint64) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    i := uint64(0)
    s := state.s[:]
    n := BYTES64(NORX_W)
    b := make([]uint8, n)

    /* undo padding */
    s[inlen / n] ^= uint64(0x01) << (inlen % n * 8)
    s[WORDS64(RATE) - 1] ^= uint64(0x80) << ((BYTES64(RATE) - 1) % n * 8)

    for inlen >= n {
        c := LOAD64(in[n*i:n*(i+1)])
        STORE64(out[n*i:n*(i+1)], s[i] ^ c)
        s[i] = c
        inlen -= n
        i++
    }

    STORE64(b,s[i])
    for j := uint64(0); j < inlen; j++ {
      c := in[n*i+j]
      out[n*i+j] = b[j] ^ c
      b[j] = c
    }
    s[i] = LOAD64(b)
    BURN8(b,n)
}

func AEAD_encrypt(
    c []uint8, clen *uint64,
    h []uint8, hlen uint64,
    m []uint8, mlen uint64,
    t []uint8, tlen uint64,
    nonce []uint8,
    key []uint8) {

    state := new(state_t)
    setup(state, key, nonce)
    process_header(state, h, hlen)
    encrypt_msg(state, c, m, mlen)
    process_trailer(state, t, tlen)
    output_tag(state, c[ mlen: ])
    *clen = mlen + BYTES64(NORX_A)
    BURN64(state.s[:], WORDS64(NORX_B))
}

func AEAD_decrypt(
    m []uint8, mlen *uint64,
    h []uint8, hlen uint64,
    c []uint8, clen uint64,
    t []uint8, tlen uint64,
    nonce []uint8,
    key []uint8) int {

    if clen < BYTES64(NORX_A) {
      return -1
    }
    result := -1
    tag := make([]uint8, NORX_A)
    state := new(state_t)
    setup(state, key, nonce)
    process_header(state, h, hlen)
    decrypt_msg(state, m, c, clen - BYTES64(NORX_A))
    process_trailer(state, t, tlen)
    output_tag(state, tag)
    *mlen = clen - BYTES64(NORX_A)
    result = verify_tag(c[clen - BYTES64(NORX_A):], tag)
    if result != 0 {
        BURN8(m, clen - BYTES64(NORX_A))
    }
    BURN64(state.s[:], WORDS64(NORX_B))
    return result
}

<commit_msg>bugfixes and cleanup<commit_after>/*
    norx.go
    ------

    This file is part of the Go reference implementation of NORX.

    :copyright: (c) 2014 Philipp Jovanovic <philipp@jovanovic.io>
    :license: BSD (3-Clause), see LICENSE
*/

package aead

const (
    NORX_W = 64              // wordsize
    NORX_R = 4               // number of rounds
    NORX_D = 1               // parallelism degree
    NORX_A = NORX_W * 4      // tag size
    NORX_N = NORX_W * 2      // nonce size
    NORX_K = NORX_W * 4      // key size
    NORX_B = NORX_W * 16     // state size
    NORX_C = NORX_W * 6      // capacity size
    RATE = NORX_B - NORX_C   // rate size
    HEADER_TAG  = 1 << 0
    PAYLOAD_TAG = 1 << 1
    TRAILER_TAG = 1 << 2
    FINAL_TAG   = 1 << 3
    BRANCH_TAG  = 1 << 4
    MERGE_TAG   = 1 << 5
    R0, R1, R2, R3 = 8, 19, 40, 63 // rotation offsets
    u0, u1 = 0x243F6A8885A308D3, 0x13198A2E03707344
    u2, u3 = 0xA4093822299F31D0, 0x082EFA98EC4E6C89
    u4, u5 = 0xAE8858DC339325A1, 0x670A134EE52D7FA6
    u6, u7 = 0xC4316D80CD967541, 0xD21DFBF8B630B762
    u8, u9 = 0x375A18D261E7F892, 0x343D1F187D92285B
)

type state_t struct {
    s [16]uint64
}

func ROTR(x,c uint64) uint64 {
    return (x >> c | x << (NORX_W - c))
}

func H(x,y uint64) uint64 {
    return (x ^ y) ^ ((x & y) << 1)
}

func G(a,b,c,d uint64) (uint64,uint64,uint64,uint64) {

    a = H(a,b)
    d = ROTR(a ^ d, R0)
    c = H(c,d)
    b = ROTR(b ^ c, R1)
    a = H(a,b)
    d = ROTR(a ^ d, R2)
    c = H(c,d)
    b = ROTR(b ^ c, R3)
    return a,b,c,d
}

func F(s []uint64) {
    // Column step
    s[ 0], s[ 4], s[ 8], s[12] = G(s[ 0], s[ 4], s[ 8], s[12])
    s[ 1], s[ 5], s[ 9], s[13] = G(s[ 1], s[ 5], s[ 9], s[13])
    s[ 2], s[ 6], s[10], s[14] = G(s[ 2], s[ 6], s[10], s[14])
    s[ 3], s[ 7], s[11], s[15] = G(s[ 3], s[ 7], s[11], s[15])
    // Diagonal step
    s[ 0], s[ 5], s[10], s[15] = G(s[ 0], s[ 5], s[10], s[15])
    s[ 1], s[ 6], s[11], s[12] = G(s[ 1], s[ 6], s[11], s[12])
    s[ 2], s[ 7], s[ 8], s[13] = G(s[ 2], s[ 7], s[ 8], s[13])
    s[ 3], s[ 4], s[ 9], s[14] = G(s[ 3], s[ 4], s[ 9], s[14])
}

func permute(state *state_t) {
    s := state.s[:]
    for i:=0; i<NORX_R; i++ { F(s) }
}

func setup(state *state_t, k []uint8, n []uint8) {

    s := state.s[:]

    s[ 0] = u0
    s[ 1] = LOAD64(n[ 0: 8])
    s[ 2] = LOAD64(n[ 8:16])
    s[ 3] = u1

    s[ 4] = LOAD64(k[ 0: 8])
    s[ 5] = LOAD64(k[ 8:16])
    s[ 6] = LOAD64(k[16:24])
    s[ 7] = LOAD64(k[24:32])

    s[ 8] = u2
    s[ 9] = u3
    s[10] = u4
    s[11] = u5

    s[12] = u6
    s[13] = u7
    s[14] = u8
    s[15] = u9

    s[14] ^= (NORX_R << 26) | (NORX_D << 18) | (NORX_W << 10) | NORX_A
    permute(state)
}

func process_header(state *state_t, in []uint8, inlen uint64) {

    if inlen > 0 {
        i := uint64(0)
        n := BYTES64(RATE)
        for inlen >= n {
            absorb_block(state, in[n*i:n*(i+1)], HEADER_TAG)
            inlen -= n
            i++
        }
        absorb_lastblock(state, in[n*i:n*i+inlen], inlen, HEADER_TAG)
    }
}

func encrypt_msg(state *state_t, out []uint8, in []uint8, inlen uint64) {

    if inlen > 0 {
        n := BYTES64(RATE)
        i := uint64(0)
        for inlen >= n {
            encrypt_block(state, out[n*i:n*(i+1)], in[n*i:n*(i+1)])
            inlen -= n
            i++
        }
        encrypt_lastblock(state, out[n*i:n*i+inlen], in[n*i:n*i+inlen], inlen)
    }
}

func decrypt_msg(state *state_t, out []uint8, in []uint8, inlen uint64) {

    if inlen > 0 {
        n := BYTES64(RATE)
        i := uint64(0)
        for inlen >= n {
            decrypt_block(state, out[n*i:n*(i+1)], in[n*i:n*(i+1)])
            inlen -= n
            i++
        }
        decrypt_lastblock(state, out[n*i:n*i+inlen], in[n*i:n*i+inlen], inlen)
    }
}

func process_trailer(state *state_t, in []uint8, inlen uint64) {

    if inlen > 0 {
        i := uint64(0)
        n := BYTES64(RATE)
        for inlen >= n {
            absorb_block(state, in[n*i:n*(i+1)], TRAILER_TAG)
            inlen -= n
            i++
        }
        absorb_lastblock(state, in[n*i:n*i+inlen], inlen, TRAILER_TAG)
    }
}

func output_tag(state *state_t, tag []uint8) {

    inject_tag(state, FINAL_TAG)
    permute(state)
    permute(state)

    s := state.s[:]
    lastblock := make([]uint8, BYTES64(RATE))
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        STORE64(lastblock[b*i:b*(i+1)], s[i])
    }
    copy(tag[:], lastblock[:])
    BURN8(lastblock[:], BYTES64(RATE))
}

func verify_tag(tag1 []uint8, tag2 []uint8) int {

    acc := 0
    for i := uint64(0); i < BYTES64(NORX_A); i++ {
        acc |= int(tag1[i] ^ tag2[i])
    }
    return (((acc - 1) >> 8) & 1) - 1
}

func pad(out []uint8, in []uint8, inlen uint64) {

    copy(out[:],in[:inlen])
    out[inlen] = 0x01
    out[BYTES64(RATE) - 1] |= 0x80
}

func inject_tag(state *state_t, tag uint64) {
    s := state.s[:]
    s[15] ^= tag
}

func absorb_block(state *state_t, in []uint8, tag uint64) {

    inject_tag(state, tag)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        s[i] ^= LOAD64(in[b*i:b*(i+1)])
    }
}

func absorb_lastblock(state *state_t, in []uint8, inlen uint64, tag uint64) {

    lastblock := make([]uint8, BYTES64(RATE))
    pad(lastblock[:], in[:], inlen)
    absorb_block(state, lastblock[:], tag)
    BURN8(lastblock[:], BYTES64(RATE))
}

func encrypt_block(state *state_t, out []uint8, in []uint8) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        s[i] ^= LOAD64(in[b*i:b*(i+1)])
        STORE64(out[b*i:b*(i+1)], s[i])
    }
}

func encrypt_lastblock(state *state_t, out []uint8, in []uint8, inlen uint64) {

    lastblock := make([]uint8, BYTES64(RATE))
    pad(lastblock[:], in[:], inlen)
    encrypt_block(state, lastblock[:], lastblock[:])
    copy(out[:], lastblock[:])
    BURN8(lastblock[:], BYTES64(RATE))
}

func decrypt_block(state *state_t, out []uint8, in []uint8) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    s := state.s[:]
    b := BYTES64(NORX_W)
    for i:= uint64(0); i < WORDS64(RATE); i++ {
        c := LOAD64(in[b*i:b*(i+1)])
        STORE64(out[b*i:b*(i+1)], s[i] ^ c)
        s[i] = c
    }
}

func decrypt_lastblock(state *state_t, out []uint8, in []uint8, inlen uint64) {

    inject_tag(state, PAYLOAD_TAG)
    permute(state)

    i := uint64(0)
    s := state.s[:]
    n := BYTES64(NORX_W)
    b := make([]uint8, n)

    /* undo padding */
    s[inlen / n] ^= uint64(0x01) << (inlen % n * 8)
    s[WORDS64(RATE) - 1] ^= uint64(0x80) << ((BYTES64(RATE) - 1) % n * 8)

    for inlen >= n {
        c := LOAD64(in[n*i:n*(i+1)])
        STORE64(out[n*i:n*(i+1)], s[i] ^ c)
        s[i] = c
        inlen -= n
        i++
    }

    STORE64(b,s[i])
    for j := uint64(0); j < inlen; j++ {
      c := in[n*i+j]
      out[n*i+j] = b[j] ^ c
      b[j] = c
    }
    s[i] = LOAD64(b)
    BURN8(b,n)
}

func AEAD_encrypt(
    c []uint8, clen *uint64,
    h []uint8, hlen uint64,
    m []uint8, mlen uint64,
    t []uint8, tlen uint64,
    nonce []uint8,
    key []uint8) {

    state := new(state_t)
    setup(state, key, nonce)
    process_header(state, h, hlen)
    encrypt_msg(state, c, m, mlen)
    process_trailer(state, t, tlen)
    output_tag(state, c[ mlen: ])
    *clen = mlen + BYTES64(NORX_A)
    BURN64(state.s[:], WORDS64(NORX_B))
}

func AEAD_decrypt(
    m []uint8, mlen *uint64,
    h []uint8, hlen uint64,
    c []uint8, clen uint64,
    t []uint8, tlen uint64,
    nonce []uint8,
    key []uint8) int {

    if clen < BYTES64(NORX_A) {
      return -1
    }
    result := -1
    tag := make([]uint8, NORX_A)
    state := new(state_t)
    setup(state, key, nonce)
    process_header(state, h, hlen)
    decrypt_msg(state, m, c, clen - BYTES64(NORX_A))
    process_trailer(state, t, tlen)
    output_tag(state, tag)
    *mlen = clen - BYTES64(NORX_A)
    result = verify_tag(c[clen - BYTES64(NORX_A):], tag)
    if result != 0 {
        BURN8(m, clen - BYTES64(NORX_A))
    }
    BURN64(state.s[:], WORDS64(NORX_B))
    return result
}

<|endoftext|>


<commit_before>// Package parser implements a parser and parse tree dumper for Dockerfiles.
package parser // import "github.com/docker/docker/builder/dockerfile/parser"

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"unicode"

	"github.com/docker/docker/builder/dockerfile/command"
	"github.com/docker/docker/pkg/system"
	"github.com/pkg/errors"
)

// Node is a structure used to represent a parse tree.
//
// In the node there are three fields, Value, Next, and Children. Value is the
// current token's string value. Next is always the next non-child token, and
// children contains all the children. Here's an example:
//
// (value next (child child-next child-next-next) next-next)
//
// This data structure is frankly pretty lousy for handling complex languages,
// but lucky for us the Dockerfile isn't very complicated. This structure
// works a little more effectively than a "proper" parse tree for our needs.
//
type Node struct {
	Value      string          // actual content
	Next       *Node           // the next item in the current sexp
	Children   []*Node         // the children of this sexp
	Attributes map[string]bool // special attributes for this node
	Original   string          // original line used before parsing
	Flags      []string        // only top Node should have this set
	StartLine  int             // the line in the original dockerfile where the node begins
	endLine    int             // the line in the original dockerfile where the node ends
}

// Dump dumps the AST defined by `node` as a list of sexps.
// Returns a string suitable for printing.
func (node *Node) Dump() string {
	str := ""
	str += node.Value

	if len(node.Flags) > 0 {
		str += fmt.Sprintf(" %q", node.Flags)
	}

	for _, n := range node.Children {
		str += "(" + n.Dump() + ")\n"
	}

	for n := node.Next; n != nil; n = n.Next {
		if len(n.Children) > 0 {
			str += " " + n.Dump()
		} else {
			str += " " + strconv.Quote(n.Value)
		}
	}

	return strings.TrimSpace(str)
}

func (node *Node) lines(start, end int) {
	node.StartLine = start
	node.endLine = end
}

// AddChild adds a new child node, and updates line information
func (node *Node) AddChild(child *Node, startLine, endLine int) {
	child.lines(startLine, endLine)
	if node.StartLine < 0 {
		node.StartLine = startLine
	}
	node.endLine = endLine
	node.Children = append(node.Children, child)
}

var (
	dispatch             map[string]func(string, *Directive) (*Node, map[string]bool, error)
	tokenWhitespace      = regexp.MustCompile(`[\t\v\f\r ]+`)
	tokenEscapeCommand   = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
	tokenPlatformCommand = regexp.MustCompile(`^#[ \t]*platform[ \t]*=[ \t]*(?P<platform>.*)$`)
	tokenComment         = regexp.MustCompile(`^#.*$`)
)

// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = '\\'

// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
	escapeToken           rune           // Current escape token
	platformToken         string         // Current platform token
	lineContinuationRegex *regexp.Regexp // Current line continuation regex
	processingComplete    bool           // Whether we are done looking for directives
	escapeSeen            bool           // Whether the escape directive has been seen
	platformSeen          bool           // Whether the platform directive has been seen
}

// setEscapeToken sets the default token for escaping characters in a Dockerfile.
func (d *Directive) setEscapeToken(s string) error {
	if s != "`" && s != "\\" {
		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
	}
	d.escapeToken = rune(s[0])
	d.lineContinuationRegex = regexp.MustCompile(`\` + s + `[ \t]*$`)
	return nil
}

// setPlatformToken sets the default platform for pulling images in a Dockerfile.
func (d *Directive) setPlatformToken(s string) error {
	s = strings.ToLower(s)
	valid := []string{runtime.GOOS}
	if system.LCOWSupported() {
		valid = append(valid, "linux")
	}
	for _, item := range valid {
		if s == item {
			d.platformToken = s
			return nil
		}
	}
	return fmt.Errorf("invalid PLATFORM '%s'. Must be one of %v", s, valid)
}

// possibleParserDirective looks for one or more parser directives '# escapeToken=<char>' and
// '# platform=<string>'. Parser directives must precede any builder instruction
// or other comments, and cannot be repeated.
func (d *Directive) possibleParserDirective(line string) error {
	if d.processingComplete {
		return nil
	}

	tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
	if len(tecMatch) != 0 {
		for i, n := range tokenEscapeCommand.SubexpNames() {
			if n == "escapechar" {
				if d.escapeSeen {
					return errors.New("only one escape parser directive can be used")
				}
				d.escapeSeen = true
				return d.setEscapeToken(tecMatch[i])
			}
		}
	}

	// Only recognise a platform token if LCOW is supported
	if system.LCOWSupported() {
		tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line))
		if len(tpcMatch) != 0 {
			for i, n := range tokenPlatformCommand.SubexpNames() {
				if n == "platform" {
					if d.platformSeen {
						return errors.New("only one platform parser directive can be used")
					}
					d.platformSeen = true
					return d.setPlatformToken(tpcMatch[i])
				}
			}
		}
	}

	d.processingComplete = true
	return nil
}

// NewDefaultDirective returns a new Directive with the default escapeToken token
func NewDefaultDirective() *Directive {
	directive := Directive{}
	directive.setEscapeToken(string(DefaultEscapeToken))
	return &directive
}

func init() {
	// Dispatch Table. see line_parsers.go for the parse functions.
	// The command is parsed and mapped to the line parser. The line parser
	// receives the arguments but not the command, and returns an AST after
	// reformulating the arguments according to the rules in the parser
	// functions. Errors are propagated up by Parse() and the resulting AST can
	// be incorporated directly into the existing AST as a next.
	dispatch = map[string]func(string, *Directive) (*Node, map[string]bool, error){
		command.Add:         parseMaybeJSONToList,
		command.Arg:         parseNameOrNameVal,
		command.Cmd:         parseMaybeJSON,
		command.Copy:        parseMaybeJSONToList,
		command.Entrypoint:  parseMaybeJSON,
		command.Env:         parseEnv,
		command.Expose:      parseStringsWhitespaceDelimited,
		command.From:        parseStringsWhitespaceDelimited,
		command.Healthcheck: parseHealthConfig,
		command.Label:       parseLabel,
		command.Maintainer:  parseString,
		command.Onbuild:     parseSubCommand,
		command.Run:         parseMaybeJSON,
		command.Shell:       parseMaybeJSON,
		command.StopSignal:  parseString,
		command.User:        parseString,
		command.Volume:      parseMaybeJSONToList,
		command.Workdir:     parseString,
	}
}

// newNodeFromLine splits the line into parts, and dispatches to a function
// based on the command and command arguments. A Node is created from the
// result of the dispatch.
func newNodeFromLine(line string, directive *Directive) (*Node, error) {
	cmd, flags, args, err := splitCommand(line)
	if err != nil {
		return nil, err
	}

	fn := dispatch[cmd]
	// Ignore invalid Dockerfile instructions
	if fn == nil {
		fn = parseIgnore
	}
	next, attrs, err := fn(args, directive)
	if err != nil {
		return nil, err
	}

	return &Node{
		Value:      cmd,
		Original:   line,
		Flags:      flags,
		Next:       next,
		Attributes: attrs,
	}, nil
}

// Result is the result of parsing a Dockerfile
type Result struct {
	AST         *Node
	EscapeToken rune
	Warnings    []string
}

// PrintWarnings to the writer
func (r *Result) PrintWarnings(out io.Writer) {
	if len(r.Warnings) == 0 {
		return
	}
	fmt.Fprintf(out, strings.Join(r.Warnings, "\n")+"\n")
}

// Parse reads lines from a Reader, parses the lines into an AST and returns
// the AST and escape token
func Parse(rwc io.Reader) (*Result, error) {
	d := NewDefaultDirective()
	currentLine := 0
	root := &Node{StartLine: -1}
	scanner := bufio.NewScanner(rwc)
	warnings := []string{}

	var err error
	for scanner.Scan() {
		bytesRead := scanner.Bytes()
		if currentLine == 0 {
			// First line, strip the byte-order-marker if present
			bytesRead = bytes.TrimPrefix(bytesRead, utf8bom)
		}
		bytesRead, err = processLine(d, bytesRead, true)
		if err != nil {
			return nil, err
		}
		currentLine++

		startLine := currentLine
		line, isEndOfLine := trimContinuationCharacter(string(bytesRead), d)
		if isEndOfLine && line == "" {
			continue
		}

		var hasEmptyContinuationLine bool
		for !isEndOfLine && scanner.Scan() {
			bytesRead, err := processLine(d, scanner.Bytes(), false)
			if err != nil {
				return nil, err
			}
			currentLine++

			if isComment(scanner.Bytes()) {
				// original line was a comment (processLine strips comments)
				continue
			}
			if isEmptyContinuationLine(bytesRead) {
				hasEmptyContinuationLine = true
				continue
			}

			continuationLine := string(bytesRead)
			continuationLine, isEndOfLine = trimContinuationCharacter(continuationLine, d)
			line += continuationLine
		}

		if hasEmptyContinuationLine {
			warning := "[WARNING]: Empty continuation line found in:\n    " + line
			warnings = append(warnings, warning)
		}

		child, err := newNodeFromLine(line, d)
		if err != nil {
			return nil, err
		}
		root.AddChild(child, startLine, currentLine)
	}

	if len(warnings) > 0 {
		warnings = append(warnings, "[WARNING]: Empty continuation lines will become errors in a future release.")
	}
	return &Result{
		AST:         root,
		Warnings:    warnings,
		EscapeToken: d.escapeToken,
	}, handleScannerError(scanner.Err())
}

func trimComments(src []byte) []byte {
	return tokenComment.ReplaceAll(src, []byte{})
}

func trimWhitespace(src []byte) []byte {
	return bytes.TrimLeftFunc(src, unicode.IsSpace)
}

func isComment(line []byte) bool {
	return tokenComment.Match(trimWhitespace(line))
}

func isEmptyContinuationLine(line []byte) bool {
	return len(trimWhitespace(line)) == 0
}

var utf8bom = []byte{0xEF, 0xBB, 0xBF}

func trimContinuationCharacter(line string, d *Directive) (string, bool) {
	if d.lineContinuationRegex.MatchString(line) {
		line = d.lineContinuationRegex.ReplaceAllString(line, "")
		return line, false
	}
	return line, true
}

// TODO: remove stripLeftWhitespace after deprecation period. It seems silly
// to preserve whitespace on continuation lines. Why is that done?
func processLine(d *Directive, token []byte, stripLeftWhitespace bool) ([]byte, error) {
	if stripLeftWhitespace {
		token = trimWhitespace(token)
	}
	return trimComments(token), d.possibleParserDirective(string(token))
}

func handleScannerError(err error) error {
	switch err {
	case bufio.ErrTooLong:
		return errors.Errorf("dockerfile line greater than max allowed size of %d", bufio.MaxScanTokenSize-1)
	default:
		return err
	}
}
<commit_msg>Builder - Parser. Remove platform parser directive<commit_after>// Package parser implements a parser and parse tree dumper for Dockerfiles.
package parser // import "github.com/docker/docker/builder/dockerfile/parser"

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/docker/docker/builder/dockerfile/command"
	"github.com/pkg/errors"
)

// Node is a structure used to represent a parse tree.
//
// In the node there are three fields, Value, Next, and Children. Value is the
// current token's string value. Next is always the next non-child token, and
// children contains all the children. Here's an example:
//
// (value next (child child-next child-next-next) next-next)
//
// This data structure is frankly pretty lousy for handling complex languages,
// but lucky for us the Dockerfile isn't very complicated. This structure
// works a little more effectively than a "proper" parse tree for our needs.
//
type Node struct {
	Value      string          // actual content
	Next       *Node           // the next item in the current sexp
	Children   []*Node         // the children of this sexp
	Attributes map[string]bool // special attributes for this node
	Original   string          // original line used before parsing
	Flags      []string        // only top Node should have this set
	StartLine  int             // the line in the original dockerfile where the node begins
	endLine    int             // the line in the original dockerfile where the node ends
}

// Dump dumps the AST defined by `node` as a list of sexps.
// Returns a string suitable for printing.
func (node *Node) Dump() string {
	str := ""
	str += node.Value

	if len(node.Flags) > 0 {
		str += fmt.Sprintf(" %q", node.Flags)
	}

	for _, n := range node.Children {
		str += "(" + n.Dump() + ")\n"
	}

	for n := node.Next; n != nil; n = n.Next {
		if len(n.Children) > 0 {
			str += " " + n.Dump()
		} else {
			str += " " + strconv.Quote(n.Value)
		}
	}

	return strings.TrimSpace(str)
}

func (node *Node) lines(start, end int) {
	node.StartLine = start
	node.endLine = end
}

// AddChild adds a new child node, and updates line information
func (node *Node) AddChild(child *Node, startLine, endLine int) {
	child.lines(startLine, endLine)
	if node.StartLine < 0 {
		node.StartLine = startLine
	}
	node.endLine = endLine
	node.Children = append(node.Children, child)
}

var (
	dispatch           map[string]func(string, *Directive) (*Node, map[string]bool, error)
	tokenWhitespace    = regexp.MustCompile(`[\t\v\f\r ]+`)
	tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
	tokenComment       = regexp.MustCompile(`^#.*$`)
)

// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = '\\'

// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
	escapeToken           rune           // Current escape token
	lineContinuationRegex *regexp.Regexp // Current line continuation regex
	processingComplete    bool           // Whether we are done looking for directives
	escapeSeen            bool           // Whether the escape directive has been seen
}

// setEscapeToken sets the default token for escaping characters in a Dockerfile.
func (d *Directive) setEscapeToken(s string) error {
	if s != "`" && s != "\\" {
		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
	}
	d.escapeToken = rune(s[0])
	d.lineContinuationRegex = regexp.MustCompile(`\` + s + `[ \t]*$`)
	return nil
}

// possibleParserDirective looks for parser directives, eg '# escapeToken=<char>'.
// Parser directives must precede any builder instruction or other comments,
// and cannot be repeated.
func (d *Directive) possibleParserDirective(line string) error {
	if d.processingComplete {
		return nil
	}

	tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
	if len(tecMatch) != 0 {
		for i, n := range tokenEscapeCommand.SubexpNames() {
			if n == "escapechar" {
				if d.escapeSeen {
					return errors.New("only one escape parser directive can be used")
				}
				d.escapeSeen = true
				return d.setEscapeToken(tecMatch[i])
			}
		}
	}

	d.processingComplete = true
	return nil
}

// NewDefaultDirective returns a new Directive with the default escapeToken token
func NewDefaultDirective() *Directive {
	directive := Directive{}
	directive.setEscapeToken(string(DefaultEscapeToken))
	return &directive
}

func init() {
	// Dispatch Table. see line_parsers.go for the parse functions.
	// The command is parsed and mapped to the line parser. The line parser
	// receives the arguments but not the command, and returns an AST after
	// reformulating the arguments according to the rules in the parser
	// functions. Errors are propagated up by Parse() and the resulting AST can
	// be incorporated directly into the existing AST as a next.
	dispatch = map[string]func(string, *Directive) (*Node, map[string]bool, error){
		command.Add:         parseMaybeJSONToList,
		command.Arg:         parseNameOrNameVal,
		command.Cmd:         parseMaybeJSON,
		command.Copy:        parseMaybeJSONToList,
		command.Entrypoint:  parseMaybeJSON,
		command.Env:         parseEnv,
		command.Expose:      parseStringsWhitespaceDelimited,
		command.From:        parseStringsWhitespaceDelimited,
		command.Healthcheck: parseHealthConfig,
		command.Label:       parseLabel,
		command.Maintainer:  parseString,
		command.Onbuild:     parseSubCommand,
		command.Run:         parseMaybeJSON,
		command.Shell:       parseMaybeJSON,
		command.StopSignal:  parseString,
		command.User:        parseString,
		command.Volume:      parseMaybeJSONToList,
		command.Workdir:     parseString,
	}
}

// newNodeFromLine splits the line into parts, and dispatches to a function
// based on the command and command arguments. A Node is created from the
// result of the dispatch.
func newNodeFromLine(line string, directive *Directive) (*Node, error) {
	cmd, flags, args, err := splitCommand(line)
	if err != nil {
		return nil, err
	}

	fn := dispatch[cmd]
	// Ignore invalid Dockerfile instructions
	if fn == nil {
		fn = parseIgnore
	}
	next, attrs, err := fn(args, directive)
	if err != nil {
		return nil, err
	}

	return &Node{
		Value:      cmd,
		Original:   line,
		Flags:      flags,
		Next:       next,
		Attributes: attrs,
	}, nil
}

// Result is the result of parsing a Dockerfile
type Result struct {
	AST         *Node
	EscapeToken rune
	Warnings    []string
}

// PrintWarnings to the writer
func (r *Result) PrintWarnings(out io.Writer) {
	if len(r.Warnings) == 0 {
		return
	}
	fmt.Fprintf(out, strings.Join(r.Warnings, "\n")+"\n")
}

// Parse reads lines from a Reader, parses the lines into an AST and returns
// the AST and escape token
func Parse(rwc io.Reader) (*Result, error) {
	d := NewDefaultDirective()
	currentLine := 0
	root := &Node{StartLine: -1}
	scanner := bufio.NewScanner(rwc)
	warnings := []string{}

	var err error
	for scanner.Scan() {
		bytesRead := scanner.Bytes()
		if currentLine == 0 {
			// First line, strip the byte-order-marker if present
			bytesRead = bytes.TrimPrefix(bytesRead, utf8bom)
		}
		bytesRead, err = processLine(d, bytesRead, true)
		if err != nil {
			return nil, err
		}
		currentLine++

		startLine := currentLine
		line, isEndOfLine := trimContinuationCharacter(string(bytesRead), d)
		if isEndOfLine && line == "" {
			continue
		}

		var hasEmptyContinuationLine bool
		for !isEndOfLine && scanner.Scan() {
			bytesRead, err := processLine(d, scanner.Bytes(), false)
			if err != nil {
				return nil, err
			}
			currentLine++

			if isComment(scanner.Bytes()) {
				// original line was a comment (processLine strips comments)
				continue
			}
			if isEmptyContinuationLine(bytesRead) {
				hasEmptyContinuationLine = true
				continue
			}

			continuationLine := string(bytesRead)
			continuationLine, isEndOfLine = trimContinuationCharacter(continuationLine, d)
			line += continuationLine
		}

		if hasEmptyContinuationLine {
			warning := "[WARNING]: Empty continuation line found in:\n    " + line
			warnings = append(warnings, warning)
		}

		child, err := newNodeFromLine(line, d)
		if err != nil {
			return nil, err
		}
		root.AddChild(child, startLine, currentLine)
	}

	if len(warnings) > 0 {
		warnings = append(warnings, "[WARNING]: Empty continuation lines will become errors in a future release.")
	}
	return &Result{
		AST:         root,
		Warnings:    warnings,
		EscapeToken: d.escapeToken,
	}, handleScannerError(scanner.Err())
}

func trimComments(src []byte) []byte {
	return tokenComment.ReplaceAll(src, []byte{})
}

func trimWhitespace(src []byte) []byte {
	return bytes.TrimLeftFunc(src, unicode.IsSpace)
}

func isComment(line []byte) bool {
	return tokenComment.Match(trimWhitespace(line))
}

func isEmptyContinuationLine(line []byte) bool {
	return len(trimWhitespace(line)) == 0
}

var utf8bom = []byte{0xEF, 0xBB, 0xBF}

func trimContinuationCharacter(line string, d *Directive) (string, bool) {
	if d.lineContinuationRegex.MatchString(line) {
		line = d.lineContinuationRegex.ReplaceAllString(line, "")
		return line, false
	}
	return line, true
}

// TODO: remove stripLeftWhitespace after deprecation period. It seems silly
// to preserve whitespace on continuation lines. Why is that done?
func processLine(d *Directive, token []byte, stripLeftWhitespace bool) ([]byte, error) {
	if stripLeftWhitespace {
		token = trimWhitespace(token)
	}
	return trimComments(token), d.possibleParserDirective(string(token))
}

func handleScannerError(err error) error {
	switch err {
	case bufio.ErrTooLong:
		return errors.Errorf("dockerfile line greater than max allowed size of %d", bufio.MaxScanTokenSize-1)
	default:
		return err
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

func readConfig(conf string) ([]status, error) {
	var config sites

	_, err := toml.DecodeFile(conf, &config)
	if err != nil {
		return nil, err
	}

	return config.Service, nil
}

func checkStatus(site_status []status) {
	count := 0
	c := make(chan int)

	for i, s := range site_status {
		site_status[i].Status = "offline"

		ck, ok := check[s.Type]
		if ok == false {
			log.Println(s.Type, s.URL, "unknown type")
			continue
		}

		count++

		go func(site status, i int) {
			healthy, err := ck.Check(site.URL)
			if err == nil && healthy {
				site_status[i].Status = "online"
			}

			if err != nil {
				log.Println(site.Type, site.URL, err)
			}

			c <- i
		}(s, i)
	}

	for i := 0; i < count; i++ {
		<-c
	}
}

func sendStatus(w http.ResponseWriter, site_status []status, file string) error {
	t, err := template.ParseFiles(file)
	if err != nil {
		return err
	}

	err = t.Execute(w, site_status)
	if err != nil {
		return err
	}

	return nil
}

var site_status []status
var next_status time.Time

func statusHandler(w http.ResponseWriter, r *http.Request) {
	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	var err error

	if next_status.Before(time.Now()) {
		site_status, err = readConfig("sitecheck.conf")
		if err != nil {
			http.Error(w, "internal error", http.StatusInternalServerError)
			log.Println(err)
			return
		}

		checkStatus(site_status)

		next_status = time.Now().Add(time.Second * 30)
	}

	err = sendStatus(w, site_status, "status.html")
	if err != nil {
		http.Error(w, "internal error", http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var port = flag.String("http", "", "HTTP service address (.e.g. :8080)")

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	http.HandleFunc("/", statusHandler)

	log.Fatal(http.ListenAndServe(*port, nil))
}
<commit_msg>serialize requests through the server<commit_after>package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"sync"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

type status struct {
	Name   string `toml:"name"`
	Type   string `toml:"type"`
	Status string
	URL    string `toml:"url"`
}

type sites struct {
	Service []status
}

type Status interface {
	Check(url string) (bool, error)
}

var check map[string]Status

func readConfig(conf string) ([]status, error) {
	var config sites

	_, err := toml.DecodeFile(conf, &config)
	if err != nil {
		return nil, err
	}

	return config.Service, nil
}

func checkStatus(site_status []status) {
	count := 0
	c := make(chan int)

	for i, s := range site_status {
		site_status[i].Status = "offline"

		ck, ok := check[s.Type]
		if ok == false {
			log.Println(s.Type, s.URL, "unknown type")
			continue
		}

		count++

		go func(site status, i int) {
			healthy, err := ck.Check(site.URL)
			if err == nil && healthy {
				site_status[i].Status = "online"
			}

			if err != nil {
				log.Println(site.Type, site.URL, err)
			}

			c <- i
		}(s, i)
	}

	for i := 0; i < count; i++ {
		<-c
	}
}

func sendStatus(w http.ResponseWriter, site_status []status, file string) error {
	t, err := template.ParseFiles(file)
	if err != nil {
		return err
	}

	err = t.Execute(w, site_status)
	if err != nil {
		return err
	}

	return nil
}

var site_status []status
var next_status time.Time
var lock sync.Mutex

func statusHandler(w http.ResponseWriter, r *http.Request) {
	lock.Lock()
	defer lock.Unlock()

	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	log.Println("request from", host)

	var err error

	if next_status.Before(time.Now()) {
		site_status, err = readConfig("sitecheck.conf")
		if err != nil {
			http.Error(w, "internal error", http.StatusInternalServerError)
			log.Println(err)
			return
		}

		checkStatus(site_status)

		next_status = time.Now().Add(time.Second * 30)
	}

	err = sendStatus(w, site_status, "status.html")
	if err != nil {
		http.Error(w, "internal error", http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func init() {
	check = map[string]Status{
		"website":  new(Website),
		"etcd":     new(Etcd),
		"docker":   new(Docker),
		"registry": new(Registry),
	}
}

func main() {
	var port = flag.String("http", "", "HTTP service address (.e.g. :8080)")

	flag.Parse()

	if *port == "" {
		flag.Usage()
		return
	}

	http.HandleFunc("/", statusHandler)

	log.Fatal(http.ListenAndServe(*port, nil))
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("从conn读取数据向ringbuffer写入失败")
		})
		return total, err
	}
	b__ := make([]byte, 0, total)
	fmt.Println(b_)
	b__ = append(b__, b[0:1+m]...)
	b__ = append(b__, b_[0:]...)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>添加测试代码<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("从conn读取数据失败")
		})
		return total, err
	}
	b__ := make([]byte, 0, total)
	fmt.Println(b_)
	b__ = append(b__, b[0:1+m]...)
	b__ = append(b__, b_[0:]...)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>package healthcheck

import (
	"database/sql"
	"errors"
	"io/ioutil"
	"log"
	"strings"

	"gopkg.in/yaml.v2"
)

// SQLHealthCheck is a data type for storing the definition
// and results of a SQL based health check
type SQLHealthCheck struct {
	Expected string `yaml:"expected"`
	Query    string `yaml:"query"`
	Title    string `yaml:"title"`
	Severity string `yaml:"severity"`
	Passed   bool
	Actual   string
}

// Format is for unmarshiling a healthcheck file
// and contains control information for a set of SQLHealthChecks
type Format struct {
	Name         string           `yaml:"name"`
	Distribution []string         `yaml:"distribution"`
	Tests        []SQLHealthCheck `yaml:"tests"`
}

// HCError is a error helper for knowing to exit early on a failed healthcheck
type HCError struct {
	Err  string
	Exit bool
}

func unmarshalHealthChecks(yamldata []byte) Format {

	var data Format

	err := yaml.Unmarshal(yamldata, &data)
	if err != nil {
		log.Fatalf("ERROR: %v", err)
	}

	// inflate queryfiles
	return data
}

// ReadYamlFromFile loads healthcheck data from a YAML file
func ReadYamlFromFile(path string) Format {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal(err)
	}
	return unmarshalHealthChecks(data)
}

// RunHealthChecks executes all health checks in the specified file
func RunHealthChecks(healthChecks Format, cxn *sql.DB) Format {

	for _, test := range healthChecks.Tests {
		test = RunHealthCheck(test, cxn)
	}
	return healthChecks
}

// EvaluateHealthChecks contains logic for handling the results of RunHealthChecks
func EvaluateHealthChecks(healthChecks Format) (results []SQLHealthCheck, err error) {

	for _, test := range healthChecks.Tests {
		results = append(results, test)
		hcErr := EvaluateHealthCheck(test)

		if hcErr.Err != "" {
			err = errors.New(hcErr.Err)
		}

		if hcErr.Exit {
			break
		}

	}
	return results, err
}

// PreformHealthChecks runs and evaluates healthChecks one at a time
func PreformHealthChecks(healthChecks Format, cxn *sql.DB) (results []SQLHealthCheck, err error) {
	for _, test := range healthChecks.Tests {
		test = RunHealthCheck(test, cxn)
		results = append(results, test)
		hcErr := EvaluateHealthCheck(test)

		if hcErr.Err != "" {
			err = errors.New(hcErr.Err)
		}

		if hcErr.Exit {
			break
		}

	}
	return results, err
}

// RunHealthCheck runs through a single healthcheck and saves the result
func RunHealthCheck(healthCheck SQLHealthCheck, cxn *sql.DB) SQLHealthCheck {
	answer := ""

	rows, err := cxn.Query(healthCheck.Query)

	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	rows.Next()
	rows.Scan(&answer)

	healthCheck.Passed = healthCheck.Expected == answer
	healthCheck.Actual = answer
	return healthCheck
}

// EvaluateHealthCheck runs through a single healthcheck and acts on the result
func EvaluateHealthCheck(healthCheck SQLHealthCheck) (err HCError) {

	prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
	if !healthCheck.Passed {
		switch strings.ToLower(healthCheck.Severity) {

		// When Fatal, return early with an error
		case "fatal":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("FATAL healthcheck failed\nBreaking Away Early\n%s\n\n", string(prettyHealthCheck))
			err = HCError{"FATAL healthCheck failure", true}

		// When Error, keep running but add an error
		case "error":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("%s healthcheck failed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
			err = HCError{"ERROR healthCheck failure", false}

		// When warn or info, print out the result and keep running
		case "warn", "info":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("%s healthcheck failed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
		default:
			log.Printf("undefined severity level:%s\n%s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
		}
	} else {
		log.Printf("%s healthcheck passed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
	}

	return err

}

// Implementation of report.Element

// HealthCheckReportHeaders headers used for GetHeaders
var HealthCheckReportHeaders = []string{"Title", "Query", "Passed", "Expected", "Actual"}

// GetHeaders Implementation for report.Element
func (hcr SQLHealthCheck) GetHeaders() []string {
	return HealthCheckReportHeaders[0:]
}

// GetValue Implementation for report.Element
func (hcr SQLHealthCheck) GetValue(key string) string {

	switch key {
	case HealthCheckReportHeaders[0]:
		return hcr.Title
	case HealthCheckReportHeaders[1]:
		return hcr.Query
	case HealthCheckReportHeaders[2]:
		if hcr.Passed {
			return "SUCCESS"
		}
		return "FAIL"
	case HealthCheckReportHeaders[3]:
		return hcr.Expected
	case HealthCheckReportHeaders[4]:
		return hcr.Actual
	}
	return ""
}
<commit_msg>return array of errors<commit_after>package healthcheck

import (
	"database/sql"
	"errors"
	"io/ioutil"
	"log"
	"strings"

	"gopkg.in/yaml.v2"
)

// SQLHealthCheck is a data type for storing the definition
// and results of a SQL based health check
type SQLHealthCheck struct {
	Expected string `yaml:"expected"`
	Query    string `yaml:"query"`
	Title    string `yaml:"title"`
	Severity string `yaml:"severity"`
	Passed   bool
	Actual   string
}

// Format is for unmarshiling a healthcheck file
// and contains control information for a set of SQLHealthChecks
type Format struct {
	Name         string           `yaml:"name"`
	Distribution []string         `yaml:"distribution"`
	Tests        []SQLHealthCheck `yaml:"tests"`
}

// HCError is a error helper for knowing to exit early on a failed healthcheck
type HCError struct {
	Err  string
	Exit bool
}

func unmarshalHealthChecks(yamldata []byte) Format {

	var data Format

	err := yaml.Unmarshal(yamldata, &data)
	if err != nil {
		log.Fatalf("ERROR: %v", err)
	}

	// inflate queryfiles
	return data
}

// ReadYamlFromFile loads healthcheck data from a YAML file
func ReadYamlFromFile(path string) Format {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal(err)
	}
	return unmarshalHealthChecks(data)
}

// RunHealthChecks executes all health checks in the specified file
func RunHealthChecks(healthChecks Format, cxn *sql.DB) Format {

	for _, test := range healthChecks.Tests {
		test = RunHealthCheck(test, cxn)
	}
	return healthChecks
}

// EvaluateHealthChecks contains logic for handling the results of RunHealthChecks
func EvaluateHealthChecks(healthChecks Format) (results []SQLHealthCheck, err error) {

	for _, test := range healthChecks.Tests {
		results = append(results, test)
		hcErr := EvaluateHealthCheck(test)

		if hcErr.Err != "" {
			err = errors.New(hcErr.Err)
		}

		if hcErr.Exit {
			break
		}

	}
	return results, err
}

// PreformHealthChecks runs and evaluates healthChecks one at a time
func PreformHealthChecks(healthChecks Format, cxn *sql.DB) (results []SQLHealthCheck, errors []HCError) {
	for _, test := range healthChecks.Tests {
		test = RunHealthCheck(test, cxn)
		results = append(results, test)
		hcErr := EvaluateHealthCheck(test)

		if hcErr.Err != "" {
			errors = append(errors, hcErr)
		}

		if hcErr.Exit {
			break
		}

	}
	return results, errors
}

// RunHealthCheck runs through a single healthcheck and saves the result
func RunHealthCheck(healthCheck SQLHealthCheck, cxn *sql.DB) SQLHealthCheck {
	answer := ""

	rows, err := cxn.Query(healthCheck.Query)

	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	rows.Next()
	rows.Scan(&answer)

	healthCheck.Passed = healthCheck.Expected == answer
	healthCheck.Actual = answer
	return healthCheck
}

// EvaluateHealthCheck runs through a single healthcheck and acts on the result
func EvaluateHealthCheck(healthCheck SQLHealthCheck) (err HCError) {

	prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
	if !healthCheck.Passed {
		switch strings.ToLower(healthCheck.Severity) {

		// When Fatal, return early with an error
		case "fatal":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("FATAL healthcheck failed\nBreaking Away Early\n%s\n\n", string(prettyHealthCheck))
			err = HCError{"FATAL healthCheck failure", true}

		// When Error, keep running but add an error
		case "error":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("%s healthcheck failed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
			err = HCError{"ERROR healthCheck failure", false}

		// When warn or info, print out the result and keep running
		case "warn", "info":
			prettyHealthCheck, _ := yaml.Marshal(&healthCheck)
			log.Printf("%s healthcheck failed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
		default:
			log.Printf("undefined severity level:%s\n%s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
		}
	} else {
		log.Printf("%s healthcheck passed\n %s\n\n", strings.ToUpper(healthCheck.Severity), string(prettyHealthCheck))
	}

	return err

}

// Implementation of report.Element

// HealthCheckReportHeaders headers used for GetHeaders
var HealthCheckReportHeaders = []string{"Title", "Query", "Passed", "Expected", "Actual"}

// GetHeaders Implementation for report.Element
func (hcr SQLHealthCheck) GetHeaders() []string {
	return HealthCheckReportHeaders[0:]
}

// GetValue Implementation for report.Element
func (hcr SQLHealthCheck) GetValue(key string) string {

	switch key {
	case HealthCheckReportHeaders[0]:
		return hcr.Title
	case HealthCheckReportHeaders[1]:
		return hcr.Query
	case HealthCheckReportHeaders[2]:
		if hcr.Passed {
			return "SUCCESS"
		}
		return "FAIL"
	case HealthCheckReportHeaders[3]:
		return hcr.Expected
	case HealthCheckReportHeaders[4]:
		return hcr.Actual
	}
	return ""
}
<|endoftext|>