<commit_before>package resource

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"

	"github.com/qor/qor"
	"github.com/qor/qor/utils"
)

func convertMapToMetaValues(values map[string]interface{}, metaors []Metaor) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorMap := make(map[string]Metaor)
	for _, metaor := range metaors {
		metaorMap[metaor.GetName()] = metaor
	}

	for key, value := range values {
		var metaValue *MetaValue
		metaor := metaorMap[key]
		var childMeta []Metaor
		if metaor != nil {
			childMeta = metaor.GetMetas()
		}

		switch result := value.(type) {
		case map[string]interface{}:
			if children, err := convertMapToMetaValues(result, childMeta); err == nil {
				metaValue = &MetaValue{Name: key, Meta: metaor, MetaValues: children}
			}
		case []interface{}:
			for idx, r := range result {
				if mr, ok := r.(map[string]interface{}); ok {
					if children, err := convertMapToMetaValues(mr, childMeta); err == nil {
						metaValue := &MetaValue{Name: key, Meta: metaor, MetaValues: children, Index: idx}
						metaValues.Values = append(metaValues.Values, metaValue)
					}
				} else {
					metaValue := &MetaValue{Name: key, Value: result, Meta: metaor}
					metaValues.Values = append(metaValues.Values, metaValue)
					break
				}
			}
		default:
			metaValue = &MetaValue{Name: key, Value: value, Meta: metaor}
		}

		if metaValue != nil {
			metaValues.Values = append(metaValues.Values, metaValue)
		}
	}
	return metaValues, nil
}

// ConvertJSONToMetaValues convert json to meta values
func ConvertJSONToMetaValues(reader io.Reader, metaors []Metaor) (*MetaValues, error) {
	var (
		err     error
		values  = map[string]interface{}{}
		decoder = json.NewDecoder(reader)
	)

	if err = decoder.Decode(&values); err == nil {
		return convertMapToMetaValues(values, metaors)
	}
	return nil, err
}

var (
	isCurrentLevel = regexp.MustCompile("^[^.]+$")
	isNextLevel    = regexp.MustCompile(`^(([^.\[\]]+)(\[\d+\])?)(?:(\.[^.]+)+)$`)
)

// ConvertFormToMetaValues convert form to meta values
func ConvertFormToMetaValues(request *http.Request, metaors []Metaor, prefix string) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorsMap := map[string]Metaor{}
	convertedNextLevel := map[string]bool{}
	nestedStructIndex := map[string]int{}
	for _, metaor := range metaors {
		metaorsMap[metaor.GetName()] = metaor
	}

	newMetaValue := func(key string, value interface{}) {
		if strings.HasPrefix(key, prefix) {
			var metaValue *MetaValue
			key = strings.TrimPrefix(key, prefix)

			if matches := isCurrentLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[0]
				metaValue = &MetaValue{Name: name, Value: value, Meta: metaorsMap[name]}
			} else if matches := isNextLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[1]
				if _, ok := convertedNextLevel[name]; !ok {
					var metaors []Metaor
					convertedNextLevel[name] = true
					metaor := metaorsMap[matches[2]]
					if metaor != nil {
						metaors = metaor.GetMetas()
					}

					if children, err := ConvertFormToMetaValues(request, metaors, prefix+name+"."); err == nil {
						nestedName := prefix + matches[2]
						if _, ok := nestedStructIndex[nestedName]; ok {
							nestedStructIndex[nestedName]++
						} else {
							nestedStructIndex[nestedName] = 0
						}
						metaValue = &MetaValue{Name: matches[2], Meta: metaor, MetaValues: children, Index: nestedStructIndex[nestedName]}
					}
				}
			}

			if metaValue != nil {
				metaValues.Values = append(metaValues.Values, metaValue)
			}
		}
	}

	var sortedFormKeys []string
	for key := range request.Form {
		sortedFormKeys = append(sortedFormKeys, key)
	}

	utils.SortFormKeys(sortedFormKeys)

	fmt.Println("sorted form keys....")
	fmt.Println(sortedFormKeys)

	for _, key := range sortedFormKeys {
		newMetaValue(key, request.Form[key])
	}

	if request.MultipartForm != nil {
		sortedFormKeys = []string{}
		for key := range request.MultipartForm.File {
			sortedFormKeys = append(sortedFormKeys, key)
		}
		utils.SortFormKeys(sortedFormKeys)

		for _, key := range sortedFormKeys {
			newMetaValue(key, request.MultipartForm.File[key])
		}
	}
	return metaValues, nil
}

// Decode decode context to result according to resource definition
func Decode(context *qor.Context, result interface{}, res Resourcer) error {
	var errors qor.Errors
	var err error
	var metaValues *MetaValues
	metaors := res.GetMetas([]string{})

	if strings.Contains(context.Request.Header.Get("Content-Type"), "json") {
		metaValues, err = ConvertJSONToMetaValues(context.Request.Body, metaors)
		context.Request.Body.Close()
	} else {
		metaValues, err = ConvertFormToMetaValues(context.Request, metaors, "QorResource.")
	}

	errors.AddError(err)
	errors.AddError(DecodeToResource(res, result, metaValues, context).Start())
	return errors
}
<commit_msg>Remove debug code<commit_after>package resource

import (
	"encoding/json"
	"io"
	"net/http"
	"regexp"
	"strings"

	"github.com/qor/qor"
	"github.com/qor/qor/utils"
)

func convertMapToMetaValues(values map[string]interface{}, metaors []Metaor) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorMap := make(map[string]Metaor)
	for _, metaor := range metaors {
		metaorMap[metaor.GetName()] = metaor
	}

	for key, value := range values {
		var metaValue *MetaValue
		metaor := metaorMap[key]
		var childMeta []Metaor
		if metaor != nil {
			childMeta = metaor.GetMetas()
		}

		switch result := value.(type) {
		case map[string]interface{}:
			if children, err := convertMapToMetaValues(result, childMeta); err == nil {
				metaValue = &MetaValue{Name: key, Meta: metaor, MetaValues: children}
			}
		case []interface{}:
			for idx, r := range result {
				if mr, ok := r.(map[string]interface{}); ok {
					if children, err := convertMapToMetaValues(mr, childMeta); err == nil {
						metaValue := &MetaValue{Name: key, Meta: metaor, MetaValues: children, Index: idx}
						metaValues.Values = append(metaValues.Values, metaValue)
					}
				} else {
					metaValue := &MetaValue{Name: key, Value: result, Meta: metaor}
					metaValues.Values = append(metaValues.Values, metaValue)
					break
				}
			}
		default:
			metaValue = &MetaValue{Name: key, Value: value, Meta: metaor}
		}

		if metaValue != nil {
			metaValues.Values = append(metaValues.Values, metaValue)
		}
	}
	return metaValues, nil
}

// ConvertJSONToMetaValues convert json to meta values
func ConvertJSONToMetaValues(reader io.Reader, metaors []Metaor) (*MetaValues, error) {
	var (
		err     error
		values  = map[string]interface{}{}
		decoder = json.NewDecoder(reader)
	)

	if err = decoder.Decode(&values); err == nil {
		return convertMapToMetaValues(values, metaors)
	}
	return nil, err
}

var (
	isCurrentLevel = regexp.MustCompile("^[^.]+$")
	isNextLevel    = regexp.MustCompile(`^(([^.\[\]]+)(\[\d+\])?)(?:(\.[^.]+)+)$`)
)

// ConvertFormToMetaValues convert form to meta values
func ConvertFormToMetaValues(request *http.Request, metaors []Metaor, prefix string) (*MetaValues, error) {
	metaValues := &MetaValues{}
	metaorsMap := map[string]Metaor{}
	convertedNextLevel := map[string]bool{}
	nestedStructIndex := map[string]int{}
	for _, metaor := range metaors {
		metaorsMap[metaor.GetName()] = metaor
	}

	newMetaValue := func(key string, value interface{}) {
		if strings.HasPrefix(key, prefix) {
			var metaValue *MetaValue
			key = strings.TrimPrefix(key, prefix)

			if matches := isCurrentLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[0]
				metaValue = &MetaValue{Name: name, Value: value, Meta: metaorsMap[name]}
			} else if matches := isNextLevel.FindStringSubmatch(key); len(matches) > 0 {
				name := matches[1]
				if _, ok := convertedNextLevel[name]; !ok {
					var metaors []Metaor
					convertedNextLevel[name] = true
					metaor := metaorsMap[matches[2]]
					if metaor != nil {
						metaors = metaor.GetMetas()
					}

					if children, err := ConvertFormToMetaValues(request, metaors, prefix+name+"."); err == nil {
						nestedName := prefix + matches[2]
						if _, ok := nestedStructIndex[nestedName]; ok {
							nestedStructIndex[nestedName]++
						} else {
							nestedStructIndex[nestedName] = 0
						}
						metaValue = &MetaValue{Name: matches[2], Meta: metaor, MetaValues: children, Index: nestedStructIndex[nestedName]}
					}
				}
			}

			if metaValue != nil {
				metaValues.Values = append(metaValues.Values, metaValue)
			}
		}
	}

	var sortedFormKeys []string
	for key := range request.Form {
		sortedFormKeys = append(sortedFormKeys, key)
	}

	utils.SortFormKeys(sortedFormKeys)

	for _, key := range sortedFormKeys {
		newMetaValue(key, request.Form[key])
	}

	if request.MultipartForm != nil {
		sortedFormKeys = []string{}
		for key := range request.MultipartForm.File {
			sortedFormKeys = append(sortedFormKeys, key)
		}
		utils.SortFormKeys(sortedFormKeys)

		for _, key := range sortedFormKeys {
			newMetaValue(key, request.MultipartForm.File[key])
		}
	}
	return metaValues, nil
}

// Decode decode context to result according to resource definition
func Decode(context *qor.Context, result interface{}, res Resourcer) error {
	var errors qor.Errors
	var err error
	var metaValues *MetaValues
	metaors := res.GetMetas([]string{})

	if strings.Contains(context.Request.Header.Get("Content-Type"), "json") {
		metaValues, err = ConvertJSONToMetaValues(context.Request.Body, metaors)
		context.Request.Body.Close()
	} else {
		metaValues, err = ConvertFormToMetaValues(context.Request, metaors, "QorResource.")
	}

	errors.AddError(err)
	errors.AddError(DecodeToResource(res, result, metaValues, context).Start())
	return errors
}
<|endoftext|>


<commit_before>// Copyright 2020, Square, Inc.

package linter

import (
	"fmt"
	"os"
	"strings"

	"github.com/alexflint/go-arg"
	"github.com/logrusorgru/aurora"

	"github.com/square/spincycle/v2/request-manager/graph"
	"github.com/square/spincycle/v2/request-manager/id"
	"github.com/square/spincycle/v2/request-manager/spec"
	v "github.com/square/spincycle/v2/version"
)

// Note that go-arg help message will show defaults if the default is not false.
type Linter struct {
	SpecsDir string `arg:"positional" help:"path to spin cycle requests directory [default: current working dir]"`

	Strict   bool `arg:"-s, --strict" help:"make all warnings into errors [default: false]"`
	Warnings bool `help:"turn warnings on or off"`
	Color    bool `help:"turn color on or off"`

	Sequences string `help:"comma-separated list of sequences for which to output info; sequence must exist in specs [default: all]"`

	errorStr   string `arg:"-"`
	warningStr string `arg:"-"`
	count      int    `arg:"-"` // warning + error counter
	anyWarning bool   `arg:"-"` // whether any warnings we would output with --warnings=true occurred
}

var splitter = "# ------------------------------------------------------------------------------"

func (linter *Linter) Version() string {
	return "linter " + v.Version()
}

func Run() bool {
	// 1. Setup
	linter := Linter{
		Strict:   false,
		Warnings: true,
		Color:    true,
		SpecsDir: "./",

		count:      1,
		anyWarning: false,
	}
	arg.MustParse(&linter)

	var sequences []string
	if len(linter.Sequences) != 0 {
		sequences = strings.Split(linter.Sequences, ",")
	}
	// else we want to print all sequences, but we don't know what they are
	// yet, so we can't fill out `sequences` properly yet

	color := aurora.NewAurora(linter.Color)
	linter.errorStr = fmt.Sprintf("%s", color.Red("Errors"))
	linter.warningStr = fmt.Sprintf("%s", color.Yellow("Warnings"))

	// 2. Parsing and static parse checks
	allSpecs, fileResults, err := spec.ParseSpecsDir(linter.SpecsDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		return false
	}
	if fileResults.AnyError {
		for file, result := range fileResults.Results {
			header := splitter + "\n" + fmt.Sprintf("# File: %s\n", file)
			linter.printCheckResult(header, result)
		}
		return false
	}
	// Check that we did indeed read some specs.
	if len(allSpecs.Sequences) == 0 {
		fmt.Fprintf(os.Stderr, "%s\n", color.Red("No specs found in directory"))
		return false
	}
	// Check that all sequences provided by --sequences arg are actually in
	// the specs.
	for _, seq := range sequences {
		missing := []string{}
		if _, ok := allSpecs.Sequences[seq]; !ok {
			missing = append(missing, seq)
		}
		if len(missing) > 0 {
			fmt.Fprintf(os.Stderr, "Args to --sequences not found in specs: %s\n", strings.Join(missing, ", "))
			return false
		}
	}
	// If no --sequences were specified, then we'll output all sequences. Update
	// `sequences` to reflect that.
	if len(sequences) == 0 {
		sequences = make([]string, 0, len(allSpecs.Sequences))
		for seq, _ := range allSpecs.Sequences {
			sequences = append(sequences, seq)
		}
	}
	// There may be other warnings/errors from later checks, but those will
	// be grouped by seqence. These can't be, so we'll just print them all
	// together now.
	if fileResults.AnyWarning {
		linter.anyWarning = true

		if linter.Warnings {
			warnings := []error{}
			// There should only be one warning per file right now
			for file, result := range fileResults.Results {
				for _, warn := range result.Warnings {
					warnings = append(warnings, fmt.Errorf("%s: %s", file, warn))
				}
			}

			fmt.Println(splitter)
			fmt.Print(linter.fmtList("# Syntax warnings\n", warnings))
		}
	}
	spec.ProcessSpecs(&allSpecs)

	// 3. Static checks
	checkFactories := []spec.CheckFactory{spec.DefaultCheckFactory{allSpecs}, spec.BaseCheckFactory{allSpecs}}
	checker, err := spec.NewChecker(checkFactories)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		return false
	}
	seqResults := checker.RunChecks(allSpecs)
	if seqResults.AnyError {
		errorPrinted := false // whether we printed anything
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			errorPrinted = linter.printCheckResult(header, seqResults.Results[seq]) || errorPrinted
		}
		if !errorPrinted {
			fmt.Printf("%s\n", color.Red("Sequence not listed in --sequences failed static check; fix errors to perform graph checks"))
		}
		return false
	}
	// We'll print warnings later along with graph errors, if any.

	// 4. Graph checks
	idgen := id.NewGeneratorFactory(4, 100)
	gr := graph.NewGrapher(allSpecs, idgen)
	_, graphResults := gr.CheckSequences()
	seqResults.Union(graphResults)
	if seqResults.AnyError {
		errorPrinted := false // whether we printed anything
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			errorPrinted = linter.printCheckResult(header, seqResults.Results[seq]) || errorPrinted
		}
		if errorPrinted {
			return false
		}
	} else if seqResults.AnyWarning {
		// No errors occurred; print all warnings now.
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			linter.printCheckResult(header, seqResults.Results[seq])
		}
	}

	// 5. No errors occurred, and we've printed all warnings. Decide whether
	// we need to print anything else, then exit.
	if linter.anyWarning && !linter.Warnings {
		// Case: warnings occurred, but were surpressed
		fmt.Println(color.Yellow("Warnings occurred and suppressed"))
	} else if !linter.anyWarning {
		// Case: no warnings and no errors
		fmt.Println(color.Green("OK, all specs are valid"))
	} // else Case: some warnings printed, no errors

	if linter.Strict {
		return !linter.anyWarning
	}
	return true
}

func fmtHeader(seqName string, allSpecs spec.Specs) (string, error) {
	seqSpec, ok := allSpecs.Sequences[seqName]
	if !ok {
		return "", fmt.Errorf("linter CLI: could not find specs for sequence %s", seqName)
	}

	return splitter + "\n" +
		fmt.Sprintf("# File: %s\n", seqSpec.Filename) +
		fmt.Sprintf("#  Seq: %s\n", seqName), nil
}

func (linter *Linter) fmtList(header string, list []error) string {
	if len(list) == 0 {
		return ""
	}
	str := fmt.Sprintf("%s\n", header)
	for _, elt := range list {
		str += fmt.Sprintf("%4d: %s\n\n", linter.count, elt)
		linter.count++
	}
	return str
}

func (linter *Linter) printCheckResult(header string, result *spec.CheckResult) bool {
	if result == nil {
		return false
	}
	toPrint := linter.fmtList(linter.errorStr, result.Errors)
	if len(result.Warnings) != 0 {
		linter.anyWarning = true
	}
	if linter.Warnings {
		toPrint += linter.fmtList(linter.warningStr, result.Warnings)
	}
	if len(toPrint) == 0 {
		return false
	}
	fmt.Println(header)
	fmt.Print(toPrint)
	return true
}
<commit_msg>Fix bug in linter CLI<commit_after>// Copyright 2020, Square, Inc.

package linter

import (
	"fmt"
	"os"
	"strings"

	"github.com/alexflint/go-arg"
	"github.com/logrusorgru/aurora"

	"github.com/square/spincycle/v2/request-manager/graph"
	"github.com/square/spincycle/v2/request-manager/id"
	"github.com/square/spincycle/v2/request-manager/spec"
	v "github.com/square/spincycle/v2/version"
)

// Note that go-arg help message will show defaults if the default is not false.
type Linter struct {
	SpecsDir string `arg:"positional" help:"path to spin cycle requests directory [default: current working dir]"`

	Strict   bool `arg:"-s, --strict" help:"make all warnings into errors [default: false]"`
	Warnings bool `help:"turn warnings on or off"`
	Color    bool `help:"turn color on or off"`

	Sequences string `help:"comma-separated list of sequences for which to output info; sequence must exist in specs [default: all]"`

	errorStr   string `arg:"-"`
	warningStr string `arg:"-"`
	count      int    `arg:"-"` // warning + error counter
	anyWarning bool   `arg:"-"` // whether any warnings we would output with --warnings=true occurred
}

var splitter = "# ------------------------------------------------------------------------------"

func (linter *Linter) Version() string {
	return "linter " + v.Version()
}

func Run() bool {
	// 1. Setup
	linter := Linter{
		Strict:   false,
		Warnings: true,
		Color:    true,
		SpecsDir: "./",

		count:      1,
		anyWarning: false,
	}
	arg.MustParse(&linter)

	var sequences []string
	if len(linter.Sequences) != 0 {
		sequences = strings.Split(linter.Sequences, ",")
	}
	// else we want to print all sequences, but we don't know what they are
	// yet, so we can't fill out `sequences` properly yet

	color := aurora.NewAurora(linter.Color)
	linter.errorStr = fmt.Sprintf("%s", color.Red("Errors"))
	linter.warningStr = fmt.Sprintf("%s", color.Yellow("Warnings"))

	// 2. Parsing and static parse checks
	allSpecs, fileResults, err := spec.ParseSpecsDir(linter.SpecsDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		return false
	}
	if fileResults.AnyError {
		for file, result := range fileResults.Results {
			header := splitter + "\n" + fmt.Sprintf("# File: %s\n", file)
			linter.printCheckResult(header, result)
		}
		return false
	}
	// Check that we did indeed read some specs.
	if len(allSpecs.Sequences) == 0 {
		fmt.Fprintf(os.Stderr, "%s\n", color.Red("No specs found in directory"))
		return false
	}
	// Check that all sequences provided by --sequences arg are actually in
	// the specs.
	for _, seq := range sequences {
		missing := []string{}
		if _, ok := allSpecs.Sequences[seq]; !ok {
			missing = append(missing, seq)
		}
		if len(missing) > 0 {
			fmt.Fprintf(os.Stderr, "Args to --sequences not found in specs: %s\n", strings.Join(missing, ", "))
			return false
		}
	}
	// If no --sequences were specified, then we'll output all sequences. Update
	// `sequences` to reflect that.
	if len(sequences) == 0 {
		sequences = make([]string, 0, len(allSpecs.Sequences))
		for seq, _ := range allSpecs.Sequences {
			sequences = append(sequences, seq)
		}
	}
	// There may be other warnings/errors from later checks, but those will
	// be grouped by seqence. These can't be, so we'll just print them all
	// together now.
	if fileResults.AnyWarning {
		linter.anyWarning = true

		if linter.Warnings {
			warnings := []error{}
			// There should only be one warning per file right now
			for file, result := range fileResults.Results {
				for _, warn := range result.Warnings {
					warnings = append(warnings, fmt.Errorf("%s: %s", file, warn))
				}
			}

			fmt.Println(splitter)
			fmt.Print(linter.fmtList("# Syntax warnings\n", warnings))
		}
	}
	spec.ProcessSpecs(&allSpecs)

	// 3. Static checks
	checkFactories := []spec.CheckFactory{spec.DefaultCheckFactory{allSpecs}, spec.BaseCheckFactory{allSpecs}}
	checker, err := spec.NewChecker(checkFactories)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		return false
	}
	seqResults := checker.RunChecks(allSpecs)
	if seqResults.AnyError {
		errorPrinted := false // whether we printed anything
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			errorPrinted = linter.printCheckResult(header, seqResults.Results[seq]) || errorPrinted
		}
		if !errorPrinted {
			fmt.Printf("%s\n", color.Red("Sequence not listed in --sequences failed static check; fix errors to perform graph checks"))
		}
		return false
	}
	// We'll print warnings later along with graph errors, if any.

	// 4. Graph checks
	idgen := id.NewGeneratorFactory(4, 100)
	gr := graph.NewGrapher(allSpecs, idgen)
	_, graphResults := gr.CheckSequences()
	seqResults.Union(graphResults)
	if seqResults.AnyError {
		errorPrinted := false // whether we printed anything
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			errorPrinted = linter.printCheckResult(header, seqResults.Results[seq]) || errorPrinted
		}
		if errorPrinted {
			return false
		}
	}

	// 5. No errors occurred. Print warnings and decide whether we need to
	// print anything else, then exit.
	if seqResults.AnyWarning {
		// No errors occurred; print all warnings now.
		for _, seq := range sequences {
			header, err := fmtHeader(seq, allSpecs)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s", err)
			}
			linter.printCheckResult(header, seqResults.Results[seq])
		}
	}

	if linter.anyWarning && !linter.Warnings {
		// Case: warnings occurred, but were surpressed
		fmt.Println(color.Yellow("Warnings occurred and suppressed"))
	} else if !linter.anyWarning {
		// Case: no warnings and no errors
		fmt.Println(color.Green("OK, all specs are valid"))
	} // else Case: some warnings printed, no errors

	if linter.Strict {
		return !linter.anyWarning
	}
	return true
}

func fmtHeader(seqName string, allSpecs spec.Specs) (string, error) {
	seqSpec, ok := allSpecs.Sequences[seqName]
	if !ok {
		return "", fmt.Errorf("linter CLI: could not find specs for sequence %s", seqName)
	}

	return splitter + "\n" +
		fmt.Sprintf("# File: %s\n", seqSpec.Filename) +
		fmt.Sprintf("#  Seq: %s\n", seqName), nil
}

func (linter *Linter) fmtList(header string, list []error) string {
	if len(list) == 0 {
		return ""
	}
	str := fmt.Sprintf("%s\n", header)
	for _, elt := range list {
		str += fmt.Sprintf("%4d: %s\n\n", linter.count, elt)
		linter.count++
	}
	return str
}

func (linter *Linter) printCheckResult(header string, result *spec.CheckResult) (didPrint bool) {
	if result == nil {
		return false
	}
	toPrint := linter.fmtList(linter.errorStr, result.Errors)
	if len(result.Warnings) != 0 {
		linter.anyWarning = true
	}
	if linter.Warnings {
		toPrint += linter.fmtList(linter.warningStr, result.Warnings)
	}
	if len(toPrint) == 0 {
		return false
	}
	fmt.Println(header)
	fmt.Print(toPrint)
	return true
}
<|endoftext|>


<commit_before>package rbtree

import (
	"fmt"
)

type Node struct {
	lchild *Node
	rchild *Node
	parent *Node
	k      int
	black  bool
}

type Tree struct {
	root *Node
}

func leftRotation(t *Tree, n *Node) {
	if t == nil || n == nil {
		return
	}
	y := n.rchild

	n.rchild = y.lchild
	if y.lchild != nil {
		y.lchild.parent = n
	}

	y.parent = n.parent
	if n.parent != nil {
		if n.k < n.parent.k {
			n.parent.lchild = y
		} else {
			n.parent.rchild = y
		}
	}

	y.lchild = n
	n.parent = y

	if y.parent == nil {
		t.root = y
	}
}

func rightRotation(t *Tree, n *Node) {
	if t == nil || n == nil {
		return
	}

	y := n.lchild

	n.lchild = y.rchild
	if y.rchild != nil {
		y.rchild.parent = n
	}

	y.parent = n.parent
	if n.parent != nil {
		if n.k < n.parent.k {
			n.parent.lchild = y
		} else {
			n.parent.rchild = y
		}
	}

	y.rchild = n
	n.parent = y

	if y.parent == nil {
		t.root = y
	}
}

func checkNodeRed(n *Node) bool {
	return n != nil && !n.black
}

func checkNodeBlack(n *Node) bool {
	return n == nil || n.black
}

func insertFixup(t *Tree, n *Node) {

	for n != nil {
		p := n.parent
		if p == nil || p.black {
			break
		}
		pp := p.parent
		if pp == nil {
			break
		}

		// left node
		if p.k < pp.k {
			ppr := pp.rchild
			if checkNodeRed(ppr) {
				ppr.black = true
				p.black = true
				pp.black = false
				n = pp
			} else {
				// make insert right child become insert left child via left rotation
				if p.k < n.k {
					leftRotation(t, p)
					n = p
				}
				n.parent.black = true
				pp.black = false
				rightRotation(t, pp)
			}
		} else {
			ppl := pp.lchild
			if checkNodeRed(ppl) {
				ppl.black = true
				p.black = true
				pp.black = false
				n = pp
			} else {
				// make insert left child become insert right child via right rotation
				if n.k < p.k {
					rightRotation(t, n)
					n = p
				}

				n.parent.black = true
				pp.black = false
				leftRotation(t, pp)
			}
		}
	}

	if t.root != nil {
		t.root.black = true
	}
}

func (t *Tree) AddElement(k int) {
	if t == nil {
		return
	}

	n := t.root
	p := n

	for n != nil {
		p = n
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			return
		}
	}

	n = &Node{k: k}
	if p == nil {
		t.root = n
	} else {
		if p.k < n.k {
			p.rchild = n
		} else {
			p.lchild = n
		}
		n.parent = p
	}

	insertFixup(t, n)
}

func getMaxOne(n *Node) *Node {
	if n != nil {
		for n.rchild != nil {
			n = n.rchild
		}
	}

	return n
}

func transplant(t *Tree, from *Node, to *Node) {
	if from.parent == nil {
		t.root = to
	} else if from.parent.k < from.k {
		from.parent.rchild = to
	} else if from.k < from.parent.k {
		from.parent.lchild = to
	}

	if to != nil {
		to.parent = from.parent
	}
}

func clearTmp(n *Node) {
	if n == nil || n.parent == nil {
		return
	}
	if n == n.parent.lchild {
		n.parent.lchild = nil
	} else if n == n.parent.rchild {
		n.parent.rchild = nil
	}
}

func removeFixup(t *Tree, n *Node, use_tmp bool) {

	for n != t.root && n.black {
		// left node
		if n == n.parent.lchild {
			r := n.parent.rchild
			if r != nil && !r.black {
				r.black = true
				n.parent.black = false
				leftRotation(t, n.parent)
				r = n.parent.rchild
			}
			if r != nil && checkNodeBlack(r.lchild) && checkNodeBlack(r.rchild) {
				r.black = false
				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}
				n = n.parent
			} else if r != nil {
				if checkNodeBlack(r.rchild) {
					r.lchild.black = true
					r.black = false
					rightRotation(t, r)
					r = n.parent.rchild
				}

				r.rchild.black = true
				r.black = n.parent.black
				n.parent.black = true
				leftRotation(t, n.parent)

				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}

				n = t.root
			}
		} else {
			l := n.parent.lchild
			if l != nil && !l.black {
				l.black = true
				n.parent.black = false
				rightRotation(t, n.parent)
				l = n.parent.lchild
			}

			if l != nil && checkNodeBlack(l.lchild) && checkNodeBlack(l.rchild) {
				l.black = false
				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}
				n = n.parent
			} else if l != nil {
				if checkNodeBlack(l.lchild) {
					l.rchild.black = true
					l.black = false
					leftRotation(t, l)
					l = n.parent.lchild
				}

				l.lchild.black = true
				l.black = n.parent.black
				n.parent.black = true
				rightRotation(t, n.parent)

				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}

				n = t.root
			}
		}
	}

	if n != nil {
		n.black = true
	}
}

func (t *Tree) RemoveElement(k int) {
	if t == nil {
		return
	}

	n := t.root

	for n != nil {
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			break
		}
	}

	if n == nil {
		return
	}

	var tmp = &Node{black: true}
	use_tmp := false
	isblack := n.black

	var replaceone *Node
	if n.lchild == nil {
		replaceone = n.rchild
		if replaceone == nil {
			replaceone = tmp
			use_tmp = true
		}
		transplant(t, n, replaceone)
	} else if n.rchild == nil {
		replaceone = n.lchild
		transplant(t, n, replaceone)
	} else {
		one := getMaxOne(n.lchild)
		isblack = one.black

		replaceone = one.lchild
		if replaceone == nil {
			replaceone = tmp
			use_tmp = true
		}

		if one.parent == n {
			replaceone.parent = n
		} else {
			transplant(t, one, replaceone)
			one.lchild = n.lchild
			one.lchild.parent = one
		}

		transplant(t, n, one)
		one.rchild = n.rchild
		one.rchild.parent = one
		one.black = n.black
	}

	if isblack {
		removeFixup(t, replaceone, use_tmp)
	} else if use_tmp {
		clearTmp(replaceone)
	}
}

func (t *Tree) SearchElement(k int) bool {
	if t == nil {
		return false
	}

	n := t.root
	for n != nil {
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			return true
		}
	}

	return false
}

func (t *Tree) FreeTree() {
	if t != nil {
		t.root = nil
	}
}

func traversePreorder(n *Node) {
	if n == nil {
		return
	}

	fmt.Printf("%v(%t) ", n.k, n.black)
	traversePreorder(n.lchild)
	traversePreorder(n.rchild)
}

func (t *Tree) TraversePreorder() {
	if t == nil {
		return
	}
	traversePreorder(t.root)
}

func traverseInorder(n *Node) {
	if n == nil {
		return
	}

	traverseInorder(n.lchild)
	fmt.Printf("%v(%t) ", n.k, n.black)
	traverseInorder(n.rchild)
}

func (t *Tree) TraverseInorder() {
	if t == nil {
		return
	}
	traverseInorder(t.root)
}

func traversePostorder(n *Node) {
	if n == nil {
		return
	}

	traversePostorder(n.lchild)
	traversePostorder(n.rchild)
	fmt.Printf("%v(%t) ", n.k, n.black)
}

func (t *Tree) TraversePostorder() {
	if t == nil {
		return
	}
	traversePostorder(t.root)
}
<commit_msg>- modify redblacktree go version<commit_after>package rbtree

import (
	"fmt"
)

type Node struct {
	lchild *Node
	rchild *Node
	parent *Node
	k      int
	black  bool
}

type Tree struct {
	root *Node
}

func leftRotation(t *Tree, n *Node) {
	if t == nil || n == nil {
		return
	}
	y := n.rchild

	n.rchild = y.lchild
	if y.lchild != nil {
		y.lchild.parent = n
	}

	y.parent = n.parent
	if n.parent != nil {
		if n.k < n.parent.k {
			n.parent.lchild = y
		} else {
			n.parent.rchild = y
		}
	}

	y.lchild = n
	n.parent = y

	if y.parent == nil {
		t.root = y
	}
}

func rightRotation(t *Tree, n *Node) {
	if t == nil || n == nil {
		return
	}

	y := n.lchild

	n.lchild = y.rchild
	if y.rchild != nil {
		y.rchild.parent = n
	}

	y.parent = n.parent
	if n.parent != nil {
		if n.k < n.parent.k {
			n.parent.lchild = y
		} else {
			n.parent.rchild = y
		}
	}

	y.rchild = n
	n.parent = y

	if y.parent == nil {
		t.root = y
	}
}

func checkNodeRed(n *Node) bool {
	return n != nil && !n.black
}

func checkNodeBlack(n *Node) bool {
	return n == nil || n.black
}

func insertFixup(t *Tree, n *Node) {

	for n != nil {
		p := n.parent
		if p == nil || p.black {
			break
		}
		pp := p.parent
		if pp == nil {
			break
		}

		// left node
		if p.k < pp.k {
			ppr := pp.rchild
			if checkNodeRed(ppr) {
				ppr.black = true
				p.black = true
				pp.black = false
				n = pp
			} else {
				// make insert right child become insert left child via left rotation
				if p.k < n.k {
					leftRotation(t, p)
					n = p
				}
				n.parent.black = true
				pp.black = false
				rightRotation(t, pp)
			}
		} else {
			ppl := pp.lchild
			if checkNodeRed(ppl) {
				ppl.black = true
				p.black = true
				pp.black = false
				n = pp
			} else {
				// make insert left child become insert right child via right rotation
				if n.k < p.k {
					rightRotation(t, n)
					n = p
				}

				n.parent.black = true
				pp.black = false
				leftRotation(t, pp)
			}
		}
	}

	if t.root != nil {
		t.root.black = true
	}
}

func (t *Tree) AddElement(k int) {
	if t == nil {
		return
	}

	n := t.root
	p := n

	for n != nil {
		p = n
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			return
		}
	}

	n = &Node{k: k}
	if p == nil {
		t.root = n
	} else {
		if p.k < n.k {
			p.rchild = n
		} else {
			p.lchild = n
		}
		n.parent = p
	}

	insertFixup(t, n)
}

func getMaxOne(n *Node) *Node {
	if n != nil {
		for n.rchild != nil {
			n = n.rchild
		}
	}

	return n
}

func transplant(t *Tree, from *Node, to *Node) {
	if from.parent == nil {
		t.root = to
	} else if from.parent.k < from.k {
		from.parent.rchild = to
	} else if from.k < from.parent.k {
		from.parent.lchild = to
	}

	if to != nil {
		to.parent = from.parent
	}
}

func clearTmp(n *Node) {
	if n == nil || n.parent == nil {
		return
	}
	if n == n.parent.lchild {
		n.parent.lchild = nil
	} else if n == n.parent.rchild {
		n.parent.rchild = nil
	}
}

func removeFixup(t *Tree, n *Node, use_tmp bool) {

	for n != t.root && n.black {
		// left node
		if n == n.parent.lchild {
			r := n.parent.rchild
			if r != nil && !r.black {
				r.black = true
				n.parent.black = false
				leftRotation(t, n.parent)
				r = n.parent.rchild
			}
			if r != nil && checkNodeBlack(r.lchild) && checkNodeBlack(r.rchild) {
				r.black = false
				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}
				n = n.parent
			} else {
				if r != nil {
					if checkNodeBlack(r.rchild) {
						r.lchild.black = true
						r.black = false
						rightRotation(t, r)
						r = n.parent.rchild
					}
					if r.rchild != nil {
						r.rchild.black = true
					}
					r.black = n.parent.black
				}
				n.parent.black = true
				leftRotation(t, n.parent)

				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}

				n = t.root
			}
		} else {
			l := n.parent.lchild
			if l != nil && !l.black {
				l.black = true
				n.parent.black = false
				rightRotation(t, n.parent)
				l = n.parent.lchild
			}

			if l != nil && checkNodeBlack(l.lchild) && checkNodeBlack(l.rchild) {
				l.black = false
				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}
				n = n.parent
			} else if l != nil {
				if checkNodeBlack(l.lchild) {
					l.rchild.black = true
					l.black = false
					leftRotation(t, l)
					l = n.parent.lchild
				}

				l.lchild.black = true
				l.black = n.parent.black
				n.parent.black = true
				rightRotation(t, n.parent)

				if use_tmp {
					use_tmp = false
					clearTmp(n)
				}

				n = t.root
			}
		}
	}

	if n != nil {
		n.black = true
	}
}

func (t *Tree) RemoveElement(k int) {
	if t == nil {
		return
	}

	n := t.root

	for n != nil {
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			break
		}
	}

	if n == nil {
		return
	}

	var tmp = &Node{black: true}
	use_tmp := false
	isblack := n.black

	var replaceone *Node
	if n.lchild == nil {
		replaceone = n.rchild
		if replaceone == nil {
			replaceone = tmp
			use_tmp = true
		}
		transplant(t, n, replaceone)
	} else if n.rchild == nil {
		replaceone = n.lchild
		transplant(t, n, replaceone)
	} else {
		one := getMaxOne(n.lchild)
		isblack = one.black

		replaceone = one.lchild
		if replaceone == nil {
			replaceone = tmp
			use_tmp = true
		}

		if one.parent == n {
			replaceone.parent = n
		} else {
			transplant(t, one, replaceone)
			one.lchild = n.lchild
			one.lchild.parent = one
		}

		transplant(t, n, one)
		one.rchild = n.rchild
		one.rchild.parent = one
		one.black = n.black
	}

	if isblack {
		removeFixup(t, replaceone, use_tmp)
	} else if use_tmp {
		clearTmp(replaceone)
	}
}

func (t *Tree) SearchElement(k int) bool {
	if t == nil {
		return false
	}

	n := t.root
	for n != nil {
		if n.k < k {
			n = n.rchild
		} else if k < n.k {
			n = n.lchild
		} else {
			return true
		}
	}

	return false
}

func (t *Tree) FreeTree() {
	if t != nil {
		t.root = nil
	}
}

func traversePreorder(n *Node) {
	if n == nil {
		return
	}

	fmt.Printf("%v(%t) ", n.k, n.black)
	traversePreorder(n.lchild)
	traversePreorder(n.rchild)
}

func (t *Tree) TraversePreorder() {
	if t == nil {
		return
	}
	traversePreorder(t.root)
}

func traverseInorder(n *Node) {
	if n == nil {
		return
	}

	traverseInorder(n.lchild)
	fmt.Printf("%v(%t) ", n.k, n.black)
	traverseInorder(n.rchild)
}

func (t *Tree) TraverseInorder() {
	if t == nil {
		return
	}
	traverseInorder(t.root)
}

func traversePostorder(n *Node) {
	if n == nil {
		return
	}

	traversePostorder(n.lchild)
	traversePostorder(n.rchild)
	fmt.Printf("%v(%t) ", n.k, n.black)
}

func (t *Tree) TraversePostorder() {
	if t == nil {
		return
	}
	traversePostorder(t.root)
}
<|endoftext|>


<commit_before>package api

import (
	"io/ioutil"
	"testing"
)

func TestValidName(t *testing.T) {
	supposedBad := []string{"£", "-", "-a", "1", "",
		// 41 characters, more than 40
		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
	supposedGood := []string{"a", "abc-123", "a12345", "abcde", "AbCdE"}
	for _, bad := range supposedBad {
		if ValidName(bad) {
			t.Error(bad + " is not a valid volume/variant name, but it passed ValidName")
		}
	}
	for _, good := range supposedGood {
		if !ValidName(good) {
			t.Error(good + " is a valid volume/variant name, but it failed ValidName")
		}
	}
}

func TestSwitchVolume(t *testing.T) {
	currentVolume := "foo"
	basePath, err := ioutil.TempDir("", "switch")
	if err != nil {
		t.Errorf("Could not create TempDir: %s\n", err)
	}
	dvol := NewDvolAPI(basePath)
	err = dvol.SwitchVolume(currentVolume)
	if err != nil {
		t.Errorf("SwitchVolume failed: %s\n", err)
	}
	activeVolume, err := dvol.ActiveVolume()
	if err != nil {
		t.Error("Could not find ActiveVolume")
	}
	if activeVolume != "foo" {
		t.Errorf("%s is not equal to 'foo'", activeVolume)
	}
}
<commit_msg>Pass necessary argument in API tests<commit_after>package api

import (
	"io/ioutil"
	"testing"
)

func TestValidName(t *testing.T) {
	supposedBad := []string{"£", "-", "-a", "1", "",
		// 41 characters, more than 40
		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
	supposedGood := []string{"a", "abc-123", "a12345", "abcde", "AbCdE"}
	for _, bad := range supposedBad {
		if ValidName(bad) {
			t.Error(bad + " is not a valid volume/variant name, but it passed ValidName")
		}
	}
	for _, good := range supposedGood {
		if !ValidName(good) {
			t.Error(good + " is a valid volume/variant name, but it failed ValidName")
		}
	}
}

func TestSwitchVolume(t *testing.T) {
	currentVolume := "foo"
	basePath, err := ioutil.TempDir("", "switch")
	if err != nil {
		t.Errorf("Could not create TempDir: %s\n", err)
	}
	disableDockerIntegration := true
	dvol := NewDvolAPI(basePath, disableDockerIntegration)
	err = dvol.SwitchVolume(currentVolume)
	if err != nil {
		t.Errorf("SwitchVolume failed: %s\n", err)
	}
	activeVolume, err := dvol.ActiveVolume()
	if err != nil {
		t.Error("Could not find ActiveVolume")
	}
	if activeVolume != "foo" {
		t.Errorf("%s is not equal to 'foo'", activeVolume)
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/compose-spec/compose-go/types"
	"github.com/docker/cli/cli/registry/client"
	moby "github.com/docker/docker/api/types"
	"github.com/docker/docker/errdefs"
	"golang.org/x/sync/errgroup"

	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type downOp func() error

func (s *composeService) Down(ctx context.Context, projectName string, options api.DownOptions) error {
	return progress.Run(ctx, func(ctx context.Context) error {
		return s.down(ctx, strings.ToLower(projectName), options)
	})
}

func (s *composeService) down(ctx context.Context, projectName string, options api.DownOptions) error {
	w := progress.ContextWriter(ctx)
	resourceToRemove := false

	var containers Containers
	containers, err := s.getContainers(ctx, projectName, oneOffInclude, true)
	if err != nil {
		return err
	}

	project := options.Project
	if project == nil {
		project, err = s.getProjectWithResources(ctx, containers, projectName)
		if err != nil {
			return err
		}
	}

	if len(containers) > 0 {
		resourceToRemove = true
	}

	err = InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {
		serviceContainers := containers.filter(isService(service))
		err := s.removeContainers(ctx, w, serviceContainers, options.Timeout, options.Volumes)
		return err
	})
	if err != nil {
		return err
	}

	orphans := containers.filter(isNotService(project.ServiceNames()...))
	if options.RemoveOrphans && len(orphans) > 0 {
		err := s.removeContainers(ctx, w, orphans, options.Timeout, false)
		if err != nil {
			return err
		}
	}

	ops := s.ensureNetworksDown(ctx, project, w)

	if options.Images != "" {
		ops = append(ops, s.ensureImagesDown(ctx, projectName, options, w)...)
	}

	if options.Volumes {
		ops = append(ops, s.ensureVolumesDown(ctx, project, w)...)
	}

	if !resourceToRemove && len(ops) == 0 {
		w.Event(progress.NewEvent(projectName, progress.Done, "Warning: No resource found to remove"))
	}

	eg, _ := errgroup.WithContext(ctx)
	for _, op := range ops {
		eg.Go(op)
	}
	return eg.Wait()
}

func (s *composeService) ensureVolumesDown(ctx context.Context, project *types.Project, w progress.Writer) []downOp {
	var ops []downOp
	for _, vol := range project.Volumes {
		if vol.External.External {
			continue
		}
		volumeName := vol.Name
		ops = append(ops, func() error {
			return s.removeVolume(ctx, volumeName, w)
		})
	}
	return ops
}

func (s *composeService) ensureImagesDown(ctx context.Context, projectName string, options api.DownOptions, w progress.Writer) []downOp {
	var ops []downOp
	for image := range s.getServiceImages(options, projectName) {
		image := image
		ops = append(ops, func() error {
			return s.removeImage(ctx, image, w)
		})
	}
	return ops
}

func (s *composeService) ensureNetworksDown(ctx context.Context, project *types.Project, w progress.Writer) []downOp {
	var ops []downOp
	for _, n := range project.Networks {
		if n.External.External {
			continue
		}
		networkName := n.Name
		_, err := s.apiClient().NetworkInspect(ctx, networkName, moby.NetworkInspectOptions{})
		if client.IsNotFound(err) {
			return nil
		}

		ops = append(ops, func() error {
			return s.removeNetwork(ctx, networkName, w)
		})
	}
	return ops
}

func (s *composeService) getServiceImages(options api.DownOptions, projectName string) map[string]struct{} {
	images := map[string]struct{}{}
	for _, service := range options.Project.Services {
		image := service.Image
		if options.Images == "local" && image != "" {
			continue
		}
		if image == "" {
			image = getImageName(service, projectName)
		}
		images[image] = struct{}{}
	}
	return images
}

func (s *composeService) removeImage(ctx context.Context, image string, w progress.Writer) error {
	id := fmt.Sprintf("Image %s", image)
	w.Event(progress.NewEvent(id, progress.Working, "Removing"))
	_, err := s.apiClient().ImageRemove(ctx, image, moby.ImageRemoveOptions{})
	if err == nil {
		w.Event(progress.NewEvent(id, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(id, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) removeVolume(ctx context.Context, id string, w progress.Writer) error {
	resource := fmt.Sprintf("Volume %s", id)
	w.Event(progress.NewEvent(resource, progress.Working, "Removing"))
	err := s.apiClient().VolumeRemove(ctx, id, true)
	if err == nil {
		w.Event(progress.NewEvent(resource, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(resource, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) stopContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration) error {
	eg, ctx := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.apiClient().ContainerStop(ctx, container.ID, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.StoppedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration, volumes bool) error {
	eg, _ := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.stopContainers(ctx, w, []moby.Container{container}, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.RemovingEvent(eventName))
			err = s.apiClient().ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{
				Force:         true,
				RemoveVolumes: volumes,
			})
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Removing"))
				return err
			}
			w.Event(progress.RemovedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) getProjectWithResources(ctx context.Context, containers Containers, projectName string) (*types.Project, error) {
	containers = containers.filter(isNotOneOff)
	project, _ := s.projectFromName(containers, projectName)

	volumes, err := s.actualVolumes(ctx, projectName)
	if err != nil {
		return nil, err
	}
	project.Volumes = volumes

	networks, err := s.actualNetworks(ctx, projectName)
	if err != nil {
		return nil, err
	}
	project.Networks = networks
	return project, nil
}
<commit_msg>use project instead of DownOptions.project to list service images in pkg.compose.down<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/compose-spec/compose-go/types"
	"github.com/docker/cli/cli/registry/client"
	moby "github.com/docker/docker/api/types"
	"github.com/docker/docker/errdefs"
	"golang.org/x/sync/errgroup"

	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type downOp func() error

func (s *composeService) Down(ctx context.Context, projectName string, options api.DownOptions) error {
	return progress.Run(ctx, func(ctx context.Context) error {
		return s.down(ctx, strings.ToLower(projectName), options)
	})
}

func (s *composeService) down(ctx context.Context, projectName string, options api.DownOptions) error {
	w := progress.ContextWriter(ctx)
	resourceToRemove := false

	var containers Containers
	containers, err := s.getContainers(ctx, projectName, oneOffInclude, true)
	if err != nil {
		return err
	}

	project := options.Project
	if project == nil {
		project, err = s.getProjectWithResources(ctx, containers, projectName)
		if err != nil {
			return err
		}
	}

	if len(containers) > 0 {
		resourceToRemove = true
	}

	err = InReverseDependencyOrder(ctx, project, func(c context.Context, service string) error {
		serviceContainers := containers.filter(isService(service))
		err := s.removeContainers(ctx, w, serviceContainers, options.Timeout, options.Volumes)
		return err
	})
	if err != nil {
		return err
	}

	orphans := containers.filter(isNotService(project.ServiceNames()...))
	if options.RemoveOrphans && len(orphans) > 0 {
		err := s.removeContainers(ctx, w, orphans, options.Timeout, false)
		if err != nil {
			return err
		}
	}

	ops := s.ensureNetworksDown(ctx, project, w)

	if options.Images != "" {
		ops = append(ops, s.ensureImagesDown(ctx, project, options, w)...)
	}

	if options.Volumes {
		ops = append(ops, s.ensureVolumesDown(ctx, project, w)...)
	}

	if !resourceToRemove && len(ops) == 0 {
		w.Event(progress.NewEvent(projectName, progress.Done, "Warning: No resource found to remove"))
	}

	eg, _ := errgroup.WithContext(ctx)
	for _, op := range ops {
		eg.Go(op)
	}
	return eg.Wait()
}

func (s *composeService) ensureVolumesDown(ctx context.Context, project *types.Project, w progress.Writer) []downOp {
	var ops []downOp
	for _, vol := range project.Volumes {
		if vol.External.External {
			continue
		}
		volumeName := vol.Name
		ops = append(ops, func() error {
			return s.removeVolume(ctx, volumeName, w)
		})
	}
	return ops
}

func (s *composeService) ensureImagesDown(ctx context.Context, project *types.Project, options api.DownOptions, w progress.Writer) []downOp {
	var ops []downOp
	for image := range s.getServiceImages(options, project) {
		image := image
		ops = append(ops, func() error {
			return s.removeImage(ctx, image, w)
		})
	}
	return ops
}

func (s *composeService) ensureNetworksDown(ctx context.Context, project *types.Project, w progress.Writer) []downOp {
	var ops []downOp
	for _, n := range project.Networks {
		if n.External.External {
			continue
		}
		networkName := n.Name
		_, err := s.apiClient().NetworkInspect(ctx, networkName, moby.NetworkInspectOptions{})
		if client.IsNotFound(err) {
			return nil
		}

		ops = append(ops, func() error {
			return s.removeNetwork(ctx, networkName, w)
		})
	}
	return ops
}

func (s *composeService) getServiceImages(options api.DownOptions, project *types.Project) map[string]struct{} {
	images := map[string]struct{}{}
	for _, service := range project.Services {
		image := service.Image
		if options.Images == "local" && image != "" {
			continue
		}
		if image == "" {
			image = getImageName(service, project.Name)
		}
		images[image] = struct{}{}
	}
	return images
}

func (s *composeService) removeImage(ctx context.Context, image string, w progress.Writer) error {
	id := fmt.Sprintf("Image %s", image)
	w.Event(progress.NewEvent(id, progress.Working, "Removing"))
	_, err := s.apiClient().ImageRemove(ctx, image, moby.ImageRemoveOptions{})
	if err == nil {
		w.Event(progress.NewEvent(id, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(id, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) removeVolume(ctx context.Context, id string, w progress.Writer) error {
	resource := fmt.Sprintf("Volume %s", id)
	w.Event(progress.NewEvent(resource, progress.Working, "Removing"))
	err := s.apiClient().VolumeRemove(ctx, id, true)
	if err == nil {
		w.Event(progress.NewEvent(resource, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(resource, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) stopContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration) error {
	eg, ctx := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.apiClient().ContainerStop(ctx, container.ID, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.StoppedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration, volumes bool) error {
	eg, _ := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.stopContainers(ctx, w, []moby.Container{container}, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.RemovingEvent(eventName))
			err = s.apiClient().ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{
				Force:         true,
				RemoveVolumes: volumes,
			})
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Removing"))
				return err
			}
			w.Event(progress.RemovedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) getProjectWithResources(ctx context.Context, containers Containers, projectName string) (*types.Project, error) {
	containers = containers.filter(isNotOneOff)
	project, _ := s.projectFromName(containers, projectName)

	volumes, err := s.actualVolumes(ctx, projectName)
	if err != nil {
		return nil, err
	}
	project.Volumes = volumes

	networks, err := s.actualNetworks(ctx, projectName)
	if err != nil {
		return nil, err
	}
	project.Networks = networks
	return project, nil
}
<|endoftext|>


<commit_before>package lfs

import (
	"io/ioutil"
	"testing"
)

type putRequest struct {
	Oid  string
	Size int
}

func tempdir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "git-lfs-test")
	if err != nil {
		t.Fatalf("Error getting temp dir: %s", err)
	}
	return dir
}
<commit_msg>アアー アアアア アー<commit_after>package lfs

import (
	"io/ioutil"
	"testing"
)

func tempdir(t *testing.T) string {
	dir, err := ioutil.TempDir("", "git-lfs-test")
	if err != nil {
		t.Fatalf("Error getting temp dir: %s", err)
	}
	return dir
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2019-2021 Authors of Cilium

//go:build !privileged_tests
// +build !privileged_tests

package elf

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"testing"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
type ELFTestSuite struct{}

var (
	_ = Suite(&ELFTestSuite{})

	baseObjPath = filepath.Join("..", "..", "test", "bpf", "elf-demo.o")
)

const elfObjCopy = "elf-demo-copy.o"

func Test(t *testing.T) {
	TestingT(t)
}

func hash(path string) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	hash := sha256.New()
	if _, err := io.Copy(hash, f); err != nil {
		return nil, err
	}

	return hash.Sum(nil), nil
}

var errDifferentFiles = fmt.Errorf("file hashes differ")

func compareFiles(path1, path2 string) error {
	h1, err := hash(path1)
	if err != nil {
		return err
	}
	h2, err := hash(path2)
	if err != nil {
		return err
	}

	if !bytes.Equal(h1, h2) {
		return errDifferentFiles
	}
	return nil
}

func (s *ELFTestSuite) TestWrite(c *C) {
	tmpDir, err := os.MkdirTemp("", "cilium_")
	c.Assert(err, IsNil)
	defer os.RemoveAll(tmpDir)

	elf, err := Open(baseObjPath)
	c.Assert(err, IsNil)
	defer elf.Close()

	validOptions := IsNil
	notValidOptions := Not(validOptions)
	type testOption struct {
		description  string
		key          string
		kind         symbolKind
		intValue     uint32
		strValue     string
		elfValid     Checker
		elfChangeErr error
	}
	testOptions := []testOption{
		{
			description: "test direct copy",
			elfValid:    validOptions,
		},
		{
			description:  "test constant substitution 1",
			key:          "FOO",
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description:  "test constant substitution 2",
			key:          "BAR",
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description:  "test map name substitution name",
			key:          "test_cilium_calls_4278124286",
			kind:         symbolString,
			strValue:     "test_cilium_calls_0000000042",
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description: "test invalid map name substitution",
			key:         "test_cilium_calls_4278124286",
			kind:        symbolString,
			strValue:    "test_cilium_calls_00",
			elfValid:    notValidOptions,
		},
	}

	for i := 1; i <= 4; i++ {
		testOptions = append(testOptions, testOption{
			description:  fmt.Sprintf("test ipv6 substitution %d", i),
			key:          fmt.Sprintf("GLOBAL_IPV6_%d", i),
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		})
	}

	for i := 1; i <= 2; i++ {
		testOptions = append(testOptions, testOption{
			description:  fmt.Sprintf("test mac substitution %d", i),
			key:          fmt.Sprintf("LOCAL_MAC_%d", i),
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		})
	}

	for i, test := range testOptions {
		c.Logf("%s", test.description)

		// Create the copy of the ELF with an optional substitution
		intOptions := make(map[string]uint32)
		strOptions := make(map[string]string)
		switch test.kind {
		case symbolUint32:
			intOptions[test.key] = test.intValue
		case symbolString:
			strOptions[test.key] = test.strValue
		}
		objectCopy := filepath.Join(tmpDir, fmt.Sprintf("%d_%s", i, elfObjCopy))
		err = elf.Write(objectCopy, intOptions, strOptions)
		c.Assert(err, test.elfValid)
		if test.elfValid == notValidOptions {
			continue
		}
		err = compareFiles(baseObjPath, objectCopy)
		c.Assert(err, Equals, test.elfChangeErr)

		// Test that the written ELF matches expectations
		modifiedElf, err := Open(objectCopy)
		c.Assert(err, IsNil)
		defer modifiedElf.Close()

		switch test.kind {
		case symbolUint32:
			value, err := modifiedElf.readOption(test.key)
			c.Assert(err, IsNil)
			c.Assert(value, Equals, test.intValue)
		case symbolString:
			err := modifiedElf.findString(test.strValue)
			c.Assert(err, IsNil)
		default:
			_, err = modifiedElf.readOption("unknown")
			c.Assert(err, NotNil)
		}
		modifiedElf.Close()
	}
}

// BenchmarkWriteELF benchmarks writing a very simple elf demo program.
func BenchmarkWriteELF(b *testing.B) {
	tmpDir, err := os.MkdirTemp("", "cilium_")
	if err != nil {
		b.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	elf, err := Open(baseObjPath)
	if err != nil {
		b.Fatal(err)
	}
	defer elf.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		intOptions := make(map[string]uint32)
		strOptions := make(map[string]string)

		objectCopy := filepath.Join(tmpDir, fmt.Sprintf("%d_%s", i, elfObjCopy))
		if err = elf.Write(objectCopy, intOptions, strOptions); err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>elf: skip BenchmarkWriteELF if ELF file wasn't built<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2019-2021 Authors of Cilium

//go:build !privileged_tests
// +build !privileged_tests

package elf

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"testing"

	. "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
type ELFTestSuite struct{}

var (
	_ = Suite(&ELFTestSuite{})

	baseObjPath = filepath.Join("..", "..", "test", "bpf", "elf-demo.o")
)

const elfObjCopy = "elf-demo-copy.o"

func Test(t *testing.T) {
	TestingT(t)
}

func hash(path string) ([]byte, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	hash := sha256.New()
	if _, err := io.Copy(hash, f); err != nil {
		return nil, err
	}

	return hash.Sum(nil), nil
}

var errDifferentFiles = fmt.Errorf("file hashes differ")

func compareFiles(path1, path2 string) error {
	h1, err := hash(path1)
	if err != nil {
		return err
	}
	h2, err := hash(path2)
	if err != nil {
		return err
	}

	if !bytes.Equal(h1, h2) {
		return errDifferentFiles
	}
	return nil
}

func (s *ELFTestSuite) TestWrite(c *C) {
	tmpDir, err := os.MkdirTemp("", "cilium_")
	c.Assert(err, IsNil)
	defer os.RemoveAll(tmpDir)

	elf, err := Open(baseObjPath)
	c.Assert(err, IsNil)
	defer elf.Close()

	validOptions := IsNil
	notValidOptions := Not(validOptions)
	type testOption struct {
		description  string
		key          string
		kind         symbolKind
		intValue     uint32
		strValue     string
		elfValid     Checker
		elfChangeErr error
	}
	testOptions := []testOption{
		{
			description: "test direct copy",
			elfValid:    validOptions,
		},
		{
			description:  "test constant substitution 1",
			key:          "FOO",
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description:  "test constant substitution 2",
			key:          "BAR",
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description:  "test map name substitution name",
			key:          "test_cilium_calls_4278124286",
			kind:         symbolString,
			strValue:     "test_cilium_calls_0000000042",
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		},
		{
			description: "test invalid map name substitution",
			key:         "test_cilium_calls_4278124286",
			kind:        symbolString,
			strValue:    "test_cilium_calls_00",
			elfValid:    notValidOptions,
		},
	}

	for i := 1; i <= 4; i++ {
		testOptions = append(testOptions, testOption{
			description:  fmt.Sprintf("test ipv6 substitution %d", i),
			key:          fmt.Sprintf("GLOBAL_IPV6_%d", i),
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		})
	}

	for i := 1; i <= 2; i++ {
		testOptions = append(testOptions, testOption{
			description:  fmt.Sprintf("test mac substitution %d", i),
			key:          fmt.Sprintf("LOCAL_MAC_%d", i),
			kind:         symbolUint32,
			intValue:     42,
			elfValid:     validOptions,
			elfChangeErr: errDifferentFiles,
		})
	}

	for i, test := range testOptions {
		c.Logf("%s", test.description)

		// Create the copy of the ELF with an optional substitution
		intOptions := make(map[string]uint32)
		strOptions := make(map[string]string)
		switch test.kind {
		case symbolUint32:
			intOptions[test.key] = test.intValue
		case symbolString:
			strOptions[test.key] = test.strValue
		}
		objectCopy := filepath.Join(tmpDir, fmt.Sprintf("%d_%s", i, elfObjCopy))
		err = elf.Write(objectCopy, intOptions, strOptions)
		c.Assert(err, test.elfValid)
		if test.elfValid == notValidOptions {
			continue
		}
		err = compareFiles(baseObjPath, objectCopy)
		c.Assert(err, Equals, test.elfChangeErr)

		// Test that the written ELF matches expectations
		modifiedElf, err := Open(objectCopy)
		c.Assert(err, IsNil)
		defer modifiedElf.Close()

		switch test.kind {
		case symbolUint32:
			value, err := modifiedElf.readOption(test.key)
			c.Assert(err, IsNil)
			c.Assert(value, Equals, test.intValue)
		case symbolString:
			err := modifiedElf.findString(test.strValue)
			c.Assert(err, IsNil)
		default:
			_, err = modifiedElf.readOption("unknown")
			c.Assert(err, NotNil)
		}
		modifiedElf.Close()
	}
}

// BenchmarkWriteELF benchmarks writing a very simple elf demo program.
func BenchmarkWriteELF(b *testing.B) {
	tmpDir, err := os.MkdirTemp("", "cilium_")
	if err != nil {
		b.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	elf, err := Open(baseObjPath)
	if errors.Is(err, fs.ErrNotExist) {
		// If the ELF file couldn't be found most likely it
		// wasn't built. See https://github.com/cilium/cilium/issues/17535
		b.Skip("ELF file not found, skipping benchmark")
	} else if err != nil {
		b.Fatal(err)
	}
	defer elf.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		intOptions := make(map[string]uint32)
		strOptions := make(map[string]string)

		objectCopy := filepath.Join(tmpDir, fmt.Sprintf("%d_%s", i, elfObjCopy))
		if err = elf.Write(objectCopy, intOptions, strOptions); err != nil {
			b.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 DGraph Labs, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package loader

import (
	"bufio"
	"context"
	"flag"
	"io"
	"math/rand"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/dgryski/go-farm"

	"github.com/dgraph-io/dgraph/posting"
	"github.com/dgraph-io/dgraph/rdf"
	"github.com/dgraph-io/dgraph/store"
	"github.com/dgraph-io/dgraph/uid"
	"github.com/dgraph-io/dgraph/x"
)

var glog = x.Log("loader")
var uidStore, dataStore *store.Store

var maxRoutines = flag.Int("maxroutines", 3000,
	"Maximum number of goroutines to execute concurrently")

type counters struct {
	read      uint64
	parsed    uint64
	processed uint64
	ignored   uint64
}

type state struct {
	sync.RWMutex
	input        chan string
	cnq          chan rdf.NQuad
	ctr          *counters
	instanceIdx  uint64
	numInstances uint64
	err          error
}

func Init(uidstore, datastore *store.Store) {
	uidStore = uidstore
	dataStore = datastore
}

func (s *state) Error() error {
	s.RLock()
	defer s.RUnlock()
	return s.err
}

func (s *state) SetError(err error) {
	s.Lock()
	defer s.Unlock()
	if s.err == nil {
		s.err = err
	}
}

// printCounters prints the counter variables at intervals
// specified by ticker.
func (s *state) printCounters(ticker *time.Ticker) {
	var prev uint64
	for _ = range ticker.C {
		processed := atomic.LoadUint64(&s.ctr.processed)
		if prev == processed {
			continue
		}
		prev = processed
		parsed := atomic.LoadUint64(&s.ctr.parsed)
		ignored := atomic.LoadUint64(&s.ctr.ignored)
		pending := parsed - ignored - processed
		glog.WithFields(logrus.Fields{
			"read":      atomic.LoadUint64(&s.ctr.read),
			"processed": processed,
			"parsed":    parsed,
			"ignored":   ignored,
			"pending":   pending,
			"len_cnq":   len(s.cnq),
		}).Info("Counters")
	}
}

// readLines reads the file and pushes the nquads onto a channel.
// Run this in a single goroutine. This function closes s.input channel.
func (s *state) readLines(r io.Reader) {
	var buf []string
	scanner := bufio.NewScanner(r)
	// Randomize lines to avoid contention on same subject.
	for i := 0; i < 1000; i++ {
		if scanner.Scan() {
			buf = append(buf, scanner.Text())
		} else {
			break
		}
	}
	ln := len(buf)
	for scanner.Scan() {
		k := rand.Intn(ln)
		s.input <- buf[k]
		buf[k] = scanner.Text()
		atomic.AddUint64(&s.ctr.read, 1)
	}
	if err := scanner.Err(); err != nil {
		glog.WithError(err).Fatal("While reading file.")
	}
	for i := 0; i < len(buf); i++ {
		s.input <- buf[i]
	}
	close(s.input)
}

// parseStream consumes the lines, converts them to nquad
// and sends them into cnq channel.
func (s *state) parseStream(wg *sync.WaitGroup) {
	defer wg.Done()

	for line := range s.input {
		if s.Error() != nil {
			return
		}
		line = strings.Trim(line, " \t")
		if len(line) == 0 {
			glog.Info("Empty line.")
			continue
		}

		glog.Debugf("Got line: %q", line)
		nq, err := rdf.Parse(line)
		if err != nil {
			s.SetError(err)
			return
		}
		s.cnq <- nq
		atomic.AddUint64(&s.ctr.parsed, 1)
	}
}

// handleNQuads converts the nQuads that satisfy the modulo
// rule into posting lists.
func (s *state) handleNQuads(wg *sync.WaitGroup) {
	defer wg.Done()

	ctx := context.Background()
	for nq := range s.cnq {
		if s.Error() != nil {
			return
		}
		// Only handle this edge if the attribute satisfies the modulo rule
		if farm.Fingerprint64([]byte(nq.Predicate))%s.numInstances != s.instanceIdx {
			atomic.AddUint64(&s.ctr.ignored, 1)
			continue
		}

		edge, err := nq.ToEdge()
		for err != nil {
			// Just put in a retry loop to tackle temporary errors.
			if err == posting.E_TMP_ERROR {
				time.Sleep(time.Microsecond)

			} else {
				s.SetError(err)
				glog.WithError(err).WithField("nq", nq).
					Error("While converting to edge")
				return
			}
			edge, err = nq.ToEdge()
		}

		key := posting.Key(edge.Entity, edge.Attribute)

		plist, decr := posting.GetOrCreate(key, dataStore)
		plist.AddMutationWithIndex(ctx, edge, posting.Set)
		decr() // Don't defer, just call because we're in a channel loop.

		atomic.AddUint64(&s.ctr.processed, 1)
	}
}

// assignUid assigns a unique integer for a given xid string.
func (s *state) assignUid(xid string) error {
	if strings.HasPrefix(xid, "_uid_:") {
		_, err := strconv.ParseUint(xid[6:], 0, 64)
		return err
	}

	_, err := uid.GetOrAssign(xid, s.instanceIdx, s.numInstances)
	for err != nil {
		// Just put in a retry loop to tackle temporary errors.
		if err == posting.E_TMP_ERROR {
			time.Sleep(time.Microsecond)
			glog.WithError(err).WithField("xid", xid).
				Debug("Temporary error")
		} else {
			glog.WithError(err).WithField("xid", xid).
				Error("While getting UID")
			return err
		}
		_, err = uid.GetOrAssign(xid, s.instanceIdx, s.numInstances)
	}
	return nil
}

// assignUidsOnly assigns uid to those entities that satisfy the
// modulo rule.
func (s *state) assignUidsOnly(wg *sync.WaitGroup) {
	defer wg.Done()

	for nq := range s.cnq {
		if s.Error() != nil {
			return
		}
		ignored := true
		if farm.Fingerprint64([]byte(nq.Subject))%s.numInstances == s.instanceIdx {
			if err := s.assignUid(nq.Subject); err != nil {
				s.SetError(err)
				glog.WithError(err).Error("While assigning Uid to subject.")
				return
			}
			ignored = false
		}

		if len(nq.ObjectId) > 0 &&
			farm.Fingerprint64([]byte(nq.ObjectId))%s.numInstances == s.instanceIdx {
			if err := s.assignUid(nq.ObjectId); err != nil {
				s.SetError(err)
				glog.WithError(err).Error("While assigning Uid to object.")
				return
			}
			ignored = false
		}

		if ignored {
			atomic.AddUint64(&s.ctr.ignored, 1)
		} else {
			atomic.AddUint64(&s.ctr.processed, 1)
		}
	}
}

// LoadEdges is called with the reader object of a file whose
// contents are to be converted to posting lists.
func LoadEdges(reader io.Reader, instanceIdx uint64,
	numInstances uint64) (uint64, error) {

	s := new(state)
	s.ctr = new(counters)
	ticker := time.NewTicker(time.Second)
	go s.printCounters(ticker)

	// Producer: Start buffering input to channel.
	s.instanceIdx = instanceIdx
	s.numInstances = numInstances
	s.input = make(chan string, 10000)
	go s.readLines(reader)

	s.cnq = make(chan rdf.NQuad, 10000)
	numr := runtime.GOMAXPROCS(-1)
	var pwg sync.WaitGroup
	pwg.Add(numr)
	for i := 0; i < numr; i++ {
		go s.parseStream(&pwg) // Input --> NQuads
	}

	nrt := *maxRoutines
	var wg sync.WaitGroup
	wg.Add(nrt)
	for i := 0; i < nrt; i++ {
		go s.handleNQuads(&wg) // NQuads --> Posting list [slow].
	}

	// Block until all parseStream goroutines are finished.
	pwg.Wait()
	close(s.cnq)
	// Okay, we've stopped input to cnq, and closed it.
	// Now wait for handleNQuads to finish.
	wg.Wait()

	ticker.Stop()
	return atomic.LoadUint64(&s.ctr.processed), s.Error()
}

// AssignUids would pick up all the external ids in RDFs read,
// and assign unique integer ids for them. This function would
// not load the edges, only assign UIDs.
func AssignUids(reader io.Reader, instanceIdx uint64,
	numInstances uint64) (uint64, error) {

	s := new(state)
	s.ctr = new(counters)
	ticker := time.NewTicker(time.Second)
	go s.printCounters(ticker)

	// Producer: Start buffering input to channel.
	s.instanceIdx = instanceIdx
	s.numInstances = numInstances
	s.input = make(chan string, 10000)
	go s.readLines(reader)

	s.cnq = make(chan rdf.NQuad, 10000)
	numr := runtime.GOMAXPROCS(-1)
	var pwg sync.WaitGroup
	pwg.Add(numr)
	for i := 0; i < numr; i++ {
		go s.parseStream(&pwg) // Input --> NQuads
	}

	wg := new(sync.WaitGroup)
	nrt := *maxRoutines
	for i := 0; i < nrt; i++ {
		wg.Add(1)
		go s.assignUidsOnly(wg)
	}

	// Block until all parseStream goroutines are finished.
	pwg.Wait()
	close(s.cnq)
	// Okay, we've stopped input to cnq, and closed it.
	// Now wait for handleNQuads to finish.
	wg.Wait()

	ticker.Stop()
	return atomic.LoadUint64(&s.ctr.processed), s.Error()
}
<commit_msg>Support long lines in rdf files.<commit_after>/*
 * Copyright 2015 DGraph Labs, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 		http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package loader

import (
	"bufio"
	"bytes"
	"context"
	"flag"
	"io"
	"log"
	"math/rand"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/dgryski/go-farm"

	"github.com/dgraph-io/dgraph/posting"
	"github.com/dgraph-io/dgraph/rdf"
	"github.com/dgraph-io/dgraph/store"
	"github.com/dgraph-io/dgraph/uid"
	"github.com/dgraph-io/dgraph/x"
)

var glog = x.Log("loader")
var uidStore, dataStore *store.Store

var maxRoutines = flag.Int("maxroutines", 3000,
	"Maximum number of goroutines to execute concurrently")

type counters struct {
	read      uint64
	parsed    uint64
	processed uint64
	ignored   uint64
}

type state struct {
	sync.RWMutex
	input        chan string
	cnq          chan rdf.NQuad
	ctr          *counters
	instanceIdx  uint64
	numInstances uint64
	err          error
}

func Init(uidstore, datastore *store.Store) {
	uidStore = uidstore
	dataStore = datastore
}

func (s *state) Error() error {
	s.RLock()
	defer s.RUnlock()
	return s.err
}

func (s *state) SetError(err error) {
	s.Lock()
	defer s.Unlock()
	if s.err == nil {
		s.err = err
	}
}

// printCounters prints the counter variables at intervals
// specified by ticker.
func (s *state) printCounters(ticker *time.Ticker) {
	var prev uint64
	for _ = range ticker.C {
		processed := atomic.LoadUint64(&s.ctr.processed)
		if prev == processed {
			continue
		}
		prev = processed
		parsed := atomic.LoadUint64(&s.ctr.parsed)
		ignored := atomic.LoadUint64(&s.ctr.ignored)
		pending := parsed - ignored - processed
		glog.WithFields(logrus.Fields{
			"read":      atomic.LoadUint64(&s.ctr.read),
			"processed": processed,
			"parsed":    parsed,
			"ignored":   ignored,
			"pending":   pending,
			"len_cnq":   len(s.cnq),
		}).Info("Counters")
	}
}

// Reads a single line from a buffered reader. The line is read into the
// passed in buffer to minimize allocations. This is the preferred
// method for loading long lines which could be longer than the buffer
// size of bufio.Scanner.
func readLine(r *bufio.Reader, buf *bytes.Buffer) error {
	isPrefix := true
	var err error
	buf.Reset()
	for isPrefix && err == nil {
		var line []byte
		// The returned line is an internal buffer in bufio and is only
		// valid until the next call to ReadLine. It needs to be copied
		// over to our own buffer.
		line, isPrefix, err = r.ReadLine()
		if err == nil {
			buf.Write(line)
		}
	}
	return err
}

// readLines reads the file and pushes the nquads onto a channel.
// Run this in a single goroutine. This function closes s.input channel.
func (s *state) readLines(r io.Reader) {
	var buf []string
	var err error
	var strBuf bytes.Buffer
	bufReader := bufio.NewReader(r)
	// Randomize lines to avoid contention on same subject.
	for i := 0; i < 1000; i++ {
		err = readLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		buf = append(buf, strBuf.String())
		atomic.AddUint64(&s.ctr.read, 1)
	}

	if err != nil && err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}

	// If we haven't yet finished reading the file read the rest of the rows.
	for {
		err = readLine(bufReader, &strBuf)
		if err != nil {
			break
		}
		k := rand.Intn(len(buf))
		s.input <- buf[k]
		buf[k] = strBuf.String()
		atomic.AddUint64(&s.ctr.read, 1)
	}

	if err != io.EOF {
		err := x.Errorf("Error while reading file: %v", err)
		log.Fatalf("%+v", err)
	}
	for i := 0; i < len(buf); i++ {
		s.input <- buf[i]
	}
	close(s.input)
}

// parseStream consumes the lines, converts them to nquad
// and sends them into cnq channel.
func (s *state) parseStream(wg *sync.WaitGroup) {
	defer wg.Done()

	for line := range s.input {
		if s.Error() != nil {
			return
		}
		line = strings.Trim(line, " \t")
		if len(line) == 0 {
			glog.Info("Empty line.")
			continue
		}

		glog.Debugf("Got line: %q", line)
		nq, err := rdf.Parse(line)
		if err != nil {
			s.SetError(err)
			return
		}
		s.cnq <- nq
		atomic.AddUint64(&s.ctr.parsed, 1)
	}
}

// handleNQuads converts the nQuads that satisfy the modulo
// rule into posting lists.
func (s *state) handleNQuads(wg *sync.WaitGroup) {
	defer wg.Done()

	ctx := context.Background()
	for nq := range s.cnq {
		if s.Error() != nil {
			return
		}
		// Only handle this edge if the attribute satisfies the modulo rule
		if farm.Fingerprint64([]byte(nq.Predicate))%s.numInstances != s.instanceIdx {
			atomic.AddUint64(&s.ctr.ignored, 1)
			continue
		}

		edge, err := nq.ToEdge()
		for err != nil {
			// Just put in a retry loop to tackle temporary errors.
			if err == posting.E_TMP_ERROR {
				time.Sleep(time.Microsecond)

			} else {
				s.SetError(err)
				glog.WithError(err).WithField("nq", nq).
					Error("While converting to edge")
				return
			}
			edge, err = nq.ToEdge()
		}

		key := posting.Key(edge.Entity, edge.Attribute)

		plist, decr := posting.GetOrCreate(key, dataStore)
		plist.AddMutationWithIndex(ctx, edge, posting.Set)
		decr() // Don't defer, just call because we're in a channel loop.

		atomic.AddUint64(&s.ctr.processed, 1)
	}
}

// assignUid assigns a unique integer for a given xid string.
func (s *state) assignUid(xid string) error {
	if strings.HasPrefix(xid, "_uid_:") {
		_, err := strconv.ParseUint(xid[6:], 0, 64)
		return err
	}

	_, err := uid.GetOrAssign(xid, s.instanceIdx, s.numInstances)
	for err != nil {
		// Just put in a retry loop to tackle temporary errors.
		if err == posting.E_TMP_ERROR {
			time.Sleep(time.Microsecond)
			glog.WithError(err).WithField("xid", xid).
				Debug("Temporary error")
		} else {
			glog.WithError(err).WithField("xid", xid).
				Error("While getting UID")
			return err
		}
		_, err = uid.GetOrAssign(xid, s.instanceIdx, s.numInstances)
	}
	return nil
}

// assignUidsOnly assigns uid to those entities that satisfy the
// modulo rule.
func (s *state) assignUidsOnly(wg *sync.WaitGroup) {
	defer wg.Done()

	for nq := range s.cnq {
		if s.Error() != nil {
			return
		}
		ignored := true
		if farm.Fingerprint64([]byte(nq.Subject))%s.numInstances == s.instanceIdx {
			if err := s.assignUid(nq.Subject); err != nil {
				s.SetError(err)
				glog.WithError(err).Error("While assigning Uid to subject.")
				return
			}
			ignored = false
		}

		if len(nq.ObjectId) > 0 &&
			farm.Fingerprint64([]byte(nq.ObjectId))%s.numInstances == s.instanceIdx {
			if err := s.assignUid(nq.ObjectId); err != nil {
				s.SetError(err)
				glog.WithError(err).Error("While assigning Uid to object.")
				return
			}
			ignored = false
		}

		if ignored {
			atomic.AddUint64(&s.ctr.ignored, 1)
		} else {
			atomic.AddUint64(&s.ctr.processed, 1)
		}
	}
}

// LoadEdges is called with the reader object of a file whose
// contents are to be converted to posting lists.
func LoadEdges(reader io.Reader, instanceIdx uint64,
	numInstances uint64) (uint64, error) {

	s := new(state)
	s.ctr = new(counters)
	ticker := time.NewTicker(time.Second)
	go s.printCounters(ticker)

	// Producer: Start buffering input to channel.
	s.instanceIdx = instanceIdx
	s.numInstances = numInstances
	s.input = make(chan string, 10000)
	go s.readLines(reader)

	s.cnq = make(chan rdf.NQuad, 10000)
	numr := runtime.GOMAXPROCS(-1)
	var pwg sync.WaitGroup
	pwg.Add(numr)
	for i := 0; i < numr; i++ {
		go s.parseStream(&pwg) // Input --> NQuads
	}

	nrt := *maxRoutines
	var wg sync.WaitGroup
	wg.Add(nrt)
	for i := 0; i < nrt; i++ {
		go s.handleNQuads(&wg) // NQuads --> Posting list [slow].
	}

	// Block until all parseStream goroutines are finished.
	pwg.Wait()
	close(s.cnq)
	// Okay, we've stopped input to cnq, and closed it.
	// Now wait for handleNQuads to finish.
	wg.Wait()

	ticker.Stop()
	return atomic.LoadUint64(&s.ctr.processed), s.Error()
}

// AssignUids would pick up all the external ids in RDFs read,
// and assign unique integer ids for them. This function would
// not load the edges, only assign UIDs.
func AssignUids(reader io.Reader, instanceIdx uint64,
	numInstances uint64) (uint64, error) {

	s := new(state)
	s.ctr = new(counters)
	ticker := time.NewTicker(time.Second)
	go s.printCounters(ticker)

	// Producer: Start buffering input to channel.
	s.instanceIdx = instanceIdx
	s.numInstances = numInstances
	s.input = make(chan string, 10000)
	go s.readLines(reader)

	s.cnq = make(chan rdf.NQuad, 10000)
	numr := runtime.GOMAXPROCS(-1)
	var pwg sync.WaitGroup
	pwg.Add(numr)
	for i := 0; i < numr; i++ {
		go s.parseStream(&pwg) // Input --> NQuads
	}

	wg := new(sync.WaitGroup)
	nrt := *maxRoutines
	for i := 0; i < nrt; i++ {
		wg.Add(1)
		go s.assignUidsOnly(wg)
	}

	// Block until all parseStream goroutines are finished.
	pwg.Wait()
	close(s.cnq)
	// Okay, we've stopped input to cnq, and closed it.
	// Now wait for handleNQuads to finish.
	wg.Wait()

	ticker.Stop()
	return atomic.LoadUint64(&s.ctr.processed), s.Error()
}
<|endoftext|>