<commit_before>package presilo

import (
	"fmt"
	"strings"
	"bytes"
)

func GenerateRuby(schema *ObjectSchema, module string) string {

	var ret bytes.Buffer
	var toWrite string

	toWrite = fmt.Sprintf("module %s\n\n", ToCamelCase(module))
	ret.WriteString(toWrite)

	ret.WriteString(generateRubySignature(schema))
	ret.WriteString("\n")
	ret.WriteString(generateRubyConstructor(schema))
	ret.WriteString("\n")
	ret.WriteString(generateRubyFunctions(schema))
	ret.WriteString("\nend\nend\n")

	return ret.String()
}

func generateRubySignature(schema *ObjectSchema) string {

	var ret bytes.Buffer
	var subschema TypeSchema
	var readers, accessors []string
	var propertyName string
	var toWrite string

	toWrite = fmt.Sprintf("class %s\n", ToCamelCase(schema.Title))
	ret.WriteString(toWrite)

	for propertyName, subschema = range schema.Properties {

		propertyName = ToSnakeCase(propertyName)

		if(subschema.HasConstraints()) {
			toWrite = fmt.Sprintf(":%s", propertyName)
			readers = append(readers, toWrite)

		} else {

			toWrite = fmt.Sprintf(":%s", propertyName)
			accessors = append(accessors, toWrite)
		}
	}

	if(len(readers) > 0) {
		ret.WriteString("\n\tattr_reader ")
		ret.WriteString(strings.Join(readers, ",\n\t\t\t\t\t\t\t"))
	}

	if(len(accessors) > 0) {
		ret.WriteString("\n\tattr_accessor ")
		ret.WriteString(strings.Join(accessors, ",\n\t\t\t\t\t\t\t\t")) // god. Ruby.
	}

	return ret.String()
}

func generateRubyConstructor(schema *ObjectSchema) string {

	var ret bytes.Buffer
	var declarations, setters []string
	var propertyName string
	var toWrite string

	ret.WriteString("\n\tdef initialize(")

	for _, propertyName = range schema.RequiredProperties {

		propertyName = ToSnakeCase(propertyName)

		declarations = append(declarations, propertyName)

		toWrite = fmt.Sprintf("\n\t\tset_%s(%s)", propertyName, propertyName)
		setters = append(setters, toWrite)
	}

	toWrite = strings.Join(declarations, ",")
	ret.WriteString(toWrite)
	ret.WriteString(")")

	for _, setter := range setters {
		ret.WriteString(setter)
	}

	ret.WriteString("\n\tend\n")
	return ret.String()
}

func generateRubyFunctions(schema *ObjectSchema) string {

	var ret bytes.Buffer
	var subschema TypeSchema
	var toWrite string
	var propertyName, snakeName string

	for propertyName, subschema = range schema.Properties {

		snakeName = ToSnakeCase(propertyName)

		// getter
		toWrite = fmt.Sprintf("\n\tdef get_%s()", snakeName)
		ret.WriteString(toWrite)

		toWrite = fmt.Sprintf("\n\t\treturn @%s\n\tend\n", snakeName)
		ret.WriteString(toWrite)

		// setter
		toWrite = fmt.Sprintf("\n\tdef set_%s(%s)", snakeName, snakeName)
		ret.WriteString(toWrite)

		switch subschema.GetSchemaType() {
		case SCHEMATYPE_BOOLEAN:
			toWrite = ""
		case SCHEMATYPE_STRING:
			toWrite = generateRubyStringSetter(subschema.(*StringSchema))
		case SCHEMATYPE_INTEGER:
			fallthrough
		case SCHEMATYPE_NUMBER:
			toWrite = generateRubyNumericSetter(subschema.(NumericSchemaType))
		case SCHEMATYPE_OBJECT:
			toWrite = generateRubyObjectSetter(subschema.(*ObjectSchema))
		case SCHEMATYPE_ARRAY:
			toWrite = generateRubyArraySetter(subschema.(*ArraySchema))
		}

		ret.WriteString(toWrite)

		toWrite = fmt.Sprintf("\n\t\t@%s = value", snakeName)
		ret.WriteString(toWrite)

		ret.WriteString("\n\tend\n")
	}

	return ret.String()
}

func generateRubyStringSetter(schema *StringSchema) string {

	var ret bytes.Buffer
	var toWrite string

	ret.WriteString(generateRubyNullCheck())

	if schema.MinLength != nil {
		ret.WriteString(generateRubyRangeCheck(*schema.MinLength, "value.length", "was shorter than allowable minimum", "%d", false, "<", ""))
	}

	if schema.MaxLength != nil {
		ret.WriteString(generateRubyRangeCheck(*schema.MaxLength, "value.length", "was longer than allowable maximum", "%d", false, ">", ""))
	}

	if schema.HasEnum() {
		ret.WriteString(generateRubyEnumCheck(schema, schema.GetEnum(), "'", "'"))
	}

	if schema.Pattern != nil {

		toWrite = fmt.Sprintf("\n\t\tif(value =~ /%s/)\n", *schema.Pattern)
		ret.WriteString(toWrite)

		toWrite = fmt.Sprintf("\n\t\t\traise StandardError.new(\"Value '#{value}' did not match pattern '%s'\")", *schema.Pattern)
		ret.WriteString(toWrite)

		ret.WriteString("\n\t\tend")
	}
	return ret.String()
}

func generateRubyNumericSetter(schema NumericSchemaType) string {

	var ret bytes.Buffer
	var toWrite string

	if schema.HasMinimum() {
		ret.WriteString(generateRubyRangeCheck(schema.GetMinimum(), "value", "is under the allowable minimum", schema.GetConstraintFormat(), schema.IsExclusiveMinimum(), "<=", "<"))
	}

	if schema.HasMaximum() {
		ret.WriteString(generateRubyRangeCheck(schema.GetMaximum(), "value", "is over the allowable maximum", schema.GetConstraintFormat(), schema.IsExclusiveMaximum(), ">=", ">"))
	}

	if schema.HasEnum() {
		ret.WriteString(generateRubyEnumCheck(schema, schema.GetEnum(), "", ""))
	}

	if schema.HasMultiple() {

		toWrite = fmt.Sprintf("\n\tif(value %% %f != 0)\n\t", schema.GetMultiple())
		ret.WriteString(toWrite)

		toWrite = fmt.Sprintf("\n\t\traise StandardError.new(\"Property '#{value}' was not a multiple of %v\")", schema.GetMultiple())
		ret.WriteString(toWrite)

		ret.WriteString("\n\tend\n")
	}
	return ret.String()
}

func generateRubyObjectSetter(schema *ObjectSchema) string {

	return generateRubyNullCheck()
}

func generateRubyArraySetter(schema *ArraySchema) string {

	var ret bytes.Buffer

	ret.WriteString(generateRubyNullCheck())

	if schema.MinItems != nil {
		ret.WriteString(generateRubyRangeCheck(*schema.MinItems, "value.Length", "does not have enough items", "%d", false, "<", ""))
	}

	if schema.MaxItems != nil {
		ret.WriteString(generateRubyRangeCheck(*schema.MaxItems, "value.Length", "does not have enough items", "%d", false, ">", ""))
	}

	return ret.String()
}

func generateRubyRangeCheck(value interface{}, reference, message, format string, exclusive bool, comparator, exclusiveComparator string) string {

	var ret bytes.Buffer
	var toWrite, compareString string

	if exclusive {
		compareString = exclusiveComparator
	} else {
		compareString = comparator
	}

	toWrite = "\n\t\tif(" + reference + " " + compareString + " " + format + ")"
	toWrite = fmt.Sprintf(toWrite, value)
	ret.WriteString(toWrite)

	toWrite = fmt.Sprintf("\n\t\t\traise StandardError.new(\"Property '#{value}' %s.\")\n\t\tend\n", message)
	ret.WriteString(toWrite)

	return ret.String()
}

/*
	Generates code which throws an error if the given [parameter]'s value is not contained in the given [validValues].
*/
func generateRubyEnumCheck(schema TypeSchema, enumValues []interface{}, prefix string, postfix string) string {

	var ret bytes.Buffer
	var stringValues []string
	var constraint string
	var length int

	length = len(enumValues)

	if length <= 0 {
		return ""
	}

	// convert enum values to strings
	for _, enum := range enumValues {
		stringValues = append(stringValues, fmt.Sprintf("%s%v%s", prefix, enum, postfix))
	}

	// write array of valid values
	constraint = fmt.Sprintf("\n\t\tvalidValues = [%s]\n", strings.Join(stringValues, ","))
	ret.WriteString(constraint)

	// compare
	ret.WriteString("\n\t\tunless(validValues.include?(value))")
	ret.WriteString("\n\t\t\traise StandardError.new(\"Given value '#{value}' was not found in list of acceptable values\")\n")
	ret.WriteString("\t\tend\n")

	return ret.String()
}

func generateRubyNullCheck() string {

	var ret bytes.Buffer

	ret.WriteString("\n\t\tif(value == nil)")
	ret.WriteString("\n\t\t\traise StandardError.new(\"Cannot set property to null value\")")
	ret.WriteString("\n\t\tend\n")

	return ret.String()
}
<commit_msg>Converted ruby to new indentation style<commit_after>package presilo

import (
	"fmt"
	"strings"
)

func GenerateRuby(schema *ObjectSchema, module string) string {

	var buffer *BufferedFormatString

	buffer = NewBufferedFormatString("  ")
	buffer.Printf("module %s\n\n", ToCamelCase(module))
	buffer.AddIndentation(1)

	generateRubySignature(schema, buffer)
	buffer.Print("\n")
	generateRubyConstructor(schema, buffer)
	buffer.Print("\n")
	generateRubyFunctions(schema, buffer)

	buffer.AddIndentation(-1)
	buffer.Print("\nend")
	buffer.AddIndentation(-1)
	buffer.Print("\nend")

	return buffer.String()
}

func generateRubySignature(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var readers, accessors []string
	var propertyName string
	var toWrite string

	buffer.Printf("class %s\n", ToCamelCase(schema.Title))
	buffer.AddIndentation(1)

	for propertyName, subschema = range schema.Properties {

		propertyName = ToSnakeCase(propertyName)

		if(subschema.HasConstraints()) {
			toWrite = fmt.Sprintf(":%s", propertyName)
			readers = append(readers, toWrite)

		} else {

			toWrite = fmt.Sprintf(":%s", propertyName)
			accessors = append(accessors, toWrite)
		}
	}

	if(len(readers) > 0) {

		buffer.Print("\nattr_reader ")
		buffer.AddIndentation(6)
		buffer.Print(strings.Join(readers, ",\n"))
		buffer.AddIndentation(-6)
	}

	if(len(accessors) > 0) {

		buffer.Print("\nattr_accessor ")
		buffer.AddIndentation(7)
		buffer.Print(strings.Join(accessors, ",\n"))
		buffer.AddIndentation(-7)
	}
}

func generateRubyConstructor(schema *ObjectSchema, buffer *BufferedFormatString) {

	var declarations []string
	var propertyName string

	buffer.Print("\ndef initialize(")

	for _, propertyName = range schema.RequiredProperties {

		propertyName = ToSnakeCase(propertyName)
		declarations = append(declarations, propertyName)
	}

	buffer.Printf("%s)\n", strings.Join(declarations, ","))
	buffer.AddIndentation(1)

	for _, propertyName = range schema.RequiredProperties {
		buffer.Printf("\nset_%s(%s)", propertyName, propertyName)
	}

	buffer.AddIndentation(-1)
	buffer.Print("\nend\n")
}

func generateRubyFunctions(schema *ObjectSchema, buffer *BufferedFormatString) {

	var subschema TypeSchema
	var propertyName, snakeName string

	for propertyName, subschema = range schema.Properties {

		snakeName = ToSnakeCase(propertyName)

		// getter
		buffer.Printf("\ndef get_%s()", snakeName)
		buffer.AddIndentation(1)

		buffer.Printf("\nreturn @%s", snakeName)

		buffer.AddIndentation(-1)
		buffer.Print("\nend\n")

		// setter
		buffer.Printf("\ndef set_%s(%s)", snakeName, snakeName)
		buffer.AddIndentation(1)

		switch subschema.GetSchemaType() {
		case SCHEMATYPE_STRING:
			generateRubyStringSetter(subschema.(*StringSchema), buffer)
		case SCHEMATYPE_INTEGER:
			fallthrough
		case SCHEMATYPE_NUMBER:
			generateRubyNumericSetter(subschema.(NumericSchemaType), buffer)
		case SCHEMATYPE_OBJECT:
			generateRubyObjectSetter(subschema.(*ObjectSchema), buffer)
		case SCHEMATYPE_ARRAY:
			generateRubyArraySetter(subschema.(*ArraySchema), buffer)
		}

		buffer.Printf("\n@%s = value", snakeName)
		buffer.AddIndentation(-1)
		buffer.Print("\nend\n")
	}
}

func generateRubyStringSetter(schema *StringSchema, buffer *BufferedFormatString) {

	generateRubyNullCheck(buffer)

	if schema.MinLength != nil {
		generateRubyRangeCheck(*schema.MinLength, "value.length", "was shorter than allowable minimum", "%d", false, "<", "", buffer)
	}

	if schema.MaxLength != nil {
		generateRubyRangeCheck(*schema.MaxLength, "value.length", "was longer than allowable maximum", "%d", false, ">", "", buffer)
	}

	if schema.HasEnum() {
		generateRubyEnumCheck(schema, buffer, schema.GetEnum(), "'", "'")
	}

	if schema.Pattern != nil {

		buffer.Printf("\nif(value =~ /%s/)\n", *schema.Pattern)
		buffer.AddIndentation(1)

		buffer.Printf("\nraise StandardError.new(\"Value '#{value}' did not match pattern '%s'\")", *schema.Pattern)

		buffer.AddIndentation(-1)
		buffer.Print("\nend")
	}
}

func generateRubyNumericSetter(schema NumericSchemaType, buffer *BufferedFormatString) {

	if schema.HasMinimum() {
		generateRubyRangeCheck(schema.GetMinimum(), "value", "is under the allowable minimum", schema.GetConstraintFormat(), schema.IsExclusiveMinimum(), "<=", "<", buffer)
	}

	if schema.HasMaximum() {
		generateRubyRangeCheck(schema.GetMaximum(), "value", "is over the allowable maximum", schema.GetConstraintFormat(), schema.IsExclusiveMaximum(), ">=", ">", buffer)
	}

	if schema.HasEnum() {
		generateRubyEnumCheck(schema, buffer, schema.GetEnum(), "", "")
	}

	if schema.HasMultiple() {

		buffer.Printf("\nif(value %% %f != 0)", schema.GetMultiple())
		buffer.AddIndentation(1)

		buffer.Printf("\nraise StandardError.new(\"Property '#{value}' was not a multiple of %v\")", schema.GetMultiple())

		buffer.AddIndentation(-1)
		buffer.Print("\nend\n")
	}
}

func generateRubyObjectSetter(schema *ObjectSchema, buffer *BufferedFormatString) {

	generateRubyNullCheck(buffer)
}

func generateRubyArraySetter(schema *ArraySchema, buffer *BufferedFormatString) {

	generateRubyNullCheck(buffer)

	if schema.MinItems != nil {
		generateRubyRangeCheck(*schema.MinItems, "value.Length", "does not have enough items", "%d", false, "<", "", buffer)
	}

	if schema.MaxItems != nil {
		generateRubyRangeCheck(*schema.MaxItems, "value.Length", "does not have enough items", "%d", false, ">", "", buffer)
	}
}

func generateRubyRangeCheck(value interface{}, reference, message, format string, exclusive bool, comparator, exclusiveComparator string, buffer *BufferedFormatString) {

	var compareString string

	if exclusive {
		compareString = exclusiveComparator
	} else {
		compareString = comparator
	}

	buffer.Printf("\nif(%s %s " + format + ")", value, compareString)
	buffer.AddIndentation(1)

	buffer.Printf("\nraise StandardError.new(\"Property '#{value}' %s.\")", message)

	buffer.AddIndentation(-1)
	buffer.Print("\nend\n")
}

/*
	Generates code which throws an error if the given [parameter]'s value is not contained in the given [validValues].
*/
func generateRubyEnumCheck(schema TypeSchema, buffer *BufferedFormatString, enumValues []interface{}, prefix string, postfix string) {

	var stringValues []string
	var length int

	length = len(enumValues)

	if length <= 0 {
		return
	}

	// convert enum values to strings
	for _, enum := range enumValues {
		stringValues = append(stringValues, fmt.Sprintf("%s%v%s", prefix, enum, postfix))
	}

	// write array of valid values
	buffer.Printf("\nvalidValues = [%s]\n", strings.Join(stringValues, ","))

	// compare
	buffer.Print("\nunless(validValues.include?(value))")
	buffer.AddIndentation(1)

	buffer.Print("\nraise StandardError.new(\"Given value '#{value}' was not found in list of acceptable values\")")

	buffer.AddIndentation(-1)
	buffer.Print("\nend\n")
}

func generateRubyNullCheck(buffer *BufferedFormatString) {

	buffer.Print("\nif(value == nil)")
	buffer.AddIndentation(1)

	buffer.Print("\nraise StandardError.new(\"Cannot set property to null value\")")

	buffer.AddIndentation(-1)
	buffer.Print("\nend\n")
}
<|endoftext|>


<commit_before>package optimize

import (
	"github.com/gonum/floats"
)

// LBFGS implements the limited-memory BFGS algorithm. While the normal BFGS algorithm
// makes a full approximation to the inverse hessian, LBFGS instead approximates the
// hessian from the last Store optimization steps. The Store parameter is a tradeoff
// between cost of the method and accuracy of the hessian approximation.
// LBFGS has a cost (both in memory and time) of O(Store * inputDimension).
// Since BFGS has a cost of O(inputDimension^2), LBFGS is more appropriate
// for very large problems. This "forgetful" nature of LBFGS may also make it perform
// better than BFGS for functions with Hessians that vary rapidly spatially.
//
// If Store is 0, Store is defaulted to 15.
// A Linesearcher for LBFGS must satisfy the strong Wolfe conditions at every
// iteration. If Linesearcher == nil, an appropriate default is chosen.
type LBFGS struct {
	Linesearcher Linesearcher
	Store        int // how many past iterations to store

	ls *LinesearchMethod

	dim    int
	oldest int // element of the history slices that is the oldest

	x    []float64 // location at the last major iteration
	grad []float64 // gradient at the last major iteration

	y []float64 // holds g_{k+1} - g_k
	s []float64 // holds x_{k+1} - x_k
	a []float64 // holds cache of hessian updates

	// History
	yHist   [][]float64 // last Store iterations of y
	sHist   [][]float64 // last Store iterations of s
	rhoHist []float64   // last Store iterations of rho
}

func (l *LBFGS) Init(loc *Location) (Operation, error) {
	if l.Linesearcher == nil {
		l.Linesearcher = &Bisection{}
	}
	if l.ls == nil {
		l.ls = &LinesearchMethod{}
	}
	l.ls.Linesearcher = l.Linesearcher
	l.ls.NextDirectioner = l
	return l.ls.Init(loc)
}

func (l *LBFGS) Iterate(loc *Location) (Operation, error) {
	return l.ls.Iterate(loc)
}

func (l *LBFGS) InitDirection(loc *Location, dir []float64) (stepSize float64) {
	dim := len(loc.X)
	l.dim = dim

	if l.Store == 0 {
		l.Store = 15
	}

	l.oldest = l.Store - 1 // the first vector will be put in at 0

	l.x = resize(l.x, dim)
	l.grad = resize(l.grad, dim)
	copy(l.x, loc.X)
	copy(l.grad, loc.Gradient)

	l.y = resize(l.y, dim)
	l.s = resize(l.s, dim)
	l.a = resize(l.a, l.Store)
	l.rhoHist = resize(l.rhoHist, l.Store)

	if cap(l.yHist) < l.Store {
		n := make([][]float64, l.Store-cap(l.yHist))
		l.yHist = append(l.yHist, n...)
	}
	if cap(l.sHist) < l.Store {
		n := make([][]float64, l.Store-cap(l.sHist))
		l.sHist = append(l.sHist, n...)
	}
	l.yHist = l.yHist[:l.Store]
	l.sHist = l.sHist[:l.Store]
	for i := range l.sHist {
		l.sHist[i] = resize(l.sHist[i], dim)
		for j := range l.sHist[i] {
			l.sHist[i][j] = 0
		}
	}
	for i := range l.yHist {
		l.yHist[i] = resize(l.yHist[i], dim)
		for j := range l.yHist[i] {
			l.yHist[i][j] = 0
		}
	}

	copy(dir, loc.Gradient)
	floats.Scale(-1, dir)

	return 1 / floats.Norm(dir, 2)
}

func (l *LBFGS) NextDirection(loc *Location, dir []float64) (stepSize float64) {
	if len(loc.X) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}
	if len(loc.Gradient) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}
	if len(dir) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}

	// Update direction. Uses two-loop correction as described in
	// Nocedal, Wright (2006), Numerical Optimization (2nd ed.). Chapter 7, page 178.
	copy(dir, loc.Gradient)
	floats.SubTo(l.y, loc.Gradient, l.grad)
	floats.SubTo(l.s, loc.X, l.x)
	copy(l.sHist[l.oldest], l.s)
	copy(l.yHist[l.oldest], l.y)
	sDotY := floats.Dot(l.y, l.s)
	l.rhoHist[l.oldest] = 1 / sDotY

	l.oldest++
	l.oldest = l.oldest % l.Store
	copy(l.x, loc.X)
	copy(l.grad, loc.Gradient)

	// two loop update. First loop starts with the most recent element
	// and goes backward, second starts with the oldest element and goes
	// forward. At the end have computed H^-1 * g, so flip the direction for
	// minimization.
	for i := 0; i < l.Store; i++ {
		idx := l.oldest - i - 1
		if idx < 0 {
			idx += l.Store
		}
		l.a[idx] = l.rhoHist[idx] * floats.Dot(l.sHist[idx], dir)
		floats.AddScaled(dir, -l.a[idx], l.yHist[idx])
	}

	// Scale the initial Hessian.
	gamma := sDotY / floats.Dot(l.y, l.y)
	floats.Scale(gamma, dir)

	for i := 0; i < l.Store; i++ {
		idx := i + l.oldest
		if idx >= l.Store {
			idx -= l.Store
		}
		beta := l.rhoHist[idx] * floats.Dot(l.yHist[idx], dir)
		floats.AddScaled(dir, l.a[idx]-beta, l.sHist[idx])
	}
	floats.Scale(-1, dir)

	return 1
}

func (*LBFGS) Needs() struct {
	Gradient bool
	Hessian  bool
} {
	return struct {
		Gradient bool
		Hessian  bool
	}{true, false}
}
<commit_msg>Remove y and s fields from LBFGS<commit_after>package optimize

import (
	"github.com/gonum/floats"
)

// LBFGS implements the limited-memory BFGS algorithm. While the normal BFGS algorithm
// makes a full approximation to the inverse hessian, LBFGS instead approximates the
// hessian from the last Store optimization steps. The Store parameter is a tradeoff
// between cost of the method and accuracy of the hessian approximation.
// LBFGS has a cost (both in memory and time) of O(Store * inputDimension).
// Since BFGS has a cost of O(inputDimension^2), LBFGS is more appropriate
// for very large problems. This "forgetful" nature of LBFGS may also make it perform
// better than BFGS for functions with Hessians that vary rapidly spatially.
//
// If Store is 0, Store is defaulted to 15.
// A Linesearcher for LBFGS must satisfy the strong Wolfe conditions at every
// iteration. If Linesearcher == nil, an appropriate default is chosen.
type LBFGS struct {
	Linesearcher Linesearcher
	Store        int // how many past iterations to store

	ls *LinesearchMethod

	dim    int
	oldest int // element of the history slices that is the oldest

	x    []float64 // location at the last major iteration
	grad []float64 // gradient at the last major iteration

	a []float64 // holds cache of hessian updates

	// History
	yHist   [][]float64 // last Store iterations of y
	sHist   [][]float64 // last Store iterations of s
	rhoHist []float64   // last Store iterations of rho
}

func (l *LBFGS) Init(loc *Location) (Operation, error) {
	if l.Linesearcher == nil {
		l.Linesearcher = &Bisection{}
	}
	if l.ls == nil {
		l.ls = &LinesearchMethod{}
	}
	l.ls.Linesearcher = l.Linesearcher
	l.ls.NextDirectioner = l
	return l.ls.Init(loc)
}

func (l *LBFGS) Iterate(loc *Location) (Operation, error) {
	return l.ls.Iterate(loc)
}

func (l *LBFGS) InitDirection(loc *Location, dir []float64) (stepSize float64) {
	dim := len(loc.X)
	l.dim = dim

	if l.Store == 0 {
		l.Store = 15
	}

	l.oldest = l.Store - 1 // the first vector will be put in at 0

	l.x = resize(l.x, dim)
	l.grad = resize(l.grad, dim)
	copy(l.x, loc.X)
	copy(l.grad, loc.Gradient)

	l.a = resize(l.a, l.Store)
	l.rhoHist = resize(l.rhoHist, l.Store)

	if cap(l.yHist) < l.Store {
		n := make([][]float64, l.Store-cap(l.yHist))
		l.yHist = append(l.yHist, n...)
	}
	if cap(l.sHist) < l.Store {
		n := make([][]float64, l.Store-cap(l.sHist))
		l.sHist = append(l.sHist, n...)
	}
	l.yHist = l.yHist[:l.Store]
	l.sHist = l.sHist[:l.Store]
	for i := range l.sHist {
		l.sHist[i] = resize(l.sHist[i], dim)
		for j := range l.sHist[i] {
			l.sHist[i][j] = 0
		}
	}
	for i := range l.yHist {
		l.yHist[i] = resize(l.yHist[i], dim)
		for j := range l.yHist[i] {
			l.yHist[i][j] = 0
		}
	}

	copy(dir, loc.Gradient)
	floats.Scale(-1, dir)

	return 1 / floats.Norm(dir, 2)
}

func (l *LBFGS) NextDirection(loc *Location, dir []float64) (stepSize float64) {
	if len(loc.X) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}
	if len(loc.Gradient) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}
	if len(dir) != l.dim {
		panic("lbfgs: unexpected size mismatch")
	}

	// Update direction. Uses two-loop correction as described in
	// Nocedal, Wright (2006), Numerical Optimization (2nd ed.). Chapter 7, page 178.
	y := l.yHist[l.oldest]
	floats.SubTo(y, loc.Gradient, l.grad)
	s := l.sHist[l.oldest]
	floats.SubTo(s, loc.X, l.x)
	sDotY := floats.Dot(s, y)
	l.rhoHist[l.oldest] = 1 / sDotY

	l.oldest = (l.oldest + 1) % l.Store

	copy(l.x, loc.X)
	copy(l.grad, loc.Gradient)
	copy(dir, loc.Gradient)

	// two loop update. First loop starts with the most recent element
	// and goes backward, second starts with the oldest element and goes
	// forward. At the end have computed H^-1 * g, so flip the direction for
	// minimization.
	for i := 0; i < l.Store; i++ {
		idx := l.oldest - i - 1
		if idx < 0 {
			idx += l.Store
		}
		l.a[idx] = l.rhoHist[idx] * floats.Dot(l.sHist[idx], dir)
		floats.AddScaled(dir, -l.a[idx], l.yHist[idx])
	}

	// Scale the initial Hessian.
	gamma := sDotY / floats.Dot(y, y)
	floats.Scale(gamma, dir)

	for i := 0; i < l.Store; i++ {
		idx := i + l.oldest
		if idx >= l.Store {
			idx -= l.Store
		}
		beta := l.rhoHist[idx] * floats.Dot(l.yHist[idx], dir)
		floats.AddScaled(dir, l.a[idx]-beta, l.sHist[idx])
	}
	floats.Scale(-1, dir)

	return 1
}

func (*LBFGS) Needs() struct {
	Gradient bool
	Hessian  bool
} {
	return struct {
		Gradient bool
		Hessian  bool
	}{true, false}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tools

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/spf13/cobra"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	defaultCRIProxySockLocation = "/run/criproxy.sock"
	sysCheckNamespace           = "kube-system"
)

type validateCommand struct {
	client KubeClient
	out    io.Writer
}

// NewValidateCommand returns a cobra.Command that validates a cluster readines
// for Virtlet deploy
func NewValidateCommand(client KubeClient, out io.Writer) *cobra.Command {
	v := &validateCommand{client: client, out: out}
	cmd := &cobra.Command{
		Use:   "validate",
		Short: "Validate cluster readiness for Virtlet deployment",
		Long:  "Check configuration of cluster nodes valiating their readiness for Virtlet deployment",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 0 {
				return errors.New("This command does not accept arguments")
			}
			return v.Run()
		},
	}
	return cmd
}

func (v *validateCommand) Run() error {
	nodes, err := v.client.GetVirtletNodeNames()
	if err != nil {
		return err
	}

	if len(nodes) == 0 {
		return errors.New("There are no nodes with label extraRuntime=virtlet")
	}

	v.info("Nodes labeled with extraRuntime=virtlet: %s", strings.Join(nodes, ", "))

	pods, errs := v.prepareSysCheckPods(nodes)
	defer v.deleteSysCheckPods(pods)
	for _, errstr := range errs {
		v.info(errstr)
	}

	if len(pods) == 0 {
		return errors.New("Could not create system check pods on any Virtlet node")
	}

	errsNumber := v.checkCNI(pods)
	errsNumber += v.checkCRIProxy(pods)
	errsNumber += v.checkKubeletArgs(pods)

	if errsNumber != 0 {
		return fmt.Errorf("Collected %d errors while running SysCheck pods", errsNumber)
	} else {
		v.info("No errors found with")
	}

	return nil
}

func (v *validateCommand) prepareSysCheckPods(nodes []string) (pods []*v1.Pod, errs []string) {
	// TODO: this whole part should be running in a timeouted context
	// TODO: paralelize pods creation
	hostPathType := v1.HostPathDirectory
	var definedPods []*v1.Pod
	for _, name := range nodes {
		v.info("Creating syscheck pod on node %q", name)
		pod, err := v.client.CreatePod(&v1.Pod{
			ObjectMeta: meta_v1.ObjectMeta{
				Name:      "virtletsyscheck-" + name,
				Namespace: sysCheckNamespace,
			},
			Spec: v1.PodSpec{
				Volumes: []v1.Volume{
					{
						Name: "hostfs",
						VolumeSource: v1.VolumeSource{
							HostPath: &v1.HostPathVolumeSource{
								Path: "/",
								Type: &hostPathType,
							},
						},
					},
				},
				Containers: []v1.Container{
					{
						Name:    "syscheck",
						Image:   "busybox",
						Command: []string{"/bin/sh", "-c", "--"},
						Args:    []string{"trap : TERM INT; (while true; do sleep 1000; done) & wait"},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "hostfs",
								MountPath: "/mnt",
								ReadOnly:  true,
							},
						},
					},
				},
				NodeSelector: map[string]string{"kubernetes.io/hostname": name},
				HostPID:      true,
			},
		})
		if err != nil {
			errs = append(errs, fmt.Sprintf("SysCheck pod creation failed on node %q: %v", name, err))
		} else {
			definedPods = append(definedPods, pod)
		}
	}

	var wg sync.WaitGroup
	wg.Add(len(definedPods))
	for _, def := range definedPods {
		go func() {
			for {
				// TODO: add checking for possible container starting failure, e.g. when there was an error while
				// downloading container image
				if pod, err := v.client.GetPod(def.Name, sysCheckNamespace); err != nil {
					errs = append(errs, fmt.Sprintf("Failure during SysCheck pod %q status checking: %v", def.Name, err))
					break
				} else if pod.Status.Phase == v1.PodRunning {
					pods = append(pods, pod)
					break
				}
				time.Sleep(250 * time.Millisecond)
			}
			wg.Done()
		}()
	}
	wg.Wait()
	v.info("SysCheck pods on all Virtlet nodes are running")

	return
}

func (v *validateCommand) info(fmtstring string, a ...interface{}) {
	fmt.Fprintf(v.out, fmtstring+"\n", a...)
}

func (v *validateCommand) deleteSysCheckPods(pods []*v1.Pod) {
	for _, pod := range pods {
		if err := v.client.DeletePod(pod.Name, sysCheckNamespace); err != nil {
			v.info("Error during removal of SysCheck pod %q/%q: %v", sysCheckNamespace, pod.Name, err)
		}
	}
}

func doInAllPods(pods []*v1.Pod, check func(*v1.Pod) int) int {
	// TODO: this func should use timeouting context

	var wg sync.WaitGroup
	wg.Add(len(pods))

	errsNumber := 0
	for _, pod := range pods {
		go func() {
			errsNumber += check(pod)
			wg.Done()
		}()
	}

	wg.Wait()
	return errsNumber
}

func (v *validateCommand) chekcInAllSysChecks(pods []*v1.Pod, description, command string, check func(nodeName, out string) int) int {
	return doInAllPods(pods, func(pod *v1.Pod) int {
		errsNumber := 0
		var out bytes.Buffer
		_, err := v.client.ExecInContainer(
			pod.Name, "syscheck", pod.Namespace, nil, bufio.NewWriter(&out), nil,
			[]string{
				"/bin/sh", "-c",
				command,
			},
		)
		if err != nil {
			v.info("Error during verification of %s on node %q: %v", description, pod.Spec.NodeName, err)
			errsNumber += 1
		}

		return errsNumber + check(pod.Spec.NodeName, strings.TrimRight(out.String(), "\r\n"))
	})
}

func (v *validateCommand) checkCNI(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "CNI configuration",
		"find /mnt/etc/cni/net.d -name \"*.conf\" -o -name \"*.conflist\" -o -name \"*.json\" | wc -l",
		func(nodeName, out string) int {
			errsNumber := 0
			if i, err := strconv.Atoi(out); err != nil {
				v.info("Internal error during conunting CNI configuration files on %q: %v", nodeName, err)
				errsNumber += 1
			} else if i == 0 {
				v.info("Node %q does not have any CNI configuration in /etc/cni/net.d", nodeName)
				errsNumber += 1
			}
			return errsNumber
		},
	)
}

func (v *validateCommand) checkCRIProxy(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "CRI Proxy",
		"pgrep criproxy | while read pid ; do cat /proc/$pid/cmdline ; done",
		func(nodeName, out string) int {
			errsNumber := 0
			if len(out) == 0 {
				v.info("Node %q does not have CRI Proxy running", nodeName)
				errsNumber += 1
			} else if !strings.Contains(out, defaultCRIProxySockLocation) {
				v.info("CRI Proxy on node %q does not have %q as socket location", nodeName, defaultCRIProxySockLocation)
				errsNumber += 1
			}
			return errsNumber
		},
	)
}

func (v *validateCommand) checkKubeletArgs(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "kubelet configuration",
		"( pgrep kubelet ; pgrep hyperkube ) | while read pid ; do cat /proc/$pid/cmdline ; done",
		func(nodeName, out string) int {
			errsNumber := 0
			if len(out) == 0 {
				v.info("Internal error - kubelet not found on node %q", nodeName)
				errsNumber += 1
			} else {
				for _, arg := range []string{
					"--container-runtime=remote",
					"--container-runtime-endpoint=unix:///run/criproxy.sock",
					"--image-service-endpoint=unix:///run/criproxy.sock",
					"--enable-controller-attach-detach=false",
				} {
					if !strings.Contains(out, arg) {
						v.info("kubelet on node %q is missing %q option", nodeName, arg)
						errsNumber += 1
					}
				}
			}
			return errsNumber
		},
	)
}
<commit_msg>Fix codeclimate issues<commit_after>/*
Copyright 2019 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tools

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/spf13/cobra"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	defaultCRIProxySockLocation = "/run/criproxy.sock"
	sysCheckNamespace           = "kube-system"
)

type validateCommand struct {
	client KubeClient
	out    io.Writer
}

// NewValidateCommand returns a cobra.Command that validates a cluster readines
// for Virtlet deploy
func NewValidateCommand(client KubeClient, out io.Writer) *cobra.Command {
	v := &validateCommand{client: client, out: out}
	cmd := &cobra.Command{
		Use:   "validate",
		Short: "Validate cluster readiness for Virtlet deployment",
		Long:  "Check configuration of cluster nodes valiating their readiness for Virtlet deployment",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 0 {
				return errors.New("This command does not accept arguments")
			}
			return v.Run()
		},
	}
	return cmd
}

func (v *validateCommand) Run() error {
	nodes, err := v.client.GetVirtletNodeNames()
	if err != nil {
		return err
	}

	if len(nodes) == 0 {
		return errors.New("There are no nodes with label extraRuntime=virtlet")
	}

	v.info("Nodes labeled with extraRuntime=virtlet: %s", strings.Join(nodes, ", "))

	pods, errs := v.prepareSysCheckPods(nodes)
	defer v.deleteSysCheckPods(pods)
	for _, errstr := range errs {
		v.info(errstr)
	}

	if len(pods) == 0 {
		return errors.New("Could not create system check pods on any Virtlet node")
	}

	errsNumber := v.checkCNI(pods)
	errsNumber += v.checkCRIProxy(pods)
	errsNumber += v.checkKubeletArgs(pods)

	if errsNumber != 0 {
		return fmt.Errorf("Collected %d errors while running SysCheck pods", errsNumber)
	}
	v.info("No errors found with")

	return nil
}

func (v *validateCommand) prepareSysCheckPods(nodes []string) (pods []*v1.Pod, errs []string) {
	// TODO: this whole part should be running in a timeouted context
	// TODO: paralelize pods creation
	hostPathType := v1.HostPathDirectory
	var definedPods []*v1.Pod
	for _, name := range nodes {
		v.info("Creating syscheck pod on node %q", name)
		pod, err := v.client.CreatePod(&v1.Pod{
			ObjectMeta: meta_v1.ObjectMeta{
				Name:      "virtletsyscheck-" + name,
				Namespace: sysCheckNamespace,
			},
			Spec: v1.PodSpec{
				Volumes: []v1.Volume{
					{
						Name: "hostfs",
						VolumeSource: v1.VolumeSource{
							HostPath: &v1.HostPathVolumeSource{
								Path: "/",
								Type: &hostPathType,
							},
						},
					},
				},
				Containers: []v1.Container{
					{
						Name:    "syscheck",
						Image:   "busybox",
						Command: []string{"/bin/sh", "-c", "--"},
						Args:    []string{"trap : TERM INT; (while true; do sleep 1000; done) & wait"},
						VolumeMounts: []v1.VolumeMount{
							{
								Name:      "hostfs",
								MountPath: "/mnt",
								ReadOnly:  true,
							},
						},
					},
				},
				NodeSelector: map[string]string{"kubernetes.io/hostname": name},
				HostPID:      true,
			},
		})
		if err != nil {
			errs = append(errs, fmt.Sprintf("SysCheck pod creation failed on node %q: %v", name, err))
		} else {
			definedPods = append(definedPods, pod)
		}
	}

	var wg sync.WaitGroup
	wg.Add(len(definedPods))
	for _, def := range definedPods {
		go func(podDef *v1.Pod) {
			for {
				// TODO: add checking for possible container starting failure, e.g. when there was an error while
				// downloading container image
				if pod, err := v.client.GetPod(podDef.Name, sysCheckNamespace); err != nil {
					errs = append(errs, fmt.Sprintf("Failure during SysCheck pod %q status checking: %v", podDef.Name, err))
					break
				} else if pod.Status.Phase == v1.PodRunning {
					pods = append(pods, pod)
					break
				}
				time.Sleep(250 * time.Millisecond)
			}
			wg.Done()
		}(def)
	}
	wg.Wait()
	v.info("SysCheck pods on all Virtlet nodes are running")

	return
}

func (v *validateCommand) info(fmtstring string, a ...interface{}) {
	fmt.Fprintf(v.out, fmtstring+"\n", a...)
}

func (v *validateCommand) deleteSysCheckPods(pods []*v1.Pod) {
	for _, pod := range pods {
		if err := v.client.DeletePod(pod.Name, sysCheckNamespace); err != nil {
			v.info("Error during removal of SysCheck pod %q/%q: %v", sysCheckNamespace, pod.Name, err)
		}
	}
}

func doInAllPods(pods []*v1.Pod, check func(*v1.Pod) int) int {
	// TODO: this func should use timeouting context

	var wg sync.WaitGroup
	wg.Add(len(pods))

	errsNumber := 0
	for _, pod := range pods {
		go func(pod_ *v1.Pod) {
			errsNumber += check(pod_)
			wg.Done()
		}(pod)
	}

	wg.Wait()
	return errsNumber
}

func (v *validateCommand) chekcInAllSysChecks(pods []*v1.Pod, description, command string, check func(nodeName, out string) int) int {
	return doInAllPods(pods, func(pod *v1.Pod) int {
		errsNumber := 0
		var out bytes.Buffer
		_, err := v.client.ExecInContainer(
			pod.Name, "syscheck", pod.Namespace, nil, bufio.NewWriter(&out), nil,
			[]string{
				"/bin/sh", "-c",
				command,
			},
		)
		if err != nil {
			v.info("Error during verification of %s on node %q: %v", description, pod.Spec.NodeName, err)
			errsNumber++
		}

		return errsNumber + check(pod.Spec.NodeName, strings.TrimRight(out.String(), "\r\n"))
	})
}

func (v *validateCommand) checkCNI(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "CNI configuration",
		"find /mnt/etc/cni/net.d -name \"*.conf\" -o -name \"*.conflist\" -o -name \"*.json\" | wc -l",
		func(nodeName, out string) int {
			errsNumber := 0
			if i, err := strconv.Atoi(out); err != nil {
				v.info("Internal error during conunting CNI configuration files on %q: %v", nodeName, err)
				errsNumber++
			} else if i == 0 {
				v.info("Node %q does not have any CNI configuration in /etc/cni/net.d", nodeName)
				errsNumber++
			}
			return errsNumber
		},
	)
}

func (v *validateCommand) checkCRIProxy(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "CRI Proxy",
		"pgrep criproxy | while read pid ; do cat /proc/$pid/cmdline ; done",
		func(nodeName, out string) int {
			errsNumber := 0
			if len(out) == 0 {
				v.info("Node %q does not have CRI Proxy running", nodeName)
				errsNumber++
			} else if !strings.Contains(out, defaultCRIProxySockLocation) {
				v.info("CRI Proxy on node %q does not have %q as socket location", nodeName, defaultCRIProxySockLocation)
				errsNumber++
			}
			return errsNumber
		},
	)
}

func (v *validateCommand) checkKubeletArgs(pods []*v1.Pod) int {
	return v.chekcInAllSysChecks(
		pods, "kubelet configuration",
		"( pgrep kubelet ; pgrep hyperkube ) | while read pid ; do cat /proc/$pid/cmdline ; done",
		func(nodeName, out string) int {
			errsNumber := 0
			if len(out) == 0 {
				v.info("Internal error - kubelet not found on node %q", nodeName)
				errsNumber++
			} else {
				for _, arg := range []string{
					"--container-runtime=remote",
					"--container-runtime-endpoint=unix:///run/criproxy.sock",
					"--image-service-endpoint=unix:///run/criproxy.sock",
					"--enable-controller-attach-detach=false",
				} {
					if !strings.Contains(out, arg) {
						v.info("kubelet on node %q is missing %q option", nodeName, arg)
						errsNumber++
					}
				}
			}
			return errsNumber
		},
	)
}
<|endoftext|>


<commit_before>package util

import (
	"os"
	"strings"

	"sigs.k8s.io/controller-runtime/pkg/client/config"

	"k8s.io/apimachinery/pkg/api/errors"

	routev1 "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1"
	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)

var (
	openshift *bool
	log       = logf.Log.WithName("util")
)

func IsOpenshift() bool {
	if openshift == nil {
		b := detectOpenshift()
		openshift = &b
	}
	return *openshift
}

func detectOpenshift() bool {

	log.Info("Detect if openshift is running")

	value, ok := os.LookupEnv("ENMASSE_OPENSHIFT")
	if ok {
		log.Info("Set by env-var 'ENMASSE_OPENSHIFT': " + value)
		return strings.ToLower(value) == "true"
	}

	// try to

	cfg, err := config.GetConfig()
	if err != nil {
		log.Error(err, "Error getting config: %v")
		return false
	}

	routeClient, err := routev1.NewForConfig(cfg)
	if err != nil {
		log.Error(err, "Failed to get routeClient")
		return false
	}

	_, err = routeClient.RESTClient().Get().DoRaw()
	if err == nil {
		log.Info("Rest call succeeded")
		return true
	}

	se, ok := err.(*errors.StatusError)
	if !ok {
		log.Info("Result is not a StatusError")
		return false
	}

	code := se.ErrStatus.Code
	log.Info("OpenShift detect", "code", code)
	return true

}
<commit_msg>Fix detection of OpenShift<commit_after>package util

import (
	"fmt"
	"os"
	"strings"

	"sigs.k8s.io/controller-runtime/pkg/client/config"

	routev1 "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1"
	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)

var (
	openshift *bool
	log       = logf.Log.WithName("util")
)

func IsOpenshift() bool {
	if openshift == nil {
		b := detectOpenshift()
		openshift = &b
	}
	return *openshift
}

func detectOpenshift() bool {

	log.Info("Detect if openshift is running")

	value, ok := os.LookupEnv("ENMASSE_OPENSHIFT")
	if ok {
		log.Info("Set by env-var 'ENMASSE_OPENSHIFT': " + value)
		return strings.ToLower(value) == "true"
	}

	// try to

	cfg, err := config.GetConfig()
	if err != nil {
		log.Error(err, "Error getting config: %v")
		return false
	}

	routeClient, err := routev1.NewForConfig(cfg)
	if err != nil {
		log.Error(err, "Failed to get routeClient")
		return false
	}

	body, err := routeClient.RESTClient().Get().DoRaw()

	log.Info(fmt.Sprintf("Request error: %v", err))
	log.V(2).Info(fmt.Sprintf("Body: %v", string(body)))

	return err == nil
}
<|endoftext|>


<commit_before>package cl11

import (
	clw "github.com/rdwilliamson/clw11"
)

type Image struct {
	id clw.Mem

	Format ImageFormat
}

type (
	ChannelOrder clw.ChannelOrder
	ChannelType  clw.ChannelType
)

type ImageFormat struct {
	ChannelOrder ChannelOrder
	ChannelType  ChannelType
}

const (
	R         = ChannelOrder(clw.R)
	A         = ChannelOrder(clw.A)
	RG        = ChannelOrder(clw.RG)
	RA        = ChannelOrder(clw.RA)
	RGB       = ChannelOrder(clw.RGB)
	RGBA      = ChannelOrder(clw.RGBA)
	BGRA      = ChannelOrder(clw.BGRA)
	ARGB      = ChannelOrder(clw.ARGB)
	Intensity = ChannelOrder(clw.Intensity)
	Luminance = ChannelOrder(clw.Luminance)
	Rx        = ChannelOrder(clw.Rx)
	RGx       = ChannelOrder(clw.RGx)
	RGBx      = ChannelOrder(clw.RGBx)
)

const (
	SnormInt8      = ChannelType(clw.SnormInt8)
	SnormInt16     = ChannelType(clw.SnormInt16)
	UnormInt8      = ChannelType(clw.UnormInt8)
	UnormInt16     = ChannelType(clw.UnormInt16)
	UnormShort565  = ChannelType(clw.UnormShort565)
	UnormShort555  = ChannelType(clw.UnormShort555)
	UnormInt101010 = ChannelType(clw.UnormInt101010)
	SignedInt8     = ChannelType(clw.SignedInt8)
	SignedInt16    = ChannelType(clw.SignedInt16)
	SignedInt32    = ChannelType(clw.SignedInt32)
	UnsignedInt8   = ChannelType(clw.UnsignedInt8)
	UnsignedInt16  = ChannelType(clw.UnsignedInt16)
	UnsignedInt32  = ChannelType(clw.UnsignedInt32)
	HalfFloat      = ChannelType(clw.HalfFloat)
	Float          = ChannelType(clw.Float)
)

// Get supported image formats.

func (c *Context) CreateImage2D(flags MemFlags, fmt ImageFormat, width, height, pitch int,
	src interface{}) (*Image, error) {

	mem, err := clw.CreateImage2D(c.id, clw.MemFlags(flags), clw.ImageFormat{clw.ChannelOrder(fmt.ChannelOrder),
		clw.ChannelType(fmt.ChannelType)}, clw.Size(width), clw.Size(height), clw.Size(pitch), nil)
	if err != nil {
		return nil, err
	}

	return &Image{id: mem, Format: fmt}, nil
}
<commit_msg>Adjusting to changes in the wrapper's image format type.<commit_after>package cl11

import (
	clw "github.com/rdwilliamson/clw11"
)

type Image struct {
	id clw.Mem

	Format ImageFormat
}

type (
	ChannelOrder clw.ChannelOrder
	ChannelType  clw.ChannelType
)

type ImageFormat struct {
	ChannelOrder ChannelOrder
	ChannelType  ChannelType
}

const (
	R         = ChannelOrder(clw.R)
	A         = ChannelOrder(clw.A)
	RG        = ChannelOrder(clw.RG)
	RA        = ChannelOrder(clw.RA)
	RGB       = ChannelOrder(clw.RGB)
	RGBA      = ChannelOrder(clw.RGBA)
	BGRA      = ChannelOrder(clw.BGRA)
	ARGB      = ChannelOrder(clw.ARGB)
	Intensity = ChannelOrder(clw.Intensity)
	Luminance = ChannelOrder(clw.Luminance)
	Rx        = ChannelOrder(clw.Rx)
	RGx       = ChannelOrder(clw.RGx)
	RGBx      = ChannelOrder(clw.RGBx)
)

const (
	SnormInt8      = ChannelType(clw.SnormInt8)
	SnormInt16     = ChannelType(clw.SnormInt16)
	UnormInt8      = ChannelType(clw.UnormInt8)
	UnormInt16     = ChannelType(clw.UnormInt16)
	UnormShort565  = ChannelType(clw.UnormShort565)
	UnormShort555  = ChannelType(clw.UnormShort555)
	UnormInt101010 = ChannelType(clw.UnormInt101010)
	SignedInt8     = ChannelType(clw.SignedInt8)
	SignedInt16    = ChannelType(clw.SignedInt16)
	SignedInt32    = ChannelType(clw.SignedInt32)
	UnsignedInt8   = ChannelType(clw.UnsignedInt8)
	UnsignedInt16  = ChannelType(clw.UnsignedInt16)
	UnsignedInt32  = ChannelType(clw.UnsignedInt32)
	HalfFloat      = ChannelType(clw.HalfFloat)
	Float          = ChannelType(clw.Float)
)

// Get supported image formats.

func (c *Context) CreateImage2D(flags MemFlags, fmt ImageFormat, width, height, pitch int,
	src interface{}) (*Image, error) {

	fmt2 := clw.CreateImageFormat(clw.ChannelOrder(fmt.ChannelOrder), clw.ChannelType(fmt.ChannelType))

	mem, err := clw.CreateImage2D(c.id, clw.MemFlags(flags), fmt2, clw.Size(width), clw.Size(height), clw.Size(pitch),
		nil)
	if err != nil {
		return nil, err
	}

	return &Image{id: mem, Format: fmt}, nil
}
<|endoftext|>


<commit_before>package webhook

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"strings"

	"github.com/open-policy-agent/gatekeeper/apis"
	"github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/keys"
	"github.com/open-policy-agent/gatekeeper/pkg/util"
	admissionv1 "k8s.io/api/admission/v1"
	authenticationv1 "k8s.io/api/authentication/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	k8sruntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

type requestResponse string

const (
	successResponse requestResponse = "success"
	errorResponse   requestResponse = "error"
	denyResponse    requestResponse = "deny"
	allowResponse   requestResponse = "allow"
	unknownResponse requestResponse = "unknown"
	skipResponse    requestResponse = "skip"
)

var log = logf.Log.WithName("webhook")

var (
	// VwhName is the metadata.name of the Gatekeeper ValidatingWebhookConfiguration.
	VwhName = "gatekeeper-validating-webhook-configuration"
	// MwhName is the metadata.name of the Gatekeeper MutatingWebhookConfiguration.
	MwhName = "gatekeeper-mutating-webhook-configuration"
)

const (
	serviceAccountName = "gatekeeper-admin"
	mutationsGroup     = "mutations.gatekeeper.sh"
	namespaceKind      = "Namespace"
)

var (
	runtimeScheme                      = k8sruntime.NewScheme()
	codecs                             = serializer.NewCodecFactory(runtimeScheme)
	deserializer                       = codecs.UniversalDeserializer()
	disableEnforcementActionValidation = flag.Bool("disable-enforcementaction-validation", false, "disable validation of the enforcementAction field of a constraint")
	logDenies                          = flag.Bool("log-denies", false, "log detailed info on each deny")
	emitAdmissionEvents                = flag.Bool("emit-admission-events", false, "(alpha) emit Kubernetes events in gatekeeper namespace for each admission violation")
	tlsMinVersion                      = flag.String("tls-min-version", "1.2", "minimum version of TLS supported")
	serviceaccount                     = fmt.Sprintf("system:serviceaccount:%s:%s", util.GetNamespace(), serviceAccountName)
	// webhookName is deprecated, set this on the manifest YAML if needed".
)

func init() {
	_ = apis.AddToScheme(runtimeScheme)
}

func isGkServiceAccount(user authenticationv1.UserInfo) bool {
	return user.Username == serviceaccount
}

type webhookHandler struct {
	client   client.Client
	reporter StatsReporter
	// reader that will be configured to use the API server
	// obtained from mgr.GetAPIReader()
	reader client.Reader
	// for testing
	injectedConfig  *v1alpha1.Config
	processExcluder *process.Excluder
	eventRecorder   record.EventRecorder
	gkNamespace     string
}

func (h *webhookHandler) getConfig(ctx context.Context) (*v1alpha1.Config, error) {
	if h.injectedConfig != nil {
		return h.injectedConfig, nil
	}
	if h.client == nil {
		return nil, errors.New("no client available to retrieve validation config")
	}
	cfg := &v1alpha1.Config{}
	return cfg, h.client.Get(ctx, keys.Config, cfg)
}

// isGatekeeperResource returns true if the request relates to a gatekeeper resource.
func (h *webhookHandler) isGatekeeperResource(req *admission.Request) bool {
	if req.AdmissionRequest.Kind.Group == "templates.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == "constraints.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == mutationsGroup ||
		req.AdmissionRequest.Kind.Group == "config.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == "status.gatekeeper.sh" {
		return true
	}

	return false
}

func (h *webhookHandler) tracingLevel(ctx context.Context, req *admission.Request) (bool, bool) {
	cfg, _ := h.getConfig(ctx)
	traceEnabled := false
	dump := false
	for _, trace := range cfg.Spec.Validation.Traces {
		if trace.User != req.AdmissionRequest.UserInfo.Username {
			continue
		}
		gvk := v1alpha1.GVK{
			Group:   req.AdmissionRequest.Kind.Group,
			Version: req.AdmissionRequest.Kind.Version,
			Kind:    req.AdmissionRequest.Kind.Kind,
		}
		if gvk == trace.Kind {
			traceEnabled = true
			if strings.EqualFold(trace.Dump, "All") {
				dump = true
			}
		}
	}
	return traceEnabled, dump
}

func (h *webhookHandler) skipExcludedNamespace(req *admissionv1.AdmissionRequest, excludedProcess process.Process) (bool, error) {
	obj := &unstructured.Unstructured{}
	if _, _, err := deserializer.Decode(req.Object.Raw, nil, obj); err != nil {
		return false, err
	}

	isNamespaceExcluded, err := h.processExcluder.IsNamespaceExcluded(excludedProcess, obj)
	if err != nil {
		return false, err
	}

	return isNamespaceExcluded, err
}
<commit_msg>update default min TLS to v1.3 (#1866)<commit_after>package webhook

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"strings"

	"github.com/open-policy-agent/gatekeeper/apis"
	"github.com/open-policy-agent/gatekeeper/apis/config/v1alpha1"
	"github.com/open-policy-agent/gatekeeper/pkg/controller/config/process"
	"github.com/open-policy-agent/gatekeeper/pkg/keys"
	"github.com/open-policy-agent/gatekeeper/pkg/util"
	admissionv1 "k8s.io/api/admission/v1"
	authenticationv1 "k8s.io/api/authentication/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	k8sruntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/client"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

type requestResponse string

const (
	successResponse requestResponse = "success"
	errorResponse   requestResponse = "error"
	denyResponse    requestResponse = "deny"
	allowResponse   requestResponse = "allow"
	unknownResponse requestResponse = "unknown"
	skipResponse    requestResponse = "skip"
)

var log = logf.Log.WithName("webhook")

var (
	// VwhName is the metadata.name of the Gatekeeper ValidatingWebhookConfiguration.
	VwhName = "gatekeeper-validating-webhook-configuration"
	// MwhName is the metadata.name of the Gatekeeper MutatingWebhookConfiguration.
	MwhName = "gatekeeper-mutating-webhook-configuration"
)

const (
	serviceAccountName = "gatekeeper-admin"
	mutationsGroup     = "mutations.gatekeeper.sh"
	namespaceKind      = "Namespace"
)

var (
	runtimeScheme                      = k8sruntime.NewScheme()
	codecs                             = serializer.NewCodecFactory(runtimeScheme)
	deserializer                       = codecs.UniversalDeserializer()
	disableEnforcementActionValidation = flag.Bool("disable-enforcementaction-validation", false, "disable validation of the enforcementAction field of a constraint")
	logDenies                          = flag.Bool("log-denies", false, "log detailed info on each deny")
	emitAdmissionEvents                = flag.Bool("emit-admission-events", false, "(alpha) emit Kubernetes events in gatekeeper namespace for each admission violation")
	tlsMinVersion                      = flag.String("tls-min-version", "1.3", "minimum version of TLS supported")
	serviceaccount                     = fmt.Sprintf("system:serviceaccount:%s:%s", util.GetNamespace(), serviceAccountName)
	// webhookName is deprecated, set this on the manifest YAML if needed".
)

func init() {
	_ = apis.AddToScheme(runtimeScheme)
}

func isGkServiceAccount(user authenticationv1.UserInfo) bool {
	return user.Username == serviceaccount
}

type webhookHandler struct {
	client   client.Client
	reporter StatsReporter
	// reader that will be configured to use the API server
	// obtained from mgr.GetAPIReader()
	reader client.Reader
	// for testing
	injectedConfig  *v1alpha1.Config
	processExcluder *process.Excluder
	eventRecorder   record.EventRecorder
	gkNamespace     string
}

func (h *webhookHandler) getConfig(ctx context.Context) (*v1alpha1.Config, error) {
	if h.injectedConfig != nil {
		return h.injectedConfig, nil
	}
	if h.client == nil {
		return nil, errors.New("no client available to retrieve validation config")
	}
	cfg := &v1alpha1.Config{}
	return cfg, h.client.Get(ctx, keys.Config, cfg)
}

// isGatekeeperResource returns true if the request relates to a gatekeeper resource.
func (h *webhookHandler) isGatekeeperResource(req *admission.Request) bool {
	if req.AdmissionRequest.Kind.Group == "templates.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == "constraints.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == mutationsGroup ||
		req.AdmissionRequest.Kind.Group == "config.gatekeeper.sh" ||
		req.AdmissionRequest.Kind.Group == "status.gatekeeper.sh" {
		return true
	}

	return false
}

func (h *webhookHandler) tracingLevel(ctx context.Context, req *admission.Request) (bool, bool) {
	cfg, _ := h.getConfig(ctx)
	traceEnabled := false
	dump := false
	for _, trace := range cfg.Spec.Validation.Traces {
		if trace.User != req.AdmissionRequest.UserInfo.Username {
			continue
		}
		gvk := v1alpha1.GVK{
			Group:   req.AdmissionRequest.Kind.Group,
			Version: req.AdmissionRequest.Kind.Version,
			Kind:    req.AdmissionRequest.Kind.Kind,
		}
		if gvk == trace.Kind {
			traceEnabled = true
			if strings.EqualFold(trace.Dump, "All") {
				dump = true
			}
		}
	}
	return traceEnabled, dump
}

func (h *webhookHandler) skipExcludedNamespace(req *admissionv1.AdmissionRequest, excludedProcess process.Process) (bool, error) {
	obj := &unstructured.Unstructured{}
	if _, _, err := deserializer.Decode(req.Object.Raw, nil, obj); err != nil {
		return false, err
	}

	isNamespaceExcluded, err := h.processExcluder.IsNamespaceExcluded(excludedProcess, obj)
	if err != nil {
		return false, err
	}

	return isNamespaceExcluded, err
}
<|endoftext|>


<commit_before>// Package kvite is a simple embedded K/V store backed by SQLite
package kvite

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/mattn/go-sqlite3" //import sqlite3 for driver
)

type (
	// DB is a wrapper around the underlying SQLite database.
	DB struct {
		db           *sql.DB
		table        string
		putQuery     string
		deleteQuery  string
		getQuery     string
		foreachQuery string
		bucketsQuery string
	}

	// Tx wraps most interactions with the datastore.
	Tx struct {
		db      *DB
		tx      *sql.Tx
		managed bool
	}

	//Bucket represents a collection of key/value pairs inside the database.
	Bucket struct {
		name string
		tx   *Tx
	}
)

// Open opens a KVite datastore. The returned DB is safe for concurrent use by multiple goroutines.
// It is rarely necessary to close a DB.
func Open(filename, table string) (*DB, error) {
	db, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	if table == "" {
		table = "kvite"
	}

	tx, err := db.Begin()
	defer tx.Rollback()
	query := fmt.Sprintf("create TABLE '%s' (key text not null, bucket text not null, value blob not null)", table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}
	query = fmt.Sprintf("create INDEX IF NOT EXISTS '%s_kvite_index' ON '%s' (key, bucket)", table, table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		return nil, err
	}

	return &DB{
		db:           db,
		table:        table,
		getQuery:     fmt.Sprintf("SELECT value FROM '%s' WHERE key = ? and bucket = ?", table),
		deleteQuery:  fmt.Sprintf("DELETE FROM '%s' WHERE key = ? AND bucket = ?", table),
		putQuery:     fmt.Sprintf("INSERT OR REPLACE INTO '%s' (key, value, bucket) VALUES (?, ?, ?)", table),
		foreachQuery: fmt.Sprintf("SELECT key, value FROM '%s' WHERE bucket = ?", table),
		bucketsQuery: fmt.Sprintf("SELECT DISTINCT bucket from '%s'", table),
	}, nil
}

// Close closes the database, releasing any open resources.
// It is rare to Close a DB, as the DB handle is meant to be long-lived and shared between many goroutines.
func (db *DB) Close() error {
	return db.db.Close()
}

// Begin starts a transaction.
func (db *DB) Begin() (*Tx, error) {
	tx, err := db.db.Begin()
	if err != nil {
		return nil, err
	}
	t := &Tx{
		db: db,
		tx: tx,
	}
	return t, nil

}

// Buckets returns all the buckets
func (db *DB) Buckets() ([]string, error) {
	rows, err := db.db.Query(db.bucketsQuery)
	if err != nil {
		return nil, err
	}

	buckets := make([]string, 0, 32)
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		buckets = append(buckets, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return buckets, nil
}

// Transaction executes a function within the context of a  managed transaction.
// If no error is returned from the function then the transaction is committed.
// If an error is returned then the entire transaction is rolled back.
// Rollback and Commit cannot be used inside of the function
func (db *DB) Transaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	// Make sure the transaction rolls back in the event of a panic.
	defer func() {
		if tx.db != nil {
			tx.Rollback()
		}
	}()

	tx.managed = true
	err = fn(tx)
	tx.managed = false
	if err != nil {
		_ = tx.Rollback()
		return err
	}

	return tx.Commit()
}

// Commit commits the transaction.
func (tx *Tx) Commit() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	if tx.db == nil {
		// should we return an error here?
		return nil
	}

	err := tx.tx.Commit()
	tx.db = nil
	return err
}

// Rollback aborts the transaction.
func (tx *Tx) Rollback() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	return tx.tx.Rollback()
}

func (tx *Tx) newBucket(name string) *Bucket {
	return &Bucket{
		tx:   tx,
		name: name,
	}
}

// Bucket gets a bucket by name.  Buckets can be created on the fly and do not "exist" until they have keys.
func (tx *Tx) Bucket(name string) (*Bucket, error) {
	return tx.newBucket(name), nil
}

// CreateBucket is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucket(name string) (*Bucket, error) {
	return tx.Bucket(name)

}

// CreateBucketIfNotExists is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucketIfNotExists(name string) (*Bucket, error) {
	return tx.Bucket(name)
}

// Put sets the value for a key in the bucket. If the key exists, then its previous value will be overwritten.
func (b *Bucket) Put(key string, value []byte) error {
	_, err := b.tx.tx.Exec(b.tx.db.putQuery, key, value, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Delete removes a key from the bucket. If the key does not exist then nothing is done and a nil error is returned.
func (b *Bucket) Delete(key string) error {
	_, err := b.tx.tx.Exec(b.tx.db.deleteQuery, key, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Get retrieves the value for a key in the bucket. Returns a nil value if the key does not exist
func (b *Bucket) Get(key string) ([]byte, error) {
	var value []byte

	if err := b.tx.tx.QueryRow(b.tx.db.getQuery, key, b.name).Scan(&value); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return value, nil
}

//ForEach executes a function for each key/value pair in a bucket. If the provided function returns an error then the iteration is stopped and the error is returned to the caller.
func (b *Bucket) ForEach(fn func(k string, v []byte) error) error {
	rows, err := b.tx.tx.Query(b.tx.db.foreachQuery, b.name)
	if err != nil {
		return err
	}
	for rows.Next() {
		var key string
		var value []byte
		if err := rows.Scan(&key, &value); err != nil {
			return err
		}
		if err := fn(key, value); err != nil {
			return err
		}
	}
	return rows.Err()
}
<commit_msg>if not exists<commit_after>// Package kvite is a simple embedded K/V store backed by SQLite
package kvite

import (
	"database/sql"
	"errors"
	"fmt"
	_ "github.com/mattn/go-sqlite3" //import sqlite3 for driver
)

type (
	// DB is a wrapper around the underlying SQLite database.
	DB struct {
		db           *sql.DB
		table        string
		putQuery     string
		deleteQuery  string
		getQuery     string
		foreachQuery string
		bucketsQuery string
	}

	// Tx wraps most interactions with the datastore.
	Tx struct {
		db      *DB
		tx      *sql.Tx
		managed bool
	}

	//Bucket represents a collection of key/value pairs inside the database.
	Bucket struct {
		name string
		tx   *Tx
	}
)

// Open opens a KVite datastore. The returned DB is safe for concurrent use by multiple goroutines.
// It is rarely necessary to close a DB.
func Open(filename, table string) (*DB, error) {
	db, err := sql.Open("sqlite3", filename)
	if err != nil {
		return nil, err
	}

	if table == "" {
		table = "kvite"
	}

	tx, err := db.Begin()
	defer tx.Rollback()
	query := fmt.Sprintf("create TABLE IF NOT EXISTS '%s' (key text not null, bucket text not null, value blob not null)", table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}
	query = fmt.Sprintf("create INDEX IF NOT EXISTS '%s_kvite_index' ON '%s' (key, bucket)", table, table)
	if _, err := tx.Exec(query); err != nil {
		return nil, err
	}

	if err := tx.Commit(); err != nil {
		return nil, err
	}

	return &DB{
		db:           db,
		table:        table,
		getQuery:     fmt.Sprintf("SELECT value FROM '%s' WHERE key = ? and bucket = ?", table),
		deleteQuery:  fmt.Sprintf("DELETE FROM '%s' WHERE key = ? AND bucket = ?", table),
		putQuery:     fmt.Sprintf("INSERT OR REPLACE INTO '%s' (key, value, bucket) VALUES (?, ?, ?)", table),
		foreachQuery: fmt.Sprintf("SELECT key, value FROM '%s' WHERE bucket = ?", table),
		bucketsQuery: fmt.Sprintf("SELECT DISTINCT bucket from '%s'", table),
	}, nil
}

// Close closes the database, releasing any open resources.
// It is rare to Close a DB, as the DB handle is meant to be long-lived and shared between many goroutines.
func (db *DB) Close() error {
	return db.db.Close()
}

// Begin starts a transaction.
func (db *DB) Begin() (*Tx, error) {
	tx, err := db.db.Begin()
	if err != nil {
		return nil, err
	}
	t := &Tx{
		db: db,
		tx: tx,
	}
	return t, nil

}

// Buckets returns all the buckets
func (db *DB) Buckets() ([]string, error) {
	rows, err := db.db.Query(db.bucketsQuery)
	if err != nil {
		return nil, err
	}

	buckets := make([]string, 0, 32)
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			return nil, err
		}
		buckets = append(buckets, name)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return buckets, nil
}

// Transaction executes a function within the context of a  managed transaction.
// If no error is returned from the function then the transaction is committed.
// If an error is returned then the entire transaction is rolled back.
// Rollback and Commit cannot be used inside of the function
func (db *DB) Transaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	// Make sure the transaction rolls back in the event of a panic.
	defer func() {
		if tx.db != nil {
			tx.Rollback()
		}
	}()

	tx.managed = true
	err = fn(tx)
	tx.managed = false
	if err != nil {
		_ = tx.Rollback()
		return err
	}

	return tx.Commit()
}

// Commit commits the transaction.
func (tx *Tx) Commit() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	if tx.db == nil {
		// should we return an error here?
		return nil
	}

	err := tx.tx.Commit()
	tx.db = nil
	return err
}

// Rollback aborts the transaction.
func (tx *Tx) Rollback() error {
	if tx.managed {
		return errors.New("managed tx commit not allowed")
	}
	return tx.tx.Rollback()
}

func (tx *Tx) newBucket(name string) *Bucket {
	return &Bucket{
		tx:   tx,
		name: name,
	}
}

// Bucket gets a bucket by name.  Buckets can be created on the fly and do not "exist" until they have keys.
func (tx *Tx) Bucket(name string) (*Bucket, error) {
	return tx.newBucket(name), nil
}

// CreateBucket is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucket(name string) (*Bucket, error) {
	return tx.Bucket(name)

}

// CreateBucketIfNotExists is provided for compatibility. It just calls Bucket.
func (tx *Tx) CreateBucketIfNotExists(name string) (*Bucket, error) {
	return tx.Bucket(name)
}

// Put sets the value for a key in the bucket. If the key exists, then its previous value will be overwritten.
func (b *Bucket) Put(key string, value []byte) error {
	_, err := b.tx.tx.Exec(b.tx.db.putQuery, key, value, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Delete removes a key from the bucket. If the key does not exist then nothing is done and a nil error is returned.
func (b *Bucket) Delete(key string) error {
	_, err := b.tx.tx.Exec(b.tx.db.deleteQuery, key, b.name)
	if err != nil {
		return err
	}
	return nil
}

// Get retrieves the value for a key in the bucket. Returns a nil value if the key does not exist
func (b *Bucket) Get(key string) ([]byte, error) {
	var value []byte

	if err := b.tx.tx.QueryRow(b.tx.db.getQuery, key, b.name).Scan(&value); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, err
	}

	return value, nil
}

//ForEach executes a function for each key/value pair in a bucket. If the provided function returns an error then the iteration is stopped and the error is returned to the caller.
func (b *Bucket) ForEach(fn func(k string, v []byte) error) error {
	rows, err := b.tx.tx.Query(b.tx.db.foreachQuery, b.name)
	if err != nil {
		return err
	}
	for rows.Next() {
		var key string
		var value []byte
		if err := rows.Scan(&key, &value); err != nil {
			return err
		}
		if err := fn(key, value); err != nil {
			return err
		}
	}
	return rows.Err()
}
<|endoftext|>


<commit_before>package migrate

import (
	"fmt"
	"strings"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func KeyMigration(store cache.Store, DBFunc func() *gorp.DbMap, u *sdk.User) {
	db := DBFunc()

	// Project migration
	ids, errQ := loadProjectIDs(db)
	if errQ != nil {
		return
	}

	for _, id := range ids {
		migrateProject(db, id, store, u)
	}

	// Application migration
	appIds, errA := loadApplicationIDs(db)
	if errA != nil {
		return
	}
	for _, id := range appIds {
		migrateApplication(db, id, store, u)
	}

	// Environment migration
	envIds, errE := loadEnvironmentIDs(db)
	if errE != nil {
		return
	}
	for _, id := range envIds {
		migrateEnvironment(db, id)
	}
}
func loadProjectIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT project_id from project_variable where var_type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadProjectIDs> Cannot load project ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadProjectIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateProject(db *gorp.DbMap, projID int64, store cache.Store, u *sdk.User) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateProject> Cannot start transaction: %s", errT)
		return errT
	}
	defer tx.Rollback()

	proj, errP := project.LoadByID(tx, store, projID, u, project.LoadOptions.WithLockNoWait, project.LoadOptions.WithVariablesWithClearPassword, project.LoadOptions.WithKeys)
	if errP != nil {
		log.Warning("migrateProject> Cannot get project %d: %s", projID, errP)
		return errP
	}

	// find private key
	keys := findKeyPair(proj.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("proj-%s", k.private.Name)
		found := false
		for _, k := range proj.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			projectKey := sdk.ProjectKey{
				ProjectID: projID,
				Builtin:   false,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeySSHParameter,
				},
			}
			if errK := project.InsertKey(tx, &projectKey); errK != nil {
				log.Warning("migrateProject> Cannot insert project key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}

	return tx.Commit()
}

func loadApplicationIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT application_id from application_variable where var_type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadApplicationIDs> Cannot load application ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadApplicationIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateApplication(db *gorp.DbMap, appID int64, store cache.Store, u *sdk.User) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateApplication> Cannot start transaction: %s", errT)
		return errT
	}

	app, errA := application.LoadAndLockByID(tx, store, appID, u, application.LoadOptions.WithVariablesWithClearPassword, application.LoadOptions.WithKeys)
	if errA != nil {
		log.Warning("migrateApplication> Cannot load application %d: %s", appID, errA)
		return errA
	}
	keys := findKeyPair(app.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("app-%s", k.private.Name)

		found := false
		for _, k := range app.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			appKey := sdk.ApplicationKey{
				ApplicationID: appID,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeySSHParameter,
				},
			}
			if errK := application.InsertKey(tx, &appKey); errK != nil {
				log.Warning("migrateApplication> Cannot insert application key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}
	return tx.Commit()
}

func loadEnvironmentIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT environment_id from environment_variable where type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadEnvironmentIDs> Cannot load environment ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadEnvironmentIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateEnvironment(db *gorp.DbMap, envID int64) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateEnvironment> Cannot start transaction: %s", errT)
		return errT
	}

	if errL := environment.LockByID(tx, envID); errL != nil {
		log.Warning("migrateEnvironment> Cannot lock environment %d: %s", envID, errL)
		return errL
	}
	env, errR := environment.LoadEnvironmentByID(tx, envID)
	if errR != nil {
		log.Warning("migrateEnvironment> Cannot load environment %d: %s", envID, errR)
		return errR
	}
	envVars, errV := environment.GetAllVariableByID(tx, envID, environment.WithClearPassword())
	if errV != nil {
		log.Warning("migrateEnvironment> Cannot load clear password")
	}
	env.Variable = envVars
	keys := findKeyPair(env.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("env-%s", k.private.Name)

		found := false
		for _, k := range env.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			envKey := sdk.EnvironmentKey{
				EnvironmentID: envID,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeySSHParameter,
				},
			}
			if errK := environment.InsertKey(tx, &envKey); errK != nil {
				log.Warning("migrateEnvironment> Cannot insert environment key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}
	return tx.Commit()
}

type keyPair struct {
	private sdk.Variable
	public  sdk.Variable
}

func findKeyPair(vs []sdk.Variable) []keyPair {
	kps := []keyPair{}
	for _, v := range vs {
		if v.Type == sdk.KeyVariable {
			for _, vp := range vs {
				if strings.HasPrefix(vp.Name, v.Name) && strings.HasSuffix(vp.Name, ".pub") {
					kp := keyPair{
						private: v,
						public:  vp,
					}
					kps = append(kps, kp)
					break
				}
			}
		}
	}
	return kps
}
<commit_msg>fix(api): key type (#2311)<commit_after>package migrate

import (
	"fmt"
	"strings"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/project"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

func KeyMigration(store cache.Store, DBFunc func() *gorp.DbMap, u *sdk.User) {
	db := DBFunc()

	// Project migration
	ids, errQ := loadProjectIDs(db)
	if errQ != nil {
		return
	}

	for _, id := range ids {
		migrateProject(db, id, store, u)
	}

	// Application migration
	appIds, errA := loadApplicationIDs(db)
	if errA != nil {
		return
	}
	for _, id := range appIds {
		migrateApplication(db, id, store, u)
	}

	// Environment migration
	envIds, errE := loadEnvironmentIDs(db)
	if errE != nil {
		return
	}
	for _, id := range envIds {
		migrateEnvironment(db, id)
	}
}
func loadProjectIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT project_id from project_variable where var_type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadProjectIDs> Cannot load project ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadProjectIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateProject(db *gorp.DbMap, projID int64, store cache.Store, u *sdk.User) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateProject> Cannot start transaction: %s", errT)
		return errT
	}
	defer tx.Rollback()

	proj, errP := project.LoadByID(tx, store, projID, u, project.LoadOptions.WithLockNoWait, project.LoadOptions.WithVariablesWithClearPassword, project.LoadOptions.WithKeys)
	if errP != nil {
		log.Warning("migrateProject> Cannot get project %d: %s", projID, errP)
		return errP
	}

	// find private key
	keys := findKeyPair(proj.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("proj-%s", k.private.Name)
		found := false
		for _, k := range proj.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			projectKey := sdk.ProjectKey{
				ProjectID: projID,
				Builtin:   false,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeyTypeSSH,
				},
			}
			if errK := project.InsertKey(tx, &projectKey); errK != nil {
				log.Warning("migrateProject> Cannot insert project key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}

	return tx.Commit()
}

func loadApplicationIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT application_id from application_variable where var_type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadApplicationIDs> Cannot load application ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadApplicationIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateApplication(db *gorp.DbMap, appID int64, store cache.Store, u *sdk.User) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateApplication> Cannot start transaction: %s", errT)
		return errT
	}

	app, errA := application.LoadAndLockByID(tx, store, appID, u, application.LoadOptions.WithVariablesWithClearPassword, application.LoadOptions.WithKeys)
	if errA != nil {
		log.Warning("migrateApplication> Cannot load application %d: %s", appID, errA)
		return errA
	}
	keys := findKeyPair(app.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("app-%s", k.private.Name)

		found := false
		for _, k := range app.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			appKey := sdk.ApplicationKey{
				ApplicationID: appID,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeyTypeSSH,
				},
			}
			if errK := application.InsertKey(tx, &appKey); errK != nil {
				log.Warning("migrateApplication> Cannot insert application key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}
	return tx.Commit()
}

func loadEnvironmentIDs(db gorp.SqlExecutor) ([]int64, error) {
	query := `SELECT environment_id from environment_variable where type = 'key'`
	rows, errQ := db.Query(query)
	if errQ != nil {
		log.Warning("loadEnvironmentIDs> Cannot load environment ids: %s", errQ)
		return nil, errQ
	}
	var ids []int64
	defer rows.Close()
	for rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			log.Warning("loadEnvironmentIDs> Cannot key next id: %s", err)
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}

func migrateEnvironment(db *gorp.DbMap, envID int64) error {
	tx, errT := db.Begin()
	if errT != nil {
		log.Warning("migrateEnvironment> Cannot start transaction: %s", errT)
		return errT
	}

	if errL := environment.LockByID(tx, envID); errL != nil {
		log.Warning("migrateEnvironment> Cannot lock environment %d: %s", envID, errL)
		return errL
	}
	env, errR := environment.LoadEnvironmentByID(tx, envID)
	if errR != nil {
		log.Warning("migrateEnvironment> Cannot load environment %d: %s", envID, errR)
		return errR
	}
	envVars, errV := environment.GetAllVariableByID(tx, envID, environment.WithClearPassword())
	if errV != nil {
		log.Warning("migrateEnvironment> Cannot load clear password")
	}
	env.Variable = envVars
	keys := findKeyPair(env.Variable)
	for _, k := range keys {
		keyName := fmt.Sprintf("env-%s", k.private.Name)

		found := false
		for _, k := range env.Keys {
			if k.Name == keyName {
				found = true
				break
			}
		}

		if !found {
			envKey := sdk.EnvironmentKey{
				EnvironmentID: envID,
				Key: sdk.Key{
					Public:  k.public.Value,
					Private: k.private.Value,
					Name:    keyName,
					Type:    sdk.KeyTypeSSH,
				},
			}
			if errK := environment.InsertKey(tx, &envKey); errK != nil {
				log.Warning("migrateEnvironment> Cannot insert environment key %s: %s", k.private.Name, errK)
				return errK
			}
		}
	}
	return tx.Commit()
}

type keyPair struct {
	private sdk.Variable
	public  sdk.Variable
}

func findKeyPair(vs []sdk.Variable) []keyPair {
	kps := []keyPair{}
	for _, v := range vs {
		if v.Type == sdk.KeyVariable {
			for _, vp := range vs {
				if strings.HasPrefix(vp.Name, v.Name) && strings.HasSuffix(vp.Name, ".pub") {
					kp := keyPair{
						private: v,
						public:  vp,
					}
					kps = append(kps, kp)
					break
				}
			}
		}
	}
	return kps
}
<|endoftext|>


<commit_before>/*
 * ZDNS Copyright 2016 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package zdns

import (
	"math/rand"
	"net"
	"sort"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
)

/* Each lookup module registers a single GlobalLookupFactory, which is
 * instantiated once.  This global factory is responsible for providing command
 * line arguments and performing any configuration that should only occur once.
 * For each thread in the worker pool, the framework calls
 * MakePerRoutineFactory(), which should return a second factory, which should
 * perform any "per-thread" initialization. Within each "thread", the framework
 * will then call MakeLookup() for each connection it will make, on which it
 * will call DoLookup().  While two layers of factories is a bit... obnoxious,
 * this allows each module to maintain global, per-thread, and per-connection
 * state.
 *
 * Each layer has access to one proceeding layer (e.g., RoutineLookupFactory
 * knows the GlobalLookupFactory, from which it was created. Therefore, modules
 * should refer to this configuration instead of copying all configuration
 * values for every connection. The Base structs implement these basic
 * pieces of functionality and should be inherited in most situations.
 */

// one Lookup per IP/name/connection ==========================================
//

type Trace []interface{}

type Lookup interface {
	DoLookup(name, nameServer string) (interface{}, Trace, Status, error)
}

type BaseLookup struct {
}

func (base *BaseLookup) DoLookup(name string, class uint16) (interface{}, Status, error) {
	log.Fatal("Unimplemented DoLookup")
	return nil, STATUS_ERROR, nil
}

// one RoutineLookupFactory per goroutine =====================================
//
type RoutineLookupFactory interface {
	MakeLookup() (Lookup, error)
}

// one RoutineLookupFactory per execution =====================================
//
type GlobalLookupFactory interface {
	// TODO: somewhat deceivingly named. This captures the values of flags,
	// but doesn't actually set them.
	SetFlags(flags *pflag.FlagSet)
	// global initialization. Gets called once globally
	// This is called after command line flags have been parsed
	Initialize(conf *GlobalConf) error
	Finalize() error
	// We can't set variables on an interface, so write functions
	// that define any settings for the factory
	AllowStdIn() bool
	// Some modules have Zonefile inputs
	ZonefileInput() bool
	// Help text for the CLI
	Help() string
	// Return a single scanner which will scan a single host
	MakeRoutineFactory(int) (RoutineLookupFactory, error)
	RandomNameServer() string
}

// handle domain input
type InputHandler interface {
	// FeedChannel takes a channel to write domains to, the WaitGroup managing them, and if it's a zonefile input
	FeedChannel(in chan<- interface{}, wg *sync.WaitGroup) error
}

// handle output results
type OutputHandler interface {
	// takes a channel (results) to write the query results to, and the WaitGroup managing the handlers
	WriteResults(results <-chan string, wg *sync.WaitGroup) error
}

type BaseGlobalLookupFactory struct {
	GlobalConf *GlobalConf
}

func (f *BaseGlobalLookupFactory) Initialize(c *GlobalConf) error {
	f.GlobalConf = c
	return nil
}

func (f *BaseGlobalLookupFactory) Finalize() error {
	return nil
}

func (s *BaseGlobalLookupFactory) SetFlags(f *pflag.FlagSet) {
}

func (s *BaseGlobalLookupFactory) Help() string {
	return ""
}

func (f *BaseGlobalLookupFactory) RandomNameServer() string {
	if f.GlobalConf == nil {
		log.Fatal("no global conf initialized")
	}
	l := len(f.GlobalConf.NameServers)
	if l == 0 {
		log.Fatal("No name servers specified")
	}
	return f.GlobalConf.NameServers[rand.Intn(l)]
}

func (f *BaseGlobalLookupFactory) RandomLocalAddr() net.IP {
	if f.GlobalConf == nil {
		log.Fatal("no global conf initialized")
	}
	l := len(f.GlobalConf.LocalAddrs)
	if l == 0 {
		log.Fatal("No local addresses specified")
	}
	return f.GlobalConf.LocalAddrs[rand.Intn(l)]
}

func (s *BaseGlobalLookupFactory) AllowStdIn() bool {
	return true
}

func (s *BaseGlobalLookupFactory) ZonefileInput() bool {
	return false
}

// keep a mapping from name to factory
var lookups map[string]GlobalLookupFactory

// keep a mapping from name to input handler
var inputHandlers map[string]InputHandler

// keep a mapping from name to output handler
var outputHandlers map[string]OutputHandler

func RegisterLookup(name string, s GlobalLookupFactory) {
	if lookups == nil {
		lookups = make(map[string]GlobalLookupFactory, 100)
	}
	lookups[name] = s
}

func ValidlookupsString() string {
	valid := make([]string, len(lookups))
	i := 0
	for k := range lookups {
		valid[i] = k
		i++
	}
	sort.Strings(valid)
	return strings.Join(valid, ", ")
}

func Validlookups() []string {
	valid := make([]string, len(lookups))
	i := 0
	for k := range lookups {
		valid[i] = k
		i++
	}
	sort.Strings(valid)
	return valid
}

func GetLookup(name string) GlobalLookupFactory {
	if factory, ok := lookups[name]; ok {
		return factory
	}
	return nil
}
<commit_msg>update docs<commit_after>/*
 * ZDNS Copyright 2016 Regents of the University of Michigan
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy
 * of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package zdns

import (
	"math/rand"
	"net"
	"sort"
	"strings"
	"sync"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/pflag"
)

/* Each lookup module registers a single GlobalLookupFactory, which is
 * instantiated once.  This global factory is responsible for providing command
 * line arguments and performing any configuration that should only occur once.
 * For each thread in the worker pool, the framework calls
 * MakePerRoutineFactory(), which should return a second factory, which should
 * perform any "per-thread" initialization. Within each "thread", the framework
 * will then call MakeLookup() for each connection it will make, on which it
 * will call DoLookup().  While two layers of factories is a bit... obnoxious,
 * this allows each module to maintain global, per-thread, and per-connection
 * state.
 *
 * Each layer has access to one proceeding layer (e.g., RoutineLookupFactory
 * knows the GlobalLookupFactory, from which it was created. Therefore, modules
 * should refer to this configuration instead of copying all configuration
 * values for every connection. The Base structs implement these basic
 * pieces of functionality and should be inherited in most situations.
 */

// one Lookup per IP/name/connection ==========================================
//

type Trace []interface{}

type Lookup interface {
	DoLookup(name, nameServer string) (interface{}, Trace, Status, error)
}

type BaseLookup struct {
}

func (base *BaseLookup) DoLookup(name string, class uint16) (interface{}, Status, error) {
	log.Fatal("Unimplemented DoLookup")
	return nil, STATUS_ERROR, nil
}

// one RoutineLookupFactory per goroutine =====================================
//
type RoutineLookupFactory interface {
	MakeLookup() (Lookup, error)
}

// one RoutineLookupFactory per execution =====================================
//
type GlobalLookupFactory interface {
	// Capture the values of cobra/viper flags and add them to the
	// global factory as appropriate.
	SetFlags(flags *pflag.FlagSet)
	// global initialization. Gets called once globally
	// This is called after command line flags have been parsed
	Initialize(conf *GlobalConf) error
	Finalize() error
	// We can't set variables on an interface, so write functions
	// that define any settings for the factory
	AllowStdIn() bool
	// Some modules have Zonefile inputs
	ZonefileInput() bool
	// Help text for the CLI
	Help() string
	// Return a single scanner which will scan a single host
	MakeRoutineFactory(int) (RoutineLookupFactory, error)
	RandomNameServer() string
}

// handle domain input
type InputHandler interface {
	// FeedChannel takes a channel to write domains to, the WaitGroup managing them, and if it's a zonefile input
	FeedChannel(in chan<- interface{}, wg *sync.WaitGroup) error
}

// handle output results
type OutputHandler interface {
	// takes a channel (results) to write the query results to, and the WaitGroup managing the handlers
	WriteResults(results <-chan string, wg *sync.WaitGroup) error
}

type BaseGlobalLookupFactory struct {
	GlobalConf *GlobalConf
}

func (f *BaseGlobalLookupFactory) Initialize(c *GlobalConf) error {
	f.GlobalConf = c
	return nil
}

func (f *BaseGlobalLookupFactory) Finalize() error {
	return nil
}

func (s *BaseGlobalLookupFactory) SetFlags(f *pflag.FlagSet) {
}

func (s *BaseGlobalLookupFactory) Help() string {
	return ""
}

func (f *BaseGlobalLookupFactory) RandomNameServer() string {
	if f.GlobalConf == nil {
		log.Fatal("no global conf initialized")
	}
	l := len(f.GlobalConf.NameServers)
	if l == 0 {
		log.Fatal("No name servers specified")
	}
	return f.GlobalConf.NameServers[rand.Intn(l)]
}

func (f *BaseGlobalLookupFactory) RandomLocalAddr() net.IP {
	if f.GlobalConf == nil {
		log.Fatal("no global conf initialized")
	}
	l := len(f.GlobalConf.LocalAddrs)
	if l == 0 {
		log.Fatal("No local addresses specified")
	}
	return f.GlobalConf.LocalAddrs[rand.Intn(l)]
}

func (s *BaseGlobalLookupFactory) AllowStdIn() bool {
	return true
}

func (s *BaseGlobalLookupFactory) ZonefileInput() bool {
	return false
}

// keep a mapping from name to factory
var lookups map[string]GlobalLookupFactory

// keep a mapping from name to input handler
var inputHandlers map[string]InputHandler

// keep a mapping from name to output handler
var outputHandlers map[string]OutputHandler

func RegisterLookup(name string, s GlobalLookupFactory) {
	if lookups == nil {
		lookups = make(map[string]GlobalLookupFactory, 100)
	}
	lookups[name] = s
}

func ValidlookupsString() string {
	valid := make([]string, len(lookups))
	i := 0
	for k := range lookups {
		valid[i] = k
		i++
	}
	sort.Strings(valid)
	return strings.Join(valid, ", ")
}

func Validlookups() []string {
	valid := make([]string, len(lookups))
	i := 0
	for k := range lookups {
		valid[i] = k
		i++
	}
	sort.Strings(valid)
	return valid
}

func GetLookup(name string) GlobalLookupFactory {
	if factory, ok := lookups[name]; ok {
		return factory
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * imgor
 * Copyright (c) 2012 Jimmy Zelinskie
 * Licensed under the MIT license.
 */

package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"text/template"
	"time"
)

// Globals
var (
	uploadTemplate *template.Template
	errorTemplate  *template.Template
	imgdir         string
	templatedir    string
)

// Check for errors
func check(err error) {
	if err != nil {
		panic(err)
	}
}

// Generate a random filename using a sha1 of the image
func generatefilename(d []byte) string {
	sha := sha1.New()
	return fmt.Sprintf("%x", string(sha.Sum(d))[0:10])
}

// MIME Validator
func validateimage(h *multipart.FileHeader) (ext string, err error) {
	mimeArray := h.Header["Content-Type"]
	mime := mimeArray[0]
	if mime == "image/jpeg" {
		ext = ".jpg"
	} else if mime == "image/png" {
		ext = ".png"
	} else {
		err = errors.New("Unsupported filetype uploaded")
	}
	return
}

// Upload Handler
func upload(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		// Load the upload page if they aren't posting an image
		uploadTemplate.Execute(w, nil)
		return
	}

	// Get image from POST
	f, h, err := r.FormFile("image")
	defer f.Close()
	check(err)

	// Check MIME and get a file extension
	ext, err := validateimage(h)
	check(err)

	// Read and write the uploaded file to disk
	filebytes, err := ioutil.ReadAll(f)
	check(err)
	filename := imgdir + generatefilename(filebytes) + ext
	err = ioutil.WriteFile(filename, filebytes, 0744)
	check(err)

	// Redirect to the view page
	http.Redirect(w, r, "/view?id="+filename[6:], http.StatusFound)
}

// View Handler
func view(w http.ResponseWriter, r *http.Request) {
	var filename string
	filename = imgdir + r.FormValue("id")

	if filename[len(filename)-3:] == "jpg" {
		w.Header().Set("Content-Type", "image/jpeg")
	} else if filename[len(filename)-3:] == "png" {
		w.Header().Set("Content-Type", "image/png")
	} else {
		panic(errors.New("No supported filetype specified"))
	}

	// Set expire headers to now+ 1 year
	yearlater := time.Now().AddDate(1, 0, 0)
	w.Header().Set("Expires", yearlater.Format(http.TimeFormat))

	http.ServeFile(w, r, filename)
}

// Error page's variables
type ErrorPage struct {
	Error error
}

// One clean error page
func errorHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				contents := ErrorPage{Error: e}
				w.WriteHeader(http.StatusInternalServerError)
				errorTemplate.Execute(w, contents)
			}
		}()
		fn(w, r)
	}
}

func main() {
	var err error

	// Set imgdir and make sure it exists!
	imgdir = "./img/"
	_ = os.Mkdir(imgdir[2:len(imgdir)-1], 0744)

	// Load up templates and check for errors
	templatedir = "./templates/"
	uploadTemplate, err = template.ParseFiles(templatedir + "upload.html")
	check(err)
	errorTemplate, err = template.ParseFiles(templatedir + "error.html")
	check(err)

	http.HandleFunc("/", errorHandler(upload))
	http.HandleFunc("/view", errorHandler(view))
	http.ListenAndServe(":3000", nil)
}
<commit_msg>changed url structure<commit_after>/*
 * imgor
 * Copyright (c) 2012 Jimmy Zelinskie
 * Licensed under the MIT license.
 */

package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"text/template"
	"time"
)

// Globals
var (
	uploadTemplate *template.Template
	errorTemplate  *template.Template
	imgdir         string
	templatedir    string
)

// Error page's variables
type ErrorPage struct {
	Error error
}

// Check for errors
func check(err error) {
	if err != nil {
		panic(err)
	}
}

// Generate a random filename using a sha1 of the image
func generatefilename(d []byte) string {
	sha := sha1.New()
	return fmt.Sprintf("%x", string(sha.Sum(d))[0:10])
}

// MIME Validator
func validateimage(h *multipart.FileHeader) (ext string, err error) {
	mimeArray := h.Header["Content-Type"]
	mime := mimeArray[0]
	if mime == "image/jpeg" {
		ext = ".jpg"
	} else if mime == "image/png" {
		ext = ".png"
	} else {
		err = errors.New("Unsupported filetype uploaded")
	}
	return
}

// Upload Handler
func upload(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		// Load the upload page if they aren't posting an image
		uploadTemplate.Execute(w, nil)
		return
	}

	// Get image from POST
	f, h, err := r.FormFile("image")
	defer f.Close()
	check(err)

	// Check MIME and get a file extension
	ext, err := validateimage(h)
	check(err)

	// Read and write the uploaded file to disk
	filebytes, err := ioutil.ReadAll(f)
	check(err)
	filename := imgdir + generatefilename(filebytes) + ext
	err = ioutil.WriteFile(filename, filebytes, 0744)
	check(err)

	// Redirect to the view page
	http.Redirect(w, r, "/view/"+filename[6:], http.StatusFound)
}

// View Handler
func view(w http.ResponseWriter, r *http.Request) {
	filename := string(imgdir + r.URL.Path[len("view/"):])

	if filename[len(filename)-3:] == "jpg" {
		w.Header().Set("Content-Type", "image/jpeg")
	} else if filename[len(filename)-3:] == "png" {
		w.Header().Set("Content-Type", "image/png")
	} else {
		panic(errors.New("No supported filetype specified"))
	}

	// Set expire headers to now + 1 year
	yearlater := time.Now().AddDate(1, 0, 0)
	w.Header().Set("Expires", yearlater.Format(http.TimeFormat))

	http.ServeFile(w, r, filename)
}

// One clean error page
func errorHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if e, ok := recover().(error); ok {
				contents := ErrorPage{Error: e}
				w.WriteHeader(http.StatusInternalServerError)
				errorTemplate.Execute(w, contents)
			}
		}()
		fn(w, r)
	}
}

func main() {
	var err error

	// Set imgdir and make sure it exists!
	imgdir = "./img/"
	_ = os.Mkdir(imgdir[2:len(imgdir)-1], 0744)

	// Load up templates and check for errors
	templatedir = "./templates/"
	uploadTemplate, err = template.ParseFiles(templatedir + "upload.html")
	check(err)
	errorTemplate, err = template.ParseFiles(templatedir + "error.html")
	check(err)

	http.HandleFunc("/", errorHandler(upload))
	http.HandleFunc("/view/", errorHandler(view))
	http.ListenAndServe(":3000", nil)
}
<|endoftext|>