<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package selectors

import (
	p "k8s.io/kubectl/pkg/framework/path/predicates"
)

// Interface is a "interface selector". It filters interfaces based on the
// "filtered" predicates.
type Interface interface {
	// Interface can be used as a Interface predicate. If the selector can't
	// select any interface from the interface, then the predicate is
	// false.
	p.Interface

	// SelectFrom finds interfaces from interfaces using this selector. The
	// list can be bigger or smaller than the initial lists,
	// depending on the select criterias.
	SelectFrom(...interface{}) []interface{}

	// AsMap returns a selector that selects Maps from the given
	// values.
	AsMap() Map
	// AsSlice returns a selector that selects Slices from the given
	// values.
	AsSlice() Slice
	// Number returns a selector taht selects Numbers from the given values.
	AsNumber() Number
	// String returns a selector that selects strings from the given values.
	AsString() String

	// Field returns the interface pointed by this specific field in the
	// map. If the field doesn't exist, the value will be filtered
	// out.
	Field(string) Interface
	// FieldP returns all the interfaces pointed by field that match the
	// string predicate. This selector can return more values than
	// it gets (for one map, it can returns multiple sub-values, one
	// for each field that matches the predicate).
	FieldP(...p.String) Interface

	// At returns a selector that select the child at the given
	// index, if the list has such an index. Otherwise, nothing is
	// returned.
	At(index int) Interface
	// AtP returns a selector that selects all the item whose index
	// matches the number predicate. More predicates can be given,
	// they are "and"-ed by this method.
	AtP(ips ...p.Number) Interface
	// Last returns a selector that selects the last value of the
	// list. If the list is empty, then nothing will be selected.
	Last() Interface

	// Children returns a selector that selects the direct children
	// of the given values.
	Children() Interface
	// All returns a selector that selects all direct and indrect
	// children of the given values.
	All() Interface

	// Filter will create a new String that filters only the values
	// who match the predicate.
	Filter(...p.Interface) Interface
}

// Field returns the interface pointed by this specific field in the
// map. If the field doesn't exist, the value will be filtered
// out.
func Field(field string) Interface {
	return FieldP(p.StringEqual(field))
}

// FieldP returns all the interfaces pointed by field that match the
// string predicate. This selector can return more values than
// it gets (for one map, it can returns multiple sub-values, one
// for each field that matches the predicate).
func FieldP(predicates ...p.String) Interface {
	return &interfaceS{vf: interfaceFieldPFilter{sp: p.StringAnd(predicates...)}}
}

// At returns a selector that select the child at the given
// index, if the list has such an index. Otherwise, nothing is
// returned.
func At(index int) Interface {
	return AtP(p.NumberEqual(float64(index)))
}

// AtP returns a selector that selects all the item whose index
// matches the number predicate. More predicates can be given,
// they are "and"-ed by this method.
func AtP(ips ...p.Number) Interface {
	return &interfaceS{vf: interfaceAtPFilter{ip: p.NumberAnd(ips...)}}
}

// Last returns a selector that selects the last value of the
// list. If the list is empty, then nothing will be selected.
func Last() Interface {
	return &interfaceS{vf: interfaceLastFilter{}}
}

// Children selects all the children of the values.
func Children() Interface {
	return &interfaceS{vf: interfaceChildrenFilter{}}
}

// All selects all the direct and indirect childrens of the values.
func All() Interface {
	return &interfaceS{vf: interfaceAllFilter{}}
}

// Filter will only return the values that match the predicate.
func Filter(predicates ...p.Interface) Interface {
	return &interfaceS{vf: &interfaceFilterP{vp: p.InterfaceAnd(predicates...)}}
}

// Interface is a "Interface Selector". It selects a list of values, maps,
// slices, strings, integer from a list of values.
type interfaceS struct {
	vs Interface
	vf interfaceFilter
}

// Match returns true if the selector can find items in the given
// value. Otherwise, it returns false.
func (s *interfaceS) Match(value interface{}) bool {
	return len(s.SelectFrom(value)) != 0
}

func (s *interfaceS) SelectFrom(interfaces ...interface{}) []interface{} {
	if s.vs != nil {
		interfaces = s.vs.SelectFrom(interfaces...)
	}
	return s.vf.SelectFrom(interfaces...)
}

func (s *interfaceS) AsMap() Map {
	return &mapS{vs: s}
}

func (s *interfaceS) AsSlice() Slice {
	return &sliceS{vs: s}
}

func (s *interfaceS) AsNumber() Number {
	return &numberS{vs: s}
}

func (s *interfaceS) AsString() String {
	return &stringS{vs: s}
}

func (s *interfaceS) At(index int) Interface {
	return s.AtP(p.NumberEqual(float64(index)))
}

func (s *interfaceS) AtP(predicates ...p.Number) Interface {
	return &interfaceS{vs: s, vf: interfaceAtPFilter{ip: p.NumberAnd(predicates...)}}
}

func (s *interfaceS) Last() Interface {
	return &interfaceS{vs: s, vf: interfaceLastFilter{}}
}

func (s *interfaceS) Field(key string) Interface {
	return s.FieldP(p.StringEqual(key))
}

func (s *interfaceS) FieldP(predicates ...p.String) Interface {
	return &interfaceS{vs: s, vf: interfaceFieldPFilter{sp: p.StringAnd(predicates...)}}
}

func (s *interfaceS) Children() Interface {
	return &interfaceS{vs: s, vf: interfaceChildrenFilter{}}
}

func (s *interfaceS) All() Interface {
	return &interfaceS{vs: s, vf: interfaceAllFilter{}}
}

func (s *interfaceS) Filter(predicates ...p.Interface) Interface {
	return &interfaceS{vs: s, vf: &interfaceFilterP{vp: p.InterfaceAnd(predicates...)}}
}
<commit_msg>fix some typos<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package selectors

import (
	p "k8s.io/kubectl/pkg/framework/path/predicates"
)

// Interface is a "interface selector". It filters interfaces based on the
// "filtered" predicates.
type Interface interface {
	// Interface can be used as a Interface predicate. If the selector can't
	// select any interface from the interface, then the predicate is
	// false.
	p.Interface

	// SelectFrom finds interfaces from interfaces using this selector. The
	// list can be bigger or smaller than the initial lists,
	// depending on the select criterias.
	SelectFrom(...interface{}) []interface{}

	// AsMap returns a selector that selects Maps from the given
	// values.
	AsMap() Map
	// AsSlice returns a selector that selects Slices from the given
	// values.
	AsSlice() Slice
	// Number returns a selector that selects Numbers from the given values.
	AsNumber() Number
	// String returns a selector that selects strings from the given values.
	AsString() String

	// Field returns the interface pointed by this specific field in the
	// map. If the field doesn't exist, the value will be filtered
	// out.
	Field(string) Interface
	// FieldP returns all the interfaces pointed by field that match the
	// string predicate. This selector can return more values than
	// it gets (for one map, it can returns multiple sub-values, one
	// for each field that matches the predicate).
	FieldP(...p.String) Interface

	// At returns a selector that select the child at the given
	// index, if the list has such an index. Otherwise, nothing is
	// returned.
	At(index int) Interface
	// AtP returns a selector that selects all the item whose index
	// matches the number predicate. More predicates can be given,
	// they are "and"-ed by this method.
	AtP(ips ...p.Number) Interface
	// Last returns a selector that selects the last value of the
	// list. If the list is empty, then nothing will be selected.
	Last() Interface

	// Children returns a selector that selects the direct children
	// of the given values.
	Children() Interface
	// All returns a selector that selects all direct and indrect
	// children of the given values.
	All() Interface

	// Filter will create a new String that filters only the values
	// who match the predicate.
	Filter(...p.Interface) Interface
}

// Field returns the interface pointed by this specific field in the
// map. If the field doesn't exist, the value will be filtered
// out.
func Field(field string) Interface {
	return FieldP(p.StringEqual(field))
}

// FieldP returns all the interfaces pointed by field that match the
// string predicate. This selector can return more values than
// it gets (for one map, it can returns multiple sub-values, one
// for each field that matches the predicate).
func FieldP(predicates ...p.String) Interface {
	return &interfaceS{vf: interfaceFieldPFilter{sp: p.StringAnd(predicates...)}}
}

// At returns a selector that select the child at the given
// index, if the list has such an index. Otherwise, nothing is
// returned.
func At(index int) Interface {
	return AtP(p.NumberEqual(float64(index)))
}

// AtP returns a selector that selects all the item whose index
// matches the number predicate. More predicates can be given,
// they are "and"-ed by this method.
func AtP(ips ...p.Number) Interface {
	return &interfaceS{vf: interfaceAtPFilter{ip: p.NumberAnd(ips...)}}
}

// Last returns a selector that selects the last value of the
// list. If the list is empty, then nothing will be selected.
func Last() Interface {
	return &interfaceS{vf: interfaceLastFilter{}}
}

// Children selects all the children of the values.
func Children() Interface {
	return &interfaceS{vf: interfaceChildrenFilter{}}
}

// All selects all the direct and indirect childrens of the values.
func All() Interface {
	return &interfaceS{vf: interfaceAllFilter{}}
}

// Filter will only return the values that match the predicate.
func Filter(predicates ...p.Interface) Interface {
	return &interfaceS{vf: &interfaceFilterP{vp: p.InterfaceAnd(predicates...)}}
}

// Interface is a "Interface Selector". It selects a list of values, maps,
// slices, strings, integer from a list of values.
type interfaceS struct {
	vs Interface
	vf interfaceFilter
}

// Match returns true if the selector can find items in the given
// value. Otherwise, it returns false.
func (s *interfaceS) Match(value interface{}) bool {
	return len(s.SelectFrom(value)) != 0
}

func (s *interfaceS) SelectFrom(interfaces ...interface{}) []interface{} {
	if s.vs != nil {
		interfaces = s.vs.SelectFrom(interfaces...)
	}
	return s.vf.SelectFrom(interfaces...)
}

func (s *interfaceS) AsMap() Map {
	return &mapS{vs: s}
}

func (s *interfaceS) AsSlice() Slice {
	return &sliceS{vs: s}
}

func (s *interfaceS) AsNumber() Number {
	return &numberS{vs: s}
}

func (s *interfaceS) AsString() String {
	return &stringS{vs: s}
}

func (s *interfaceS) At(index int) Interface {
	return s.AtP(p.NumberEqual(float64(index)))
}

func (s *interfaceS) AtP(predicates ...p.Number) Interface {
	return &interfaceS{vs: s, vf: interfaceAtPFilter{ip: p.NumberAnd(predicates...)}}
}

func (s *interfaceS) Last() Interface {
	return &interfaceS{vs: s, vf: interfaceLastFilter{}}
}

func (s *interfaceS) Field(key string) Interface {
	return s.FieldP(p.StringEqual(key))
}

func (s *interfaceS) FieldP(predicates ...p.String) Interface {
	return &interfaceS{vs: s, vf: interfaceFieldPFilter{sp: p.StringAnd(predicates...)}}
}

func (s *interfaceS) Children() Interface {
	return &interfaceS{vs: s, vf: interfaceChildrenFilter{}}
}

func (s *interfaceS) All() Interface {
	return &interfaceS{vs: s, vf: interfaceAllFilter{}}
}

func (s *interfaceS) Filter(predicates ...p.Interface) Interface {
	return &interfaceS{vs: s, vf: &interfaceFilterP{vp: p.InterfaceAnd(predicates...)}}
}
<|endoftext|>


<commit_before>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate goyacc -p extendedSyntax -o extended_syntax_parser.go extended_syntax_parser.y
package parser

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

const (
	testStandardSelectStmt = `
SELECT employee.age, last_name, salary
FROM   employee
WHERE
  employee.age % 10 < (salary / 10000)
  AND
  strings.Upper(last_name) = "WANG"
LIMIT  100
`
	testTrainSelect = testStandardSelectStmt + `TO TRAIN DNNClassifier
WITH
  n_classes = 3,
  hidden_units = [10, 20]
COLUMN
  employee.name,
  bucketize(last_name, 1000),
  cross(embedding(employee.name), bucketize(last_name, 1000))
LABEL "employee.salary"
INTO sqlflow_models.my_dnn_model
`
	testMultiColumnTrainSelect = testStandardSelectStmt + `TO TRAIN DNNClassifier
WITH
  n_classes = 3,
  hidden_units = [10, 20]
COLUMN
  employee.name,
  bucketize(last_name, 1000),
  cross(embedding(employee.name), bucketize(last_name, 1000))
COLUMN
  cross(embedding(employee.name), bucketize(last_name, 1000)) FOR C2
LABEL employee.salary
INTO sqlflow_models.my_dnn_model;
`
	testPredictSelect = testStandardSelectStmt + `TO PREDICT db.table.field
USING sqlflow_models.my_dnn_model;`

	testMaxcomputeUDFPredict = `
SELECT predict_fun(concat(",", col_1, col_2)) AS (info, score) FROM db.table
TO PREDICT db.predict_result
WITH OSS_KEY=a, OSS_ID=b
USING sqlflow_models.my_model;
	`
)

func TestStandardSelect(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testStandardSelectStmt + ";")
	a.NoError(e)
	a.False(r.Extended)
	a.Equal([]string{"employee.age", "last_name", "salary"},
		r.Fields.Strings())
	a.Equal([]string{"employee"}, r.Tables)
	a.Equal("100", r.limit)
	a.Equal(AND, r.where.Sexp[0].Type)
	a.Equal('<', rune(r.where.Sexp[1].Sexp[0].Type))
	a.Equal('=', rune(r.where.Sexp[2].Sexp[0].Type))
	a.Equal(`employee.age % 10 < (salary / 10000) AND `+
		`strings.Upper(last_name) = "WANG"`,
		r.where.String())
}

func TestTrainParser(t *testing.T) {
	a := assert.New(t)
	// NOTE(tony): Test optional semicolon at the end of the statement
	for _, s := range []string{``, `;`} {
		r, e := parseSQLFlowStmt(testTrainSelect + s)
		a.NoError(e)
		a.True(r.Extended)
		a.True(r.Train)
		a.Equal("DNNClassifier", r.Estimator)
		a.Equal("[10, 20]", r.TrainAttrs["hidden_units"].String())
		a.Equal("3", r.TrainAttrs["n_classes"].String())
		a.Equal(`employee.name`,
			r.Columns["feature_columns"][0].String())
		a.Equal(`bucketize(last_name, 1000)`,
			r.Columns["feature_columns"][1].String())
		a.Equal(
			`cross(embedding(employee.name), bucketize(last_name, 1000))`,
			r.Columns["feature_columns"][2].String())
		a.Equal("employee.salary", r.Label)
		a.Equal("sqlflow_models.my_dnn_model", r.Save)
	}
}

func TestMultiColumnTrainParser(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testMultiColumnTrainSelect)
	a.NoError(e)
	a.True(r.Extended)
	a.True(r.Train)
	a.Equal("DNNClassifier", r.Estimator)
	a.Equal("[10, 20]", r.TrainAttrs["hidden_units"].String())
	a.Equal("3", r.TrainAttrs["n_classes"].String())
	a.Equal(`employee.name`,
		r.Columns["feature_columns"][0].String())
	a.Equal(`bucketize(last_name, 1000)`,
		r.Columns["feature_columns"][1].String())
	a.Equal(
		`cross(embedding(employee.name), bucketize(last_name, 1000))`,
		r.Columns["feature_columns"][2].String())
	a.Equal(
		`cross(embedding(employee.name), bucketize(last_name, 1000))`,
		r.Columns["C2"][0].String())
	a.Equal("employee.salary", r.Label)
	a.Equal("sqlflow_models.my_dnn_model", r.Save)
}

func TestPredictParser(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testPredictSelect)
	a.NoError(e)
	a.True(r.Extended)
	a.False(r.Train)
	a.Equal("sqlflow_models.my_dnn_model", r.Model)
	a.Equal("db.table.field", r.Into)
}

func TestExplainParser(t *testing.T) {
	a := assert.New(t)
	{
		r, e := parseSQLFlowStmt(`select * from mytable
TO EXPLAIN my_model
USING TreeExplainer;`)
		a.NoError(e)
		a.True(r.Extended)
		a.False(r.Train)
		a.True(r.Explain)
		a.Equal("my_model", r.TrainedModel)
		a.Equal("TreeExplainer", r.Explainer)
	}
	{
		r, e := parseSQLFlowStmt(`select * from mytable
TO EXPLAIN my_model
WITH
  plots = force
USING TreeExplainer;`)
		a.NoError(e)
		a.True(r.Extended)
		a.False(r.Train)
		a.True(r.Explain)
		a.Equal("my_model", r.TrainedModel)
		a.Equal("force", r.ExplainAttrs["plots"].String())
		a.Equal("TreeExplainer", r.Explainer)
	}
}

func TestSelectStarAndPrint(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(`SELECT *, b FROM a LIMIT 10;`)
	a.NoError(e)
	a.Equal(2, len(r.Fields.Strings()))
	a.Equal("*", r.Fields.Strings()[0])
	a.False(r.Extended)
	a.False(r.Train)
	a.Equal("SELECT *, b\nFROM a\nLIMIT 10", r.StandardSelect.String())
}

func TestStandardDropTable(t *testing.T) {
	a := assert.New(t)
	_, e := parseSQLFlowStmt(`DROP TABLE TO PREDICT`)
	a.Error(e)
	// Note: currently, our parser doesn't accept anything statements other than SELECT.
	// It will support parsing any SQL statements and even dialects in the future.
}

func TestDuplicatedFrom(t *testing.T) {
	a := assert.New(t)
	_, e := parseSQLFlowStmt(`SELECT table.field FROM table FROM tttt;`)
	a.Error(e)
}

func TestSelectMaxcomputeUDF(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testMaxcomputeUDFPredict)
	a.NoError(e)
	a.Equal(3, len(r.Fields.Strings()))
	a.Equal(r.Fields[0].String(), `predict_fun(concat(",", col_1, col_2))`)
	a.Equal(r.Fields[1].String(), `AS`)
	a.Equal(r.Fields[2].String(), `(info, score)`)
	a.Equal(r.PredictClause.Into, "db.predict_result")
	a.Equal(r.PredAttrs["OSS_KEY"].String(), "a")
	a.Equal(r.PredAttrs["OSS_ID"].String(), "b")
	a.Equal(r.PredictClause.Model, "sqlflow_models.my_model")
}
<commit_msg>Rename some unit tests with longer and precise names (#1471)<commit_after>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate goyacc -p extendedSyntax -o extended_syntax_parser.go extended_syntax_parser.y
package parser

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

const (
	testStandardSelect = `
SELECT employee.age, last_name, salary
FROM   employee
WHERE
  employee.age % 10 < (salary / 10000)
  AND
  strings.Upper(last_name) = "WANG"
LIMIT  100
`
	testToTrain = `TO TRAIN DNNClassifier
WITH
  n_classes = 3,
  hidden_units = [10, 20]
COLUMN
  employee.name,
  bucketize(last_name, 1000),
  cross(embedding(employee.name), bucketize(last_name, 1000))
LABEL "employee.salary"
INTO sqlflow_models.my_dnn_model
`
	testSelectToTrain           = testStandardSelect + testToTrain
	testToTrainWithMultiColumns = `TO TRAIN DNNClassifier
WITH
  n_classes = 3,
  hidden_units = [10, 20]
COLUMN
  employee.name,
  bucketize(last_name, 1000),
  cross(embedding(employee.name), bucketize(last_name, 1000))
COLUMN
  cross(embedding(employee.name), bucketize(last_name, 1000)) FOR C2
LABEL employee.salary
INTO sqlflow_models.my_dnn_model;
`
	testSelectToTrainWithMultiColumns = testStandardSelect + testToTrainWithMultiColumns
	testToPredict                     = `TO PREDICT db.table.field
USING sqlflow_models.my_dnn_model;`
	testSelectToPredict = testStandardSelect + testToPredict

	testSelectToPredictWithMaxComputeUDF = `
SELECT predict_fun(concat(",", col_1, col_2)) AS (info, score) FROM db.table
TO PREDICT db.predict_result
WITH OSS_KEY=a, OSS_ID=b
USING sqlflow_models.my_model;
	`
)

// TODO(wangkuiyi): Remove this test after we remove the rules to
// parse "standard" select.
func TestExtendedSyntaxParseStandardSelect(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testStandardSelect + ";")
	a.NoError(e)
	a.False(r.Extended)
	a.Equal([]string{"employee.age", "last_name", "salary"},
		r.Fields.Strings())
	a.Equal([]string{"employee"}, r.Tables)
	a.Equal("100", r.limit)
	a.Equal(AND, r.where.Sexp[0].Type)
	a.Equal('<', rune(r.where.Sexp[1].Sexp[0].Type))
	a.Equal('=', rune(r.where.Sexp[2].Sexp[0].Type))
	a.Equal(`employee.age % 10 < (salary / 10000) AND `+
		`strings.Upper(last_name) = "WANG"`,
		r.where.String())
}

// TODO(wangkuiyi): Remove this test after we remove the rules to
// parse "standard" select.
func TestExtendedSyntaxParseSelectToTrain(t *testing.T) {
	a := assert.New(t)
	// NOTE(tony): Test optional semicolon at the end of the statement
	for _, s := range []string{``, `;`} {
		r, e := parseSQLFlowStmt(testSelectToTrain + s)
		a.NoError(e)
		a.True(r.Extended)
		a.True(r.Train)
		a.Equal("DNNClassifier", r.Estimator)
		a.Equal("[10, 20]", r.TrainAttrs["hidden_units"].String())
		a.Equal("3", r.TrainAttrs["n_classes"].String())
		a.Equal(`employee.name`,
			r.Columns["feature_columns"][0].String())
		a.Equal(`bucketize(last_name, 1000)`,
			r.Columns["feature_columns"][1].String())
		a.Equal(
			`cross(embedding(employee.name), bucketize(last_name, 1000))`,
			r.Columns["feature_columns"][2].String())
		a.Equal("employee.salary", r.Label)
		a.Equal("sqlflow_models.my_dnn_model", r.Save)
	}
}

// TODO(wangkuiyi): Remove this test after we remove the rules to
// parse "standard" select.
func TestExtendedSyntaxParseSelectToTrainWithMultiColumns(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testSelectToTrainWithMultiColumns)
	a.NoError(e)
	a.True(r.Extended)
	a.True(r.Train)
	a.Equal("DNNClassifier", r.Estimator)
	a.Equal("[10, 20]", r.TrainAttrs["hidden_units"].String())
	a.Equal("3", r.TrainAttrs["n_classes"].String())
	a.Equal(`employee.name`,
		r.Columns["feature_columns"][0].String())
	a.Equal(`bucketize(last_name, 1000)`,
		r.Columns["feature_columns"][1].String())
	a.Equal(
		`cross(embedding(employee.name), bucketize(last_name, 1000))`,
		r.Columns["feature_columns"][2].String())
	a.Equal(
		`cross(embedding(employee.name), bucketize(last_name, 1000))`,
		r.Columns["C2"][0].String())
	a.Equal("employee.salary", r.Label)
	a.Equal("sqlflow_models.my_dnn_model", r.Save)
}

// TODO(wangkuiyi): Remove this test after we remove the rules to
// parse "standard" select.
func TestExtendedSyntaxParseSelectToPredict(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testSelectToPredict)
	a.NoError(e)
	a.True(r.Extended)
	a.False(r.Train)
	a.Equal("sqlflow_models.my_dnn_model", r.Model)
	a.Equal("db.table.field", r.Into)
}

// TODO(wangkuiyi): Remove this test after we remove the rules to
// parse "standard" select.
func TestExtendedSyntaxParseSelectToExplain(t *testing.T) {
	a := assert.New(t)
	{
		r, e := parseSQLFlowStmt(`select * from mytable
TO EXPLAIN my_model
USING TreeExplainer;`)
		a.NoError(e)
		a.True(r.Extended)
		a.False(r.Train)
		a.True(r.Explain)
		a.Equal("my_model", r.TrainedModel)
		a.Equal("TreeExplainer", r.Explainer)
	}
	{
		r, e := parseSQLFlowStmt(`select * from mytable
TO EXPLAIN my_model
WITH
  plots = force
USING TreeExplainer;`)
		a.NoError(e)
		a.True(r.Extended)
		a.False(r.Train)
		a.True(r.Explain)
		a.Equal("my_model", r.TrainedModel)
		a.Equal("force", r.ExplainAttrs["plots"].String())
		a.Equal("TreeExplainer", r.Explainer)
	}
}

func TestExtendedSyntaxParseSelectStarAndPrint(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(`SELECT *, b FROM a LIMIT 10;`)
	a.NoError(e)
	a.Equal(2, len(r.Fields.Strings()))
	a.Equal("*", r.Fields.Strings()[0])
	a.False(r.Extended)
	a.False(r.Train)
	a.Equal("SELECT *, b\nFROM a\nLIMIT 10", r.StandardSelect.String())
}

func TestExtendedSyntaxParseStandardDropTable(t *testing.T) {
	a := assert.New(t)
	_, e := parseSQLFlowStmt(`DROP TABLE TO PREDICT`)
	a.Error(e)
	// Note: currently, our parser doesn't accept anything statements other than SELECT.
	// It will support parsing any SQL statements and even dialects in the future.
}

func TestExtendedSyntaxParseSelectWithDuplicatedFromClauses(t *testing.T) {
	a := assert.New(t)
	_, e := parseSQLFlowStmt(`SELECT table.field FROM table FROM tttt;`)
	a.Error(e)
}

func TestExtendedSyntaxParseSelectToPredictWithMaxcomputeUDF(t *testing.T) {
	a := assert.New(t)
	r, e := parseSQLFlowStmt(testSelectToPredictWithMaxComputeUDF)
	a.NoError(e)
	a.Equal(3, len(r.Fields.Strings()))
	a.Equal(r.Fields[0].String(), `predict_fun(concat(",", col_1, col_2))`)
	a.Equal(r.Fields[1].String(), `AS`)
	a.Equal(r.Fields[2].String(), `(info, score)`)
	a.Equal(r.PredictClause.Into, "db.predict_result")
	a.Equal(r.PredAttrs["OSS_KEY"].String(), "a")
	a.Equal(r.PredAttrs["OSS_ID"].String(), "b")
	a.Equal(r.PredictClause.Model, "sqlflow_models.my_model")
}
<|endoftext|>


<commit_before>package cfpgen

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"

	"reflect"

	"bytes"

	"gopkg.in/yaml.v2"
)

var (
	ErrMissingInFile     = errors.New("Missing required argument -f")
	ErrMissingParameters = errors.New("Parameters not found in file")
)

type Config struct {
	InFile                       string
	OutFile                      string
	Minimize                     bool
	Indent                       int
	Overwrite                    bool
	RemoveOldParamsNotInTemplate bool
	OutYaml                      bool
	InYaml                       bool
	Verbose                      bool
}

// TODO - refactor this into smaller functions
func Generate(c *Config) error {
	flag.Parse()

	if c.InFile == "" {
		return ErrMissingInFile
	}

	m := make(map[string]interface{})
	data, err := ioutil.ReadFile(c.InFile)
	if err != nil {
		return err
	}
	err = unmarshal(data, &m, c)
	if err != nil {
		return err
	}

	if _, ok := m["Parameters"]; !ok {
		return ErrMissingParameters
	}
	params := make(map[string]interface{})
	if c.InYaml {
		RecurseMapInterface(m["Parameters"].(map[interface{}]interface{}), params)
	} else {
		params = m["Parameters"].(map[string]interface{})
	}

	pl := make(paramList, 0)

	for k := range params {
		m := make(map[string]interface{})
		v := params[k]
		switch val := v.(type) {
		case map[interface{}]interface{}:
			RecurseMapInterface(val, m)
		case map[string]interface{}:
			m = val
		}
		p := parameter{ParameterKey: k}
		for k, v := range m {
			switch k {
			case "Default":
				p.Default = v
			case "AllowedValues":
				v, ok := v.([]interface{})
				if ok {
					p.AllowedValues = v
				} else {
					return errors.New("AllowedValues was not an []interface{}")
				}
			case "Description":
				if s, ok := v.(string); ok {
					p.Description = s
				} else {
					return errors.New("Description was not a string")
				}
			case "Type":
				if s, ok := v.(string); ok {
					p.Type = s
				} else {
					return errors.New("Type was not a string")
				}
			case "AllowedPattern":
				if s, ok := v.(string); ok {
					p.AllowedPattern = s
				} else {
					return errors.New("AllowedPattern was not a string")
				}
			}
		}
		pl = append(pl, p)
	}

	for i := range pl {
		var s string
		s = fmt.Sprintf("Type: %s", pl[i].Type)
		if c.Verbose {
			if pl[i].Default != nil {
				s += fmt.Sprintf(", Default: %v", pl[i].Default)
			}
			if pl[i].AllowedValues != nil {
				s += fmt.Sprintf(", AllowedValues: %v", pl[i].AllowedValues)
			}
			if pl[i].AllowedPattern != "" {
				s += fmt.Sprintf(", AllowedPattern: %s", pl[i].AllowedPattern)
			}
			if pl[i].Description != "" {
				s += fmt.Sprintf(", Description: %s", pl[i].Description)
			}
		}
		pl[i].ParameterValue = s
	}

	sort.Sort(pl)
	data, err = marshal(pl, c)
	if err != nil {
		return err
	}
	if c.OutFile != "" {
		if !c.Overwrite {
			odata, err := ioutil.ReadFile(c.OutFile)
			if err == nil {
				// if we have an empty file just write it out
				if len(odata) > 0 {
					// original param list
					opl := make(paramList, 0)
					err = unmarshal(odata, &opl, c)
					if err != nil {
						return err
					}
					//new values from template
					for _, nv := range pl {
						found := false

						// old values from params file
						for i, ov := range opl {
							if nv.ParameterKey == ov.ParameterKey {
								found = true
							}
							if i == len(opl)-1 && !found {
								opl = append(opl, nv)
							}
						}
					}

					if c.RemoveOldParamsNotInTemplate {
						//new values from template
					outer:
						for i := 0; i < len(opl); i++ {
							found := false
							// old values from params file
							for _, nv := range pl {
								if nv.ParameterKey == opl[i].ParameterKey {
									found = true
								}
							}

							if !found {
								fmt.Println("Removing value", opl[i])
								opl = append(opl[:i], opl[i+1:]...)
								i--
								continue outer
							}
						}
					}
					sort.Sort(opl)
					data, err = marshal(opl, c)
					if err != nil {
						return err
					}
				}
			}
		}
		data = unescapeBrackets(data)
		err = ioutil.WriteFile(c.OutFile, data, 0644)
		if err != nil {
			return err
		}
	} else {
		data = unescapeBrackets(data)
		fmt.Print(string(data))
	}
	return nil
}

func unescapeBrackets(data []byte) []byte {
	data = bytes.Replace(data, []byte(`\u003c`), []byte(`<`), -1)
	data = bytes.Replace(data, []byte(`\u003e`), []byte(`>`), -1)
	return data
}

func marshal(i interface{}, c *Config) ([]byte, error) {
	if c.OutYaml {
		return yaml.Marshal(i)
	}
	if c.Minimize {
		return json.Marshal(i)
	}
	return json.MarshalIndent(i, "", strings.Repeat(" ", c.Indent))
}

func unmarshal(data []byte, i interface{}, c *Config) error {
	if c.InYaml {
		return yaml.Unmarshal(data, i)
	}
	return json.Unmarshal(data, i)
}

type parameter struct {
	ParameterKey   string
	ParameterValue string
	Type           string        `json:"-"`
	Description    string        `json:"-"`
	AllowedValues  []interface{} `json:"-"`
	Default        interface{}   `json:"-"`
	AllowedPattern string        `json:"-"`
}

type paramList []parameter

func (p paramList) Less(i, j int) bool {
	return p[i].ParameterKey < p[j].ParameterKey
}

func (p paramList) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p paramList) Len() int {
	return len(p)
}

func RecurseMapInterface(m map[interface{}]interface{}, newMap map[string]interface{}) {
	nm := make(map[string]interface{})
	for k, v := range m {
		nk := ""
		switch t := k.(type) {
		case string:
			nk = t
		case int:
			nk = strconv.Itoa(t)
		case float64:
			nk = strconv.FormatFloat(t, 'E', -1, 64)
		case map[interface{}]interface{}:
			RecurseMapInterface(t, newMap)
		}

		var nv interface{}
		switch t := v.(type) {
		case map[interface{}]interface{}:
			m := make(map[string]interface{})
			nm[nk] = m
			RecurseMapInterface(t, m)
		case []interface{}:
			RecurseArray(nk, t, nm)
		default:
			nv = t
		}
		if nv != nil {
			nm[nk] = nv
		}
	}

	for k, v := range nm {
		newMap[k] = v
	}

}

func RecurseArray(k string, slc []interface{}, container interface{}) {
	nslc := make([]interface{}, 0)
	for _, i := range slc {
		var v interface{}
		switch i := i.(type) {
		case []interface{}:
			RecurseArray(k, i, &nslc)
		case map[interface{}]interface{}:
			m := make(map[string]interface{})
			nslc = append(nslc, m)
			RecurseMapInterface(i, m)
		default:
			v = i
		}
		if v != nil {
			nslc = append(nslc, v)
		}
	}

	rv := reflect.ValueOf(container)
	if rv.Kind() == reflect.Ptr {
		rv = reflect.Indirect(rv)
	}
	switch rv.Kind() {
	case reflect.Slice:
		*container.(*[]interface{}) = append(*container.(*[]interface{}), nslc)
	case reflect.Map:
		m := container.(map[string]interface{})
		m[k] = nslc
	}
}
<commit_msg>refactors and adds comments to methods<commit_after>package cfpgen

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"

	"reflect"

	"bytes"

	"gopkg.in/yaml.v2"
)

var (
	ErrMissingInFile     = errors.New("Missing required argument -f")
	ErrMissingParameters = errors.New("Parameters not found in file")
)

// Config represents a config struct holding information about how to format output and where to write said output
type Config struct {
	InFile                       string
	OutFile                      string
	Minimize                     bool
	Indent                       int
	Overwrite                    bool
	RemoveOldParamsNotInTemplate bool
	OutYaml                      bool
	InYaml                       bool
	Verbose                      bool
}

// Generate generates a cloud formation parameters file template and writes either to a file or stdout
func Generate(c *Config) error {
	flag.Parse()

	if c.InFile == "" {
		return ErrMissingInFile
	}

	m := make(map[string]interface{})
	data, err := ioutil.ReadFile(c.InFile)
	if err != nil {
		return err
	}
	err = c.unmarshal(data, &m)
	if err != nil {
		return err
	}

	if _, ok := m["Parameters"]; !ok {
		return ErrMissingParameters
	}
	params := make(map[string]interface{})
	if c.InYaml {
		RecurseMapInterface(m["Parameters"].(map[interface{}]interface{}), params)
	} else {
		params = m["Parameters"].(map[string]interface{})
	}

	pl, err := c.GetParamList(params)
	if err != nil {
		return err
	}
	sort.Sort(pl)

	return c.ProcessPL(pl)
}

// GetParamList takes the given config and generates a paramList ([]parameter)
func (c *Config) GetParamList(params map[string]interface{}) (paramList, error) {
	pl := make(paramList, 0)

	for k := range params {
		m := make(map[string]interface{})
		v := params[k]
		switch val := v.(type) {
		case map[interface{}]interface{}:
			RecurseMapInterface(val, m)
		case map[string]interface{}:
			m = val
		}
		p := parameter{ParameterKey: k}
		for k, v := range m {
			switch k {
			case "Default":
				p.Default = v
			case "AllowedValues":
				v, ok := v.([]interface{})
				if ok {
					p.AllowedValues = v
				} else {
					return nil, errors.New("AllowedValues was not an []interface{}")
				}
			case "Description":
				if s, ok := v.(string); ok {
					p.Description = s
				} else {
					return nil, errors.New("Description was not a string")
				}
			case "Type":
				if s, ok := v.(string); ok {
					p.Type = s
				} else {
					return nil, errors.New("Type was not a string")
				}
			case "AllowedPattern":
				if s, ok := v.(string); ok {
					p.AllowedPattern = s
				} else {
					return nil, errors.New("AllowedPattern was not a string")
				}
			}
		}
		pl = append(pl, p)
	}

	for i := range pl {
		var s string
		s = fmt.Sprintf("Type: %s", pl[i].Type)
		if c.Verbose {
			if pl[i].Default != nil {
				s += fmt.Sprintf(", Default: %v", pl[i].Default)
			}
			if pl[i].AllowedValues != nil {
				s += fmt.Sprintf(", AllowedValues: %v", pl[i].AllowedValues)
			}
			if pl[i].AllowedPattern != "" {
				s += fmt.Sprintf(", AllowedPattern: %s", pl[i].AllowedPattern)
			}
			if pl[i].Description != "" {
				s += fmt.Sprintf(", Description: %s", pl[i].Description)
			}
		}
		pl[i].ParameterValue = s
	}
	return pl, nil
}

// ProcessPL processes a paramList and writes to a file or stdout
func (c *Config) ProcessPL(pl paramList) error {
	data, err := c.marshal(pl)
	if err != nil {
		return err
	}
	if c.OutFile != "" {
		if !c.Overwrite {
			odata, err := ioutil.ReadFile(c.OutFile)
			if err == nil {
				// if we have an empty file just write it out
				if len(odata) > 0 {
					// original param list
					opl := make(paramList, 0)
					err = c.unmarshal(odata, &opl)
					if err != nil {
						return err
					}
					//new values from template
					for _, nv := range pl {
						found := false

						// old values from params file
						for i, ov := range opl {
							if nv.ParameterKey == ov.ParameterKey {
								found = true
							}
							if i == len(opl)-1 && !found {
								opl = append(opl, nv)
							}
						}
					}

					if c.RemoveOldParamsNotInTemplate {
						//new values from template
					outer:
						for i := 0; i < len(opl); i++ {
							found := false
							// old values from params file
							for _, nv := range pl {
								if nv.ParameterKey == opl[i].ParameterKey {
									found = true
								}
							}

							if !found {
								fmt.Println("Removing value", opl[i])
								opl = append(opl[:i], opl[i+1:]...)
								i--
								continue outer
							}
						}
					}
					sort.Sort(opl)
					data, err = c.marshal(opl)
					if err != nil {
						return err
					}
				}
			}
		}
		data = unescapeBrackets(data)
		err := ioutil.WriteFile(c.OutFile, data, 0644)
		if err != nil {
			return err
		}
	} else {
		data = unescapeBrackets(data)
		fmt.Print(string(data))
	}
	return nil
}

// marshal marshals data in either yaml or json
func (c *Config) marshal(i interface{}) ([]byte, error) {
	if c.OutYaml {
		return yaml.Marshal(i)
	}
	if c.Minimize {
		return json.Marshal(i)
	}
	return json.MarshalIndent(i, "", strings.Repeat(" ", c.Indent))
}

// unmarshal unmarshals data into i
func (c *Config) unmarshal(data []byte, i interface{}) error {
	if c.InYaml {
		return yaml.Unmarshal(data, i)
	}
	return json.Unmarshal(data, i)
}

// unescapeBrackets replaces the unicode points \u003c and \u003e with plaintext brackets - this is designed to be human readable
func unescapeBrackets(data []byte) []byte {
	data = bytes.Replace(data, []byte(`\u003c`), []byte(`<`), -1)
	data = bytes.Replace(data, []byte(`\u003e`), []byte(`>`), -1)
	return data
}

// parameter represents a cloudformation parameter
type parameter struct {
	ParameterKey   string
	ParameterValue string
	Type           string        `json:"-"`
	Description    string        `json:"-"`
	AllowedValues  []interface{} `json:"-"`
	Default        interface{}   `json:"-"`
	AllowedPattern string        `json:"-"`
}

// paramList is a slice of type parameter
type paramList []parameter

// Less implements the sort interface
func (p paramList) Less(i, j int) bool {
	return p[i].ParameterKey < p[j].ParameterKey
}

// Swap implements the sort interface
func (p paramList) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// Len implements the sort interface
func (p paramList) Len() int {
	return len(p)
}

// RecurseArray converts types to supported types. Specifically interfaces stored in slices or the special case map[interface{}]interface{}
// yaml supports keys of arbitrary types whereas json does not, so we do this conversion to maintain compatibility between types
func RecurseArray(k string, slc []interface{}, container interface{}) {
	nslc := make([]interface{}, 0)
	for _, i := range slc {
		var v interface{}
		switch i := i.(type) {
		case []interface{}:
			RecurseArray(k, i, &nslc)
		case map[interface{}]interface{}:
			m := make(map[string]interface{})
			nslc = append(nslc, m)
			RecurseMapInterface(i, m)
		default:
			v = i
		}
		if v != nil {
			nslc = append(nslc, v)
		}
	}

	rv := reflect.ValueOf(container)
	if rv.Kind() == reflect.Ptr {
		rv = reflect.Indirect(rv)
	}
	switch rv.Kind() {
	case reflect.Slice:
		*container.(*[]interface{}) = append(*container.(*[]interface{}), nslc)
	case reflect.Map:
		m := container.(map[string]interface{})
		m[k] = nslc
	}
}

// RecurseMapInterface converts types to supported types. Specifically interfaces stored in slices or the special case map[interface{}]interface{}
// yaml supports keys of arbitrary types whereas json does not, so we do this conversion to maintain compatibility between types
func RecurseMapInterface(m map[interface{}]interface{}, newMap map[string]interface{}) {
	nm := make(map[string]interface{})
	for k, v := range m {
		nk := ""
		switch t := k.(type) {
		case string:
			nk = t
		case int:
			nk = strconv.Itoa(t)
		case float64:
			nk = strconv.FormatFloat(t, 'E', -1, 64)
		case map[interface{}]interface{}:
			RecurseMapInterface(t, newMap)
		}

		var nv interface{}
		switch t := v.(type) {
		case map[interface{}]interface{}:
			m := make(map[string]interface{})
			nm[nk] = m
			RecurseMapInterface(t, m)
		case []interface{}:
			RecurseArray(nk, t, nm)
		default:
			nv = t
		}
		if nv != nil {
			nm[nk] = nv
		}
	}

	for k, v := range nm {
		newMap[k] = v
	}

}
<|endoftext|>


<commit_before>package goa

import (
	"errors"
	"fmt"
	"net"
	"net/url"
	"regexp/syntax"
	"testing"
	"time"
)

func TestValidateFormat(t *testing.T) {
	var (
		validDateTime   = "2015-10-26T08:31:23Z"
		invalidDateTime = "201510-26T08:31:23Z"
		validUUID       = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"
		invalidUUID     = "96054a62-a9e45ed26688389b"
		validEmail      = "raphael@goa.design"
		invalidEmail    = "foo"
		validHostname   = "goa.design"
		invalidHostname = "_hi_"
		validIPv4       = "192.168.0.1"
		invalidIPv4     = "192-168.0.1"
		validIPv6       = "::1"
		invalidIPv6     = "foo"
		validURI        = "hhp://goa.design/contact"
		invalidURI      = "foo_"
		validMAC        = "06-00-00-00-00-00"
		invalidMAC      = "bar"
		validCIDR       = "10.0.0.0/8"
		invalidCIDR     = "foo"
		validRegexp     = "^goa$"
		invalidRegexp   = "foo["
		validRFC1123    = "Mon, 04 Jun 2017 23:52:05 MST"
		invalidRFC1123  = "Mon 04 Jun 2017 23:52:05 MST"
	)
	cases := map[string]struct {
		name     string
		val      string
		format   Format
		expected error
	}{
		"valid date-time":    {"validDateTime", validDateTime, FormatDateTime, nil},
		"invalid date-time":  {"invalidDateTime", invalidDateTime, FormatDateTime, InvalidFormatError("invalidDateTime", invalidDateTime, FormatDateTime, &time.ParseError{time.RFC3339, invalidDateTime, "-", invalidDateTime[4:], ""})},
		"valid uuid":         {"validUUID", validUUID, FormatUUID, nil},
		"invalid uuid":       {"invalidUUID", invalidUUID, FormatUUID, InvalidFormatError("invalidUUID", invalidUUID, FormatUUID, fmt.Errorf("uuid: UUID string too short: %s", invalidUUID))},
		"valid email":        {"validEmail", validEmail, FormatEmail, nil},
		"invalid email":      {"invalidEmail", invalidEmail, FormatEmail, InvalidFormatError("invalidEmail", invalidEmail, FormatEmail, errors.New("mail: no angle-addr"))},
		"valid hostname":     {"validHostname", validHostname, FormatHostname, nil},
		"invalid hostname":   {"invalidHostname", invalidHostname, FormatHostname, InvalidFormatError("invalidHostname", invalidHostname, FormatHostname, fmt.Errorf("hostname value '%s' does not match %s", invalidHostname, `^[[:alnum:]][[:alnum:]\-]{0,61}[[:alnum:]]|[[:alpha:]]$`))},
		"valid ipv4":         {"validIPv4", validIPv4, FormatIPv4, nil},
		"valid ipv6 as ipv4": {"validIPv6", validIPv6, FormatIPv4, InvalidFormatError("validIPv6", validIPv6, FormatIPv4, fmt.Errorf("\"%s\" is an invalid %s value", validIPv6, FormatIPv4))},
		"invalid ipv4":       {"invalidIPv4", invalidIPv4, FormatIPv4, InvalidFormatError("invalidIPv4", invalidIPv4, FormatIPv4, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv4, FormatIPv4))},
		"valid ipv6":         {"validIPv6", validIPv6, FormatIPv6, nil},
		"valid ipv4 as ipv6": {"validIPv4", validIPv4, FormatIPv6, InvalidFormatError("validIPv4", validIPv4, FormatIPv6, fmt.Errorf("\"%s\" is an invalid %s value", validIPv4, FormatIPv6))},
		"invalid ipv6":       {"invalidIPv6", invalidIPv6, FormatIPv6, InvalidFormatError("invalidIPv6", invalidIPv6, FormatIPv6, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv6, FormatIPv6))},
		"valid ipv4 as ip":   {"validIPv4", validIPv4, FormatIP, nil},
		"valid ipv6 as ip":   {"validIPv6", validIPv6, FormatIP, nil},
		"invalid ipv4 as ip": {"invalidIPv4", invalidIPv4, FormatIP, InvalidFormatError("invalidIPv4", invalidIPv4, FormatIP, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv4, FormatIP))},
		"invalid ipv6 as ip": {"invalidIPv6", invalidIPv6, FormatIP, InvalidFormatError("invalidIPv6", invalidIPv6, FormatIP, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv6, FormatIP))},
		"valid uri":          {"validURI", validURI, FormatURI, nil},
		"invalid uri":        {"invalidURI", invalidURI, FormatURI, InvalidFormatError("invalidURI", invalidURI, FormatURI, &url.Error{"parse", invalidURI, errors.New("invalid URI for request")})},
		"valid mac":          {"validMAC", validMAC, FormatMAC, nil},
		"invalid mac":        {"invalidMAC", invalidMAC, FormatMAC, InvalidFormatError("invalidMAC", invalidMAC, FormatMAC, &net.AddrError{Err: "invalid MAC address", Addr: invalidMAC})},
		"valid cidr":         {"validCIDR", validCIDR, FormatCIDR, nil},
		"invalid cidr":       {"invalidCIDR", invalidCIDR, FormatCIDR, InvalidFormatError("invalidCIDR", invalidCIDR, FormatCIDR, &net.ParseError{Type: "CIDR address", Text: invalidCIDR})},
		"valid regexp":       {"validRegexp", validRegexp, FormatRegexp, nil},
		"invalid regexp":     {"invalidRegexp", invalidRegexp, FormatRegexp, InvalidFormatError("invalidRegexp", invalidRegexp, FormatRegexp, &syntax.Error{Code: syntax.ErrMissingBracket, Expr: invalidRegexp[3:4]})},
		"valid rfc1123":      {"validRFC1123", validRFC1123, FormatRFC1123, nil},
		"invalid rfc1123":    {"invalidRFC1123", invalidRFC1123, FormatRFC1123, InvalidFormatError("invalidRFC1123", invalidRFC1123, FormatRFC1123, &time.ParseError{time.RFC1123, invalidRFC1123, ", ", invalidRFC1123[3:], ""})},
	}

	for k, tc := range cases {
		actual := ValidateFormat(tc.name, tc.val, tc.format)
		if actual != tc.expected {
			// Compare only the messages because the error has always a new error ID.
			if actual.Error() != tc.expected.Error() {
				t.Errorf("%s: got %#v, expected %#v", k, actual, tc.expected)
			}
		}
	}
}
<commit_msg>Add tests for goa.ValidatePattern() (#1474)<commit_after>package goa

import (
	"errors"
	"fmt"
	"net"
	"net/url"
	"regexp/syntax"
	"testing"
	"time"
)

func TestValidateFormat(t *testing.T) {
	var (
		validDateTime   = "2015-10-26T08:31:23Z"
		invalidDateTime = "201510-26T08:31:23Z"
		validUUID       = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"
		invalidUUID     = "96054a62-a9e45ed26688389b"
		validEmail      = "raphael@goa.design"
		invalidEmail    = "foo"
		validHostname   = "goa.design"
		invalidHostname = "_hi_"
		validIPv4       = "192.168.0.1"
		invalidIPv4     = "192-168.0.1"
		validIPv6       = "::1"
		invalidIPv6     = "foo"
		validURI        = "hhp://goa.design/contact"
		invalidURI      = "foo_"
		validMAC        = "06-00-00-00-00-00"
		invalidMAC      = "bar"
		validCIDR       = "10.0.0.0/8"
		invalidCIDR     = "foo"
		validRegexp     = "^goa$"
		invalidRegexp   = "foo["
		validRFC1123    = "Mon, 04 Jun 2017 23:52:05 MST"
		invalidRFC1123  = "Mon 04 Jun 2017 23:52:05 MST"
	)
	cases := map[string]struct {
		name     string
		val      string
		format   Format
		expected error
	}{
		"valid date-time":    {"validDateTime", validDateTime, FormatDateTime, nil},
		"invalid date-time":  {"invalidDateTime", invalidDateTime, FormatDateTime, InvalidFormatError("invalidDateTime", invalidDateTime, FormatDateTime, &time.ParseError{time.RFC3339, invalidDateTime, "-", invalidDateTime[4:], ""})},
		"valid uuid":         {"validUUID", validUUID, FormatUUID, nil},
		"invalid uuid":       {"invalidUUID", invalidUUID, FormatUUID, InvalidFormatError("invalidUUID", invalidUUID, FormatUUID, fmt.Errorf("uuid: UUID string too short: %s", invalidUUID))},
		"valid email":        {"validEmail", validEmail, FormatEmail, nil},
		"invalid email":      {"invalidEmail", invalidEmail, FormatEmail, InvalidFormatError("invalidEmail", invalidEmail, FormatEmail, errors.New("mail: no angle-addr"))},
		"valid hostname":     {"validHostname", validHostname, FormatHostname, nil},
		"invalid hostname":   {"invalidHostname", invalidHostname, FormatHostname, InvalidFormatError("invalidHostname", invalidHostname, FormatHostname, fmt.Errorf("hostname value '%s' does not match %s", invalidHostname, `^[[:alnum:]][[:alnum:]\-]{0,61}[[:alnum:]]|[[:alpha:]]$`))},
		"valid ipv4":         {"validIPv4", validIPv4, FormatIPv4, nil},
		"valid ipv6 as ipv4": {"validIPv6", validIPv6, FormatIPv4, InvalidFormatError("validIPv6", validIPv6, FormatIPv4, fmt.Errorf("\"%s\" is an invalid %s value", validIPv6, FormatIPv4))},
		"invalid ipv4":       {"invalidIPv4", invalidIPv4, FormatIPv4, InvalidFormatError("invalidIPv4", invalidIPv4, FormatIPv4, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv4, FormatIPv4))},
		"valid ipv6":         {"validIPv6", validIPv6, FormatIPv6, nil},
		"valid ipv4 as ipv6": {"validIPv4", validIPv4, FormatIPv6, InvalidFormatError("validIPv4", validIPv4, FormatIPv6, fmt.Errorf("\"%s\" is an invalid %s value", validIPv4, FormatIPv6))},
		"invalid ipv6":       {"invalidIPv6", invalidIPv6, FormatIPv6, InvalidFormatError("invalidIPv6", invalidIPv6, FormatIPv6, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv6, FormatIPv6))},
		"valid ipv4 as ip":   {"validIPv4", validIPv4, FormatIP, nil},
		"valid ipv6 as ip":   {"validIPv6", validIPv6, FormatIP, nil},
		"invalid ipv4 as ip": {"invalidIPv4", invalidIPv4, FormatIP, InvalidFormatError("invalidIPv4", invalidIPv4, FormatIP, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv4, FormatIP))},
		"invalid ipv6 as ip": {"invalidIPv6", invalidIPv6, FormatIP, InvalidFormatError("invalidIPv6", invalidIPv6, FormatIP, fmt.Errorf("\"%s\" is an invalid %s value", invalidIPv6, FormatIP))},
		"valid uri":          {"validURI", validURI, FormatURI, nil},
		"invalid uri":        {"invalidURI", invalidURI, FormatURI, InvalidFormatError("invalidURI", invalidURI, FormatURI, &url.Error{"parse", invalidURI, errors.New("invalid URI for request")})},
		"valid mac":          {"validMAC", validMAC, FormatMAC, nil},
		"invalid mac":        {"invalidMAC", invalidMAC, FormatMAC, InvalidFormatError("invalidMAC", invalidMAC, FormatMAC, &net.AddrError{Err: "invalid MAC address", Addr: invalidMAC})},
		"valid cidr":         {"validCIDR", validCIDR, FormatCIDR, nil},
		"invalid cidr":       {"invalidCIDR", invalidCIDR, FormatCIDR, InvalidFormatError("invalidCIDR", invalidCIDR, FormatCIDR, &net.ParseError{Type: "CIDR address", Text: invalidCIDR})},
		"valid regexp":       {"validRegexp", validRegexp, FormatRegexp, nil},
		"invalid regexp":     {"invalidRegexp", invalidRegexp, FormatRegexp, InvalidFormatError("invalidRegexp", invalidRegexp, FormatRegexp, &syntax.Error{Code: syntax.ErrMissingBracket, Expr: invalidRegexp[3:4]})},
		"valid rfc1123":      {"validRFC1123", validRFC1123, FormatRFC1123, nil},
		"invalid rfc1123":    {"invalidRFC1123", invalidRFC1123, FormatRFC1123, InvalidFormatError("invalidRFC1123", invalidRFC1123, FormatRFC1123, &time.ParseError{time.RFC1123, invalidRFC1123, ", ", invalidRFC1123[3:], ""})},
	}

	for k, tc := range cases {
		actual := ValidateFormat(tc.name, tc.val, tc.format)
		if actual != tc.expected {
			// Compare only the messages because the error has always a new error ID.
			if actual.Error() != tc.expected.Error() {
				t.Errorf("%s: got %#v, expected %#v", k, actual, tc.expected)
			}
		}
	}
}

func TestValidatePattern(t *testing.T) {
	var (
		name      = "foo"
		pattern   = "^goa$"
		matched   = "goa"
		unmatched = "foo["
	)
	cases := map[string]struct {
		name     string
		val      string
		pattern  string
		expected error
	}{
		"matched value":   {name, matched, pattern, nil},
		"unmatched value": {name, unmatched, pattern, InvalidPatternError(name, unmatched, pattern)},
	}

	for k, tc := range cases {
		actual := ValidatePattern(tc.name, tc.val, tc.pattern)
		if actual != tc.expected {
			// Compare only the messages because the error has always a new error ID.
			if actual.Error() != tc.expected.Error() {
				t.Errorf("%s: got %#v, expected %#v", k, actual, tc.expected)
			}
		}
	}
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"github.com/BurntSushi/toml"
)

// ValidateConfigFile detect unexpected key in configfile
func ValidateConfigFile(file string) ([]string, error) {
	config := &Config{}
	md, err := toml.DecodeFile(file, config)
	if err != nil {
		return nil, fmt.Errorf("failed to test config: %s", err)
	}

	var unexpectedKey []string = make([]string, 0)
	for _, v := range md.Undecoded() {
		unexpectedKey = append(unexpectedKey, v.String())
	}

	for k1, v := range config.Plugin {
		/*
			detect [plugin.<unexpected>.<unexpected>]
			don't have to detect critical syntax error about plugin here because error should have occured while loading config
			```
			[plugin.metrics.correct]
			```
			-> A configuration value of `command` should be string or string slice, but <nil>
			```
			[plugin.metrics]
			command = "test command"
			```
			-> type mismatch for config.PluginConfig: expected table but found string
		*/
		if k1 != "metrics" && k1 != "checks" && k1 != "metadata" {
			for k2 := range v {
				unexpectedKey = append(unexpectedKey, fmt.Sprintf("plugin.%s.%s", k1, k2))
			}
		}
	}

	return unexpectedKey, nil
}
<commit_msg>unexpectedKey -> unexpectedKeys<commit_after>package config

import (
	"fmt"
	"github.com/BurntSushi/toml"
)

// ValidateConfigFile detect unexpected key in configfile
func ValidateConfigFile(file string) ([]string, error) {
	config := &Config{}
	md, err := toml.DecodeFile(file, config)
	if err != nil {
		return nil, fmt.Errorf("failed to test config: %s", err)
	}

	var unexpectedKeys []string
	for _, v := range md.Undecoded() {
		unexpectedKeys = append(unexpectedKeys, v.String())
	}

	for k1, v := range config.Plugin {
		/*
			detect [plugin.<unexpected>.<unexpected>]
			don't have to detect critical syntax error about plugin here because error should have occured while loading config
			```
			[plugin.metrics.correct]
			```
			-> A configuration value of `command` should be string or string slice, but <nil>
			```
			[plugin.metrics]
			command = "test command"
			```
			-> type mismatch for config.PluginConfig: expected table but found string
		*/
		if k1 != "metrics" && k1 != "checks" && k1 != "metadata" {
			for k2 := range v {
				unexpectedKeys = append(unexpectedKeys, fmt.Sprintf("plugin.%s.%s", k1, k2))
			}
		}
	}

	return unexpectedKeys, nil
}
<|endoftext|>


<commit_before>package console

import (
	"github.com/golang-collections/collections/set"
	"github.com/urfave/cli"
	"errors"
	"net/url"
	"github.com/golang-collections/collections/stack"
)

// Set of visited links
var VisitedLinks = set.New()
// Start analyse URL
var StartUrl *url.URL
// Links for analyse
var LinksStack = stack.New()
// Result links
var ResultLinks = set.New()

func Parse(c *cli.Context) error {
	prepareUrl, prepareErr := parseStartUrl(c.String("url"))
	if prepareErr != nil {
		return prepareErr
	}
	StartUrl = prepareUrl

	LinksStack.Push(Link{
		Link: c.String("url"),
		Source: "",
	})

	for LinksStack.Len() > 0 {
		analyse(LinksStack.Pop().(Link))
	}

	saveResult()

	return nil
}
func parseStartUrl(inputUrl string) (*url.URL, error) {
	if "" == inputUrl {
		return nil, errors.New("URL isn't set.")
	}

	parseUrl, parseErr := url.Parse(inputUrl)
	if parseErr != nil {
		return nil, errors.New("Incorrect URL.")
	}

	return parseUrl, nil
}

func analyse(link Link) {
	if VisitedLinks.Has(link.Link) {
		return
	}
	VisitedLinks.Insert(link.Link);

	parseUrl, parseErr := url.Parse(link.Link)
	if parseErr != nil {
		// TODO: write to log url parse errors
		return
	}

	if parseUrl.Host == StartUrl.Host {
		RunCrawler(parseUrl.String(), link.Source)
	} else {
		ResultLinks.Insert(Page{
			Link: parseUrl.String(),
			Source: link.Source,
			Type: "ext",
			Status: 0,
		})
	}

	return
}

func saveResult() {
	SaveVisitedLinks(VisitedLinks, "visited-links.txt")
	SaveResultLinks(ResultLinks, "output.csv")
}
<commit_msg>refactor to console application<commit_after>package console

import (
	"github.com/golang-collections/collections/set"
	"github.com/urfave/cli"
	"errors"
	"net/url"
	"github.com/golang-collections/collections/stack"
)

// Set of visited links
var VisitedLinks = set.New()
// Start analyse URL
var StartUrl *url.URL
// Links for analyse
var LinksStack = stack.New()
// Result links
var ResultLinks = set.New()

// Start parse site
func Parse(c *cli.Context) error {
	prepareUrl, prepareErr := parseStartUrl(c.String("url"))
	if prepareErr != nil {
		return prepareErr
	}
	StartUrl = prepareUrl

	LinksStack.Push(Link{
		Link: c.String("url"),
		Source: "",
	})

	for LinksStack.Len() > 0 {
		analyse(LinksStack.Pop().(Link))
	}

	saveResult()

	return nil
}
func parseStartUrl(inputUrl string) (*url.URL, error) {
	if "" == inputUrl {
		return nil, errors.New("URL isn't set.")
	}

	parseUrl, parseErr := url.Parse(inputUrl)
	if parseErr != nil {
		return nil, errors.New("Incorrect URL.")
	}

	return parseUrl, nil
}

func analyse(link Link) {
	if VisitedLinks.Has(link.Link) {
		return
	}
	VisitedLinks.Insert(link.Link);

	parseUrl, parseErr := url.Parse(link.Link)
	if parseErr != nil {
		// TODO: write to log url parse errors
		return
	}

	if parseUrl.Host == StartUrl.Host {
		RunCrawler(parseUrl.String(), link.Source)
	} else {
		ResultLinks.Insert(Page{
			Link: parseUrl.String(),
			Source: link.Source,
			Type: "ext",
			Status: 0,
		})
	}

	return
}

func saveResult() {
	SaveVisitedLinks(VisitedLinks, "visited-links.txt")
	SaveResultLinks(ResultLinks, "output.csv")
}
<|endoftext|>


<commit_before>package disco

import (
	"bytes"
	"context"
	"encoding/gob"
	"errors"
	"net"
	"sync"
	"time"

	"gitlab.fg/go/disco/multicast"
	"gitlab.fg/go/disco/node"
)

// Disco represents a list of discovered devices
type Disco struct {
	mu             sync.Mutex      // protects members
	members        []*node.Node    // stores all nodes registered
	discoveredChan chan *node.Node // node.Serve() sends nodes to this chan
}

// Members returns all node in members slice
func (d *Disco) Members() []*node.Node {
	d.mu.Lock()
	defer d.mu.Unlock()

	return d.members
}

// Discover listens for multicast sends and registers any nodes it finds
func (d *Disco) Discover(ctx context.Context, multicastAddress string) (<-chan *node.Node, error) {
	if multicastAddress == "" {
		return nil, errors.New("Address is blank")
	}

	ip, _, err := net.SplitHostPort(multicastAddress)
	if err != nil {
		return nil, err
	}

	if !net.ParseIP(ip).IsMulticast() {
		return nil, errors.New("multicastAddress is not valid")
	}

	if d.discoveredChan == nil {
		d.discoveredChan = make(chan *node.Node)
	}

	results := make(chan *node.Node)

	m := &multicast.Multicast{Address: multicastAddress}
	respChan, err := m.Listen(ctx)
	if err != nil {
		return nil, err
	}

	if d.members == nil {
		d.members = []*node.Node{}
	}

	go func() {
		for {
			select {
			case resp := <-respChan:
				buffer := bytes.NewBuffer(resp.Payload)
				rn := &node.Node{}
				dec := gob.NewDecoder(buffer)
				dec.Decode(rn)
				rn.SrcIP = resp.SrcIP // set the source address
				if d.addToMembers(rn) {
					d.register(results, rn)
				} else {
					if index := d.indexOfMember(rn); index != -1 {
						d.mu.Lock()
						d.members[index].KeepRegistered()
						d.mu.Unlock()
					}
				}
			case <-ctx.Done():
				return
			}

		}
	}()

	return results, nil
}

// register adds newly discovered nodes to the d.members slice and sending the node
// over the result chan. Then it creates a new goroutine for each node that checks
// if it can read on it's registerCh. If it can't within rn.SendInterval * 2 it derigesters
func (d *Disco) register(results chan *node.Node, rn *node.Node) {
	// If it's new to the members send it as a result
	rn.Action = node.RegisterAction

	d.mu.Lock()
	d.members = append(d.members, rn)
	d.mu.Unlock()

	results <- rn

	go func() {
		for {
			t := time.NewTimer(rn.SendInterval * 2)
			select {
			case <-rn.RegisterCh():
				t.Stop()
				continue
			case <-t.C:
				t.Stop()
				// Deregister if it times out
				rn.Action = node.DeregisterAction
				d.deregister(rn)
				results <- rn
				return
			}
		}
	}()
}

// Deregister takes a node and removes it from the d.members slice
func (d *Disco) deregister(n *node.Node) {
	d.mu.Lock()
	defer d.mu.Unlock()
	// Remove node from regsistered
	for i, m := range d.members {
		// make sure the node we sent matches
		if m == n {
			// remove it from the slice
			d.members = append(d.members[:i], d.members[i+1:]...)
		}
	}
}

// Check if the members slice already has the node if it doesn't add it
func (d *Disco) addToMembers(n *node.Node) bool {
	for _, m := range d.Members() {
		if m.Equal(n) {
			return false // node is already a member
		}
	}

	return true
}

// indexOfMember checks if a node is in the d.members slice
// and returns it's index, if it isn't there it returns -1
func (d *Disco) indexOfMember(n *node.Node) int {
	for i, a := range d.Members() {
		if a.Equal(n) {
			return i
		}
	}
	return -1
}
<commit_msg>Changing how deregister removes nodes<commit_after>package disco

import (
	"bytes"
	"context"
	"encoding/gob"
	"errors"
	"net"
	"sync"
	"time"

	"gitlab.fg/go/disco/multicast"
	"gitlab.fg/go/disco/node"
)

// Disco represents a list of discovered devices
type Disco struct {
	mu             sync.Mutex      // protects members
	members        []*node.Node    // stores all nodes registered
	discoveredChan chan *node.Node // node.Serve() sends nodes to this chan
}

// Members returns all node in members slice
func (d *Disco) Members() []*node.Node {
	d.mu.Lock()
	defer d.mu.Unlock()

	return d.members
}

// Discover listens for multicast sends and registers any nodes it finds
func (d *Disco) Discover(ctx context.Context, multicastAddress string) (<-chan *node.Node, error) {
	if multicastAddress == "" {
		return nil, errors.New("Address is blank")
	}

	ip, _, err := net.SplitHostPort(multicastAddress)
	if err != nil {
		return nil, err
	}

	if !net.ParseIP(ip).IsMulticast() {
		return nil, errors.New("multicastAddress is not valid")
	}

	if d.discoveredChan == nil {
		d.discoveredChan = make(chan *node.Node)
	}

	results := make(chan *node.Node)

	m := &multicast.Multicast{Address: multicastAddress}
	respChan, err := m.Listen(ctx)
	if err != nil {
		return nil, err
	}

	if d.members == nil {
		d.members = []*node.Node{}
	}

	go func() {
		for {
			select {
			case resp := <-respChan:
				buffer := bytes.NewBuffer(resp.Payload)
				rn := &node.Node{}
				dec := gob.NewDecoder(buffer)
				dec.Decode(rn)
				rn.SrcIP = resp.SrcIP // set the source address
				if d.addToMembers(rn) {
					d.register(results, rn)
				} else {
					if index := d.indexOfMember(rn); index != -1 {
						d.mu.Lock()
						d.members[index].KeepRegistered()
						d.mu.Unlock()
					}
				}
			case <-ctx.Done():
				return
			}

		}
	}()

	return results, nil
}

// register adds newly discovered nodes to the d.members slice and sending the node
// over the result chan. Then it creates a new goroutine for each node that checks
// if it can read on it's registerCh. If it can't within rn.SendInterval * 2 it derigesters
func (d *Disco) register(results chan *node.Node, rn *node.Node) {
	// If it's new to the members send it as a result
	rn.Action = node.RegisterAction

	d.mu.Lock()
	d.members = append(d.members, rn)
	d.mu.Unlock()

	results <- rn

	go func() {
		for {
			t := time.NewTimer(rn.SendInterval * 2)
			select {
			case <-rn.RegisterCh():
				t.Stop()
				continue
			case <-t.C:
				t.Stop()
				// Deregister if it times out
				rn.Action = node.DeregisterAction
				d.deregister(rn)
				results <- rn
				return
			}
		}
	}()
}

// Deregister takes a node and removes it from the d.members slice
func (d *Disco) deregister(n *node.Node) {
	d.mu.Lock()
	defer d.mu.Unlock()
	// Remove node from regsistered
	for i := len(d.members) - 1; i >= 0; i-- {
		m := d.members[i]
		// make sure the node we sent matches
		if m == n {
			// remove it from the slice
			d.members = append(d.members[:i], d.members[i+1:]...)
		}
	}
}

// Check if the members slice already has the node if it doesn't add it
func (d *Disco) addToMembers(n *node.Node) bool {
	for _, m := range d.Members() {
		if m.Equal(n) {
			return false // node is already a member
		}
	}

	return true
}

// indexOfMember checks if a node is in the d.members slice
// and returns it's index, if it isn't there it returns -1
func (d *Disco) indexOfMember(n *node.Node) int {
	for i, a := range d.Members() {
		if a.Equal(n) {
			return i
		}
	}
	return -1
}
<|endoftext|>


<commit_before>package lxc0

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
)

const (
	containersPath = "/sys/fs/cgroup/cpu/lxc"
)

type LXC struct {
}

type Usage struct {
	System int
	User   int
}

func (l LXC) GetContainers() []string {
	var containers = []string{}
	files, _ := ioutil.ReadDir(containersPath)
	for _, f := range files {
		if f.IsDir() {
			containers = append(containers, f.Name())
		}
	}

	return containers
}

func (l LXC) GetCPUUsage(container string) (Usage, error) {
	content, err := ioutil.ReadFile(fmt.Sprint(containersPath, "/", container, "/cpuacct.stat"))
	if err != nil {
		return Usage{}, err
	}

	reg := regexp.MustCompile("\\s\\s+")
	content = reg.ReplaceAll(content, []byte(" "))
	lines := strings.Split(string(content), "\n")
	user := strings.Split(lines[0], " ")
	system := strings.Split(lines[1], " ")

	return Usage{User: forceToInt(user[1]), System: forceToInt(system[1])}, nil
}

func forceToInt(variable string) int {
	value, _ := strconv.Atoi(variable)
	return value
}
<commit_msg>Remove part of proof of concept<commit_after><|endoftext|>


<commit_before>package magicnum

import (
	"bytes"
	"compress/lzw"
	"io"
	"testing"

	"github.com/pierrec/lz4"
)

var testVal = []byte(`
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
 incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
 nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
 Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore
 eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt
 in culpa qui officia deserunt mollit anim id est laborum.
`)

// The tests to check format involve creating a compressed version using the
// desired algorithm and then checking its header.
//
// All algorithm specific tests will also call GetFormat() to validate its
// behavior for that algorithm.
func TestIsLZ4(t *testing.T) {
	b := make([]byte, 0, 512)
	r := bytes.NewReader(testVal)
	w := bytes.NewBuffer(b)
	lw := lz4.NewWriter(w)
	n, err := io.Copy(lw, r)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if n != 452 {
		t.Errorf("Expected 452 bytes to be copied; %d were", n)
	}
	lw.Close()
	rr := bytes.NewReader(w.Bytes())
	ok, err := IsLZ4(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if !ok {
		t.Error("Expected ok to be true, got false")
	}
	format, err := GetFormat(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if format != LZ4 {
		t.Errorf("Expected format to be LZ4 got %s", format)
	}
}

// TODO: commented out because LZW output doesn't have the magic number.
// Figure out why and resolve.
// Another example http://play.golang.org/p/zGLAj1ruoh
/*
func TestIsLZW(t *testing.T) {
	//r := bytes.NewReader(testVal)
	var buf bytes.Buffer
	lw := lzw.NewWriter(&buf, lzw.LSB, 8)
	n, err := lw.Write(testVal)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if n != 452 {
		t.Errorf("Expected 452 bytes to be copied; %d were", n)
	}
	lw.Close()
	t.Errorf("%x", buf.Bytes())
	rr := bytes.NewReader(buf.Bytes())
	ok, err := IsLZW(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if !ok {
		t.Error("Expected ok to be true, got false")
	}
	format, err := GetFormat(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if format != LZW {
		t.Errorf("Expected format to be LZW got %s", format)
	}
}
*/
<commit_msg>LZW w/o test as Go's LZW package doesn't appear to emit the magic numbers; this is probably ignorance on my part, but I'm moving on to detecting other things<commit_after>package magicnum

import (
	"bytes"
	//"compress/lzw"
	//"io"
	"testing"

	"github.com/pierrec/lz4"
)

var testVal = []byte(`
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
 incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis
 nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
 Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore
 eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt
 in culpa qui officia deserunt mollit anim id est laborum.
`)

// The tests to check format involve creating a compressed version using the
// desired algorithm and then checking its header.
//
// All algorithm specific tests will also call GetFormat() to validate its
// behavior for that algorithm.
//func TestIsBzip2(t *testing.T) {

//}

func TestIsLZ4(t *testing.T) {
	var buf bytes.Buffer
	lw := lz4.NewWriter(&buf)
	n, err := lw.Write(testVal)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if n != 452 {
		t.Errorf("Expected 452 bytes to be copied; %d were", n)
	}
	lw.Close()
	r := bytes.NewReader(buf.Bytes())
	ok, err := IsLZ4(r)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if !ok {
		t.Error("Expected ok to be true, got false")
	}
	format, err := GetFormat(r)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if format != LZ4 {
		t.Errorf("Expected format to be LZ4 got %s", format)
	}
}

// TODO: commented out because LZW output doesn't have the magic number.
// Figure out why and resolve.
// Another example http://play.golang.org/p/zGLAj1ruoh
/*
func TestIsLZW(t *testing.T) {
	//r := bytes.NewReader(testVal)
	var buf bytes.Buffer
	lw := lzw.NewWriter(&buf, lzw.LSB, 8)
	n, err := lw.Write(testVal)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if n != 452 {
		t.Errorf("Expected 452 bytes to be copied; %d were", n)
	}
	lw.Close()
	t.Errorf("%x", buf.Bytes())
	rr := bytes.NewReader(buf.Bytes())
	ok, err := IsLZW(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if !ok {
		t.Error("Expected ok to be true, got false")
	}
	format, err := GetFormat(rr)
	if err != nil {
		t.Errorf("Unexpected error: %s", err)
	}
	if format != LZW {
		t.Errorf("Expected format to be LZW got %s", format)
	}
}
*/
<|endoftext|>


<commit_before>package main

import (
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pborman/uuid"

	"flag"

	couchbasearray "github.com/andrewwebber/couchbase-array"
)

var servicePathFlag = flag.String("s", "/services/couchbase-array", "etcd directory")
var heartBeatFlag = flag.Int("h", 3, "heart beat loop in seconds")
var ttlFlag = flag.Int("ttl", 30, "time to live in seconds")
var debugFlag = flag.Bool("v", false, "verbose")
var rebalanceOnExitFlag = flag.Bool("r", false, "rebalance on exit")
var machineIdentiferFlag = flag.String("ip", "", "machine ip address")
var whatIfFlag = flag.Bool("t", false, "what if")
var cliBase = flag.String("cli", "/opt/couchbase/bin/couchbase-cli", "path to couchbase cli")
var statefulSet = flag.Bool("statefulset", false, "use hostnames instead of ip addresses")
var masterNodeAnnouncePathFlag = flag.String("m", "/services/couchbase", "announce etcd path for the master IP")

func main() {
	log.SetFlags(log.Llongfile)
	flag.Parse()
	log.Println("Couchbase Cluster Node")
	couchbasearray.TTL = uint64(*ttlFlag)
	log.Printf("TTL %v\n", couchbasearray.TTL)

	machineIdentifier := *machineIdentiferFlag
	if machineIdentifier == "" {
		var err error
		machineIdentifier, err = getMachineIdentifier()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("Machine ID: %s\n", machineIdentifier)

	sessionID := uuid.New()
	var isClusterMember bool

	go func() {
		for {
			announcments, err := couchbasearray.GetClusterAnnouncements(*servicePathFlag)
			if err != nil {
				log.Println(err)
				continue
			}

			machineState, ok := announcments[sessionID]
			if !ok {
				machineState = couchbasearray.NodeState{
					IPAddress:    machineIdentifier,
					SessionID:    sessionID,
					Master:       false,
					State:        "",
					DesiredState: ""}
			}

			currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)

			master, err := couchbasearray.GetMasterNode(currentStates)
			if err != nil {
				err := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
				if err == nil {
					stopScheduler := make(chan bool)
					go couchbasearray.StartScheduler(*servicePathFlag, *heartBeatFlag, stopScheduler, *masterNodeAnnouncePathFlag)
					go func() {
						failoverSet := false
						for {
							lockErr := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
							if lockErr != nil {
								log.Println(lockErr)
								stopScheduler <- true
								return
							}

							if !failoverSet {
								if failOverErr := setAutoFailover(machineIdentifier, 31); err != nil {
									log.Println(failOverErr)
								} else {
									failoverSet = true
								}
							}

							time.Sleep(4 * time.Second)
						}
					}()
				}

				if err != nil && err != couchbasearray.ErrLockInUse {
					log.Println(err)
					continue
				}
			} else {
				if state, ok := currentStates[sessionID]; ok {
					if state.DesiredState != machineState.State {
						log.Printf("DesiredState: %s - Current State: %s", state.DesiredState, machineState.State)

						switch state.DesiredState {
						case couchbasearray.SchedulerStateClustered:
							log.Println("rebalancing")

							if master.IPAddress == machineIdentifier {
								log.Println("Already master no action required")
							} else {
								log.Printf("rebalancing with master node %s\n", master.IPAddress)
								if !*whatIfFlag {
									if isClusterMember {
										err = recoverNode(master.IPAddress, machineIdentifier)
									}

									err = rebalanceNode(master.IPAddress, machineIdentifier)
								}
							}

							if err == nil {
								machineState.State = state.DesiredState
							} else {
								log.Println(err)
							}

						case couchbasearray.SchedulerStateNew:
							log.Println("adding server to cluster")
							master, err := couchbasearray.GetMasterNode(currentStates)
							if err != nil {
								log.Println(err)
							} else {
								if master.IPAddress == machineIdentifier {
									log.Println("Already master no action required")
								} else {
									log.Printf("Adding to master node %s\n", master.IPAddress)
									if !*whatIfFlag {
										isClusterMember, err = addNodeToCluster(master.IPAddress, machineIdentifier)
									}
								}

								if err == nil {
									machineState.State = state.DesiredState
								} else {
									log.Println(err)
								}
							}
						default:
							log.Println(state.DesiredState)
							log.Fatal("unknown state")
						}
					}
				} else {
					log.Println("Running")
				}
			}

			err = couchbasearray.SetClusterAnnouncement(*servicePathFlag, machineState)
			if err != nil {
				log.Println(err)
			}

			time.Sleep(time.Duration(*heartBeatFlag) * time.Second)
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
	log.Println(<-ch)
	log.Println("Failing over")

	currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)
	if err != nil {
		log.Fatal(err)
		return
	}

	master, err := couchbasearray.GetMasterNode(currentStates)
	if err != nil {
		log.Fatal(err)
		return
	}

	err = failoverClusterNode(master.IPAddress, machineIdentifier)
	if err != nil {
		log.Fatal(err)
	}

	if *rebalanceOnExitFlag {
		err = rebalanceNode(master.IPAddress, machineIdentifier)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMachineIdentifier() (string, error) {
	if *statefulSet {
		return os.Hostname()
	}

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatalln(err)
	}

	var result string
	for _, a := range addrs {
		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				result = ipnet.IP.String()
				log.Println(ipnet.Network())
				log.Printf("Found IP %s\n", result)
				break
			}
		}
	}

	if result != "" {
		return result, nil
	}

	return os.Hostname()
}

type function func() error

func exponential(operation function, maxRetries int) error {
	var err error
	var sleepTime int
	for i := 0; i < maxRetries; i++ {
		err = operation()
		if err == nil {
			return nil
		}
		if i == 0 {
			sleepTime = 1
		} else {
			sleepTime = int(math.Exp2(float64(i)) * 100)
		}
		time.Sleep(time.Duration(sleepTime) * time.Millisecond)
		log.Printf("Retry exponential: Attempt %d, sleep %d", i, sleepTime)
	}

	return err
}
<commit_msg>stateful sets<commit_after>package main

import (
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/pborman/uuid"

	"flag"

	couchbasearray "github.com/andrewwebber/couchbase-array"
)

var servicePathFlag = flag.String("s", "/services/couchbase-array", "etcd directory")
var heartBeatFlag = flag.Int("h", 3, "heart beat loop in seconds")
var ttlFlag = flag.Int("ttl", 30, "time to live in seconds")
var debugFlag = flag.Bool("v", false, "verbose")
var rebalanceOnExitFlag = flag.Bool("r", false, "rebalance on exit")
var machineIdentiferFlag = flag.String("ip", "", "machine ip address")
var whatIfFlag = flag.Bool("t", false, "what if")
var cliBase = flag.String("cli", "/opt/couchbase/bin/couchbase-cli", "path to couchbase cli")
var statefulSet = flag.String("statefulset", "", "use stateful")
var masterNodeAnnouncePathFlag = flag.String("m", "/services/couchbase", "announce etcd path for the master IP")

func main() {
	log.SetFlags(log.Llongfile)
	flag.Parse()
	log.Println("Couchbase Cluster Node")
	couchbasearray.TTL = uint64(*ttlFlag)
	log.Printf("TTL %v\n", couchbasearray.TTL)

	machineIdentifier := *machineIdentiferFlag
	if machineIdentifier == "" {
		var err error
		machineIdentifier, err = getMachineIdentifier()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Printf("Machine ID: %s\n", machineIdentifier)

	sessionID := uuid.New()
	var isClusterMember bool

	go func() {
		for {
			announcments, err := couchbasearray.GetClusterAnnouncements(*servicePathFlag)
			if err != nil {
				log.Println(err)
				continue
			}

			machineState, ok := announcments[sessionID]
			if !ok {
				machineState = couchbasearray.NodeState{
					IPAddress:    machineIdentifier,
					SessionID:    sessionID,
					Master:       false,
					State:        "",
					DesiredState: ""}
			}

			currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)

			master, err := couchbasearray.GetMasterNode(currentStates)
			if err != nil {
				err := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
				if err == nil {
					stopScheduler := make(chan bool)
					go couchbasearray.StartScheduler(*servicePathFlag, *heartBeatFlag, stopScheduler, *masterNodeAnnouncePathFlag)
					go func() {
						failoverSet := false
						for {
							lockErr := couchbasearray.AcquireLock(sessionID, *servicePathFlag+"/master", 5)
							if lockErr != nil {
								log.Println(lockErr)
								stopScheduler <- true
								return
							}

							if !failoverSet {
								if failOverErr := setAutoFailover(machineIdentifier, 31); err != nil {
									log.Println(failOverErr)
								} else {
									failoverSet = true
								}
							}

							time.Sleep(4 * time.Second)
						}
					}()
				}

				if err != nil && err != couchbasearray.ErrLockInUse {
					log.Println(err)
					continue
				}
			} else {
				if state, ok := currentStates[sessionID]; ok {
					if state.DesiredState != machineState.State {
						log.Printf("DesiredState: %s - Current State: %s", state.DesiredState, machineState.State)

						switch state.DesiredState {
						case couchbasearray.SchedulerStateClustered:
							log.Println("rebalancing")

							if master.IPAddress == machineIdentifier {
								log.Println("Already master no action required")
							} else {
								log.Printf("rebalancing with master node %s\n", master.IPAddress)
								if !*whatIfFlag {
									if isClusterMember {
										err = recoverNode(master.IPAddress, machineIdentifier)
									}

									err = rebalanceNode(master.IPAddress, machineIdentifier)
								}
							}

							if err == nil {
								machineState.State = state.DesiredState
							} else {
								log.Println(err)
							}

						case couchbasearray.SchedulerStateNew:
							log.Println("adding server to cluster")
							master, err := couchbasearray.GetMasterNode(currentStates)
							if err != nil {
								log.Println(err)
							} else {
								if master.IPAddress == machineIdentifier {
									log.Println("Already master no action required")
								} else {
									log.Printf("Adding to master node %s\n", master.IPAddress)
									if !*whatIfFlag {
										isClusterMember, err = addNodeToCluster(master.IPAddress, machineIdentifier)
									}
								}

								if err == nil {
									machineState.State = state.DesiredState
								} else {
									log.Println(err)
								}
							}
						default:
							log.Println(state.DesiredState)
							log.Fatal("unknown state")
						}
					}
				} else {
					log.Println("Running")
				}
			}

			err = couchbasearray.SetClusterAnnouncement(*servicePathFlag, machineState)
			if err != nil {
				log.Println(err)
			}

			time.Sleep(time.Duration(*heartBeatFlag) * time.Second)
		}
	}()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
	log.Println(<-ch)
	log.Println("Failing over")

	currentStates, err := couchbasearray.GetClusterStates(*servicePathFlag)
	if err != nil {
		log.Fatal(err)
		return
	}

	master, err := couchbasearray.GetMasterNode(currentStates)
	if err != nil {
		log.Fatal(err)
		return
	}

	err = failoverClusterNode(master.IPAddress, machineIdentifier)
	if err != nil {
		log.Fatal(err)
	}

	if *rebalanceOnExitFlag {
		err = rebalanceNode(master.IPAddress, machineIdentifier)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getMachineIdentifier() (string, error) {
	if *statefulSet != "" {
		return fmt.Sprintf("%s%s", os.Hostname(), *statefulSet)
	}

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		log.Fatalln(err)
	}

	var result string
	for _, a := range addrs {
		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
			if ipnet.IP.To4() != nil {
				result = ipnet.IP.String()
				log.Println(ipnet.Network())
				log.Printf("Found IP %s\n", result)
				break
			}
		}
	}

	if result != "" {
		return result, nil
	}

	return os.Hostname()
}

type function func() error

func exponential(operation function, maxRetries int) error {
	var err error
	var sleepTime int
	for i := 0; i < maxRetries; i++ {
		err = operation()
		if err == nil {
			return nil
		}
		if i == 0 {
			sleepTime = 1
		} else {
			sleepTime = int(math.Exp2(float64(i)) * 100)
		}
		time.Sleep(time.Duration(sleepTime) * time.Millisecond)
		log.Printf("Retry exponential: Attempt %d, sleep %d", i, sleepTime)
	}

	return err
}
<|endoftext|>


<commit_before>// Package common provides common operation helpers to the generators
package common

import (
	"strings"
	"text/template"

	"github.com/cihangir/schema"
	"github.com/cihangir/stringext"
)

// TemplateFuncs provides utility functions for template operations
var TemplateFuncs = template.FuncMap{
	"Pointerize":              stringext.Pointerize,
	"ToLower":                 strings.ToLower,
	"ToLowerFirst":            stringext.ToLowerFirst,
	"ToUpperFirst":            stringext.ToUpperFirst,
	"AsComment":               stringext.AsComment,
	"DepunctWithInitialUpper": stringext.DepunctWithInitialUpper,
	"DepunctWithInitialLower": stringext.DepunctWithInitialLower,
	"Equal":                   stringext.Equal,
	"ToFieldName":             stringext.ToFieldName,
	"Argumentize":             schema.Argumentize,
}
<commit_msg>Generator: added new helper function<commit_after>// Package common provides common operation helpers to the generators
package common

import (
	"strings"
	"text/template"

	"github.com/cihangir/schema"
	"github.com/cihangir/stringext"
)

// TemplateFuncs provides utility functions for template operations
var TemplateFuncs = template.FuncMap{
	"Pointerize":              stringext.Pointerize,
	"ToLower":                 strings.ToLower,
	"ToUpper":                 strings.ToUpper,
	"ToLowerFirst":            stringext.ToLowerFirst,
	"ToUpperFirst":            stringext.ToUpperFirst,
	"AsComment":               stringext.AsComment,
	"DepunctWithInitialUpper": stringext.DepunctWithInitialUpper,
	"DepunctWithInitialLower": stringext.DepunctWithInitialLower,
	"Equal":                   stringext.Equal,
	"ToFieldName":             stringext.ToFieldName,
	"Argumentize":             schema.Argumentize,
}
<|endoftext|>


<commit_before>// qrutil provides wrapper functions for https://github.com/skip2/go-qrcode s.
package qrutil

import (
	"image"

	"github.com/skip2/go-qrcode"
)

type Opts struct {
	QRCode    qrcode.QRCode
	Filename  string
	Filesize  int
	FileWrite bool
}

// New creates a new `*qrcode.QRCode` using `qrcode.QRCode`
// as the request parameter.
func New(opts qrcode.QRCode) (*qrcode.QRCode, error) {
	q, err := qrcode.New(opts.Content, opts.Level)
	if err != nil {
		return nil, err
	}

	if opts.BackgroundColor != opts.ForegroundColor {
		q.BackgroundColor = opts.BackgroundColor
		q.ForegroundColor = opts.ForegroundColor
	}

	if opts.DisableBorder {
		q.DisableBorder = true
	}
	return q, nil
}

func NewImage(opts qrcode.QRCode, size int) (image.Image, error) {
	qr, err := New(opts)
	if err != nil {
		return nil, err
	}
	return qr.Image(size), nil
}

func Create(opts Opts) (*qrcode.QRCode, error) {
	q, err := New(opts.QRCode)
	if err != nil {
		return nil, err
	}

	if opts.FileWrite {
		err = q.WriteFile(opts.Filesize, opts.Filename)
	}

	return q, err
}
<commit_msg>fix: qrutil: typo<commit_after>// qrutil provides wrapper functions for https://github.com/skip2/go-qrcode
package qrutil

import (
	"image"

	"github.com/skip2/go-qrcode"
)

type Opts struct {
	QRCode    qrcode.QRCode
	Filename  string
	Filesize  int
	FileWrite bool
}

// New creates a new `*qrcode.QRCode` using `qrcode.QRCode`
// as the request parameter.
func New(opts qrcode.QRCode) (*qrcode.QRCode, error) {
	q, err := qrcode.New(opts.Content, opts.Level)
	if err != nil {
		return nil, err
	}

	if opts.BackgroundColor != opts.ForegroundColor {
		q.BackgroundColor = opts.BackgroundColor
		q.ForegroundColor = opts.ForegroundColor
	}

	if opts.DisableBorder {
		q.DisableBorder = true
	}
	return q, nil
}

func NewImage(opts qrcode.QRCode, size int) (image.Image, error) {
	qr, err := New(opts)
	if err != nil {
		return nil, err
	}
	return qr.Image(size), nil
}

func Create(opts Opts) (*qrcode.QRCode, error) {
	q, err := New(opts.QRCode)
	if err != nil {
		return nil, err
	}

	if opts.FileWrite {
		err = q.WriteFile(opts.Filesize, opts.Filename)
	}

	return q, err
}
<|endoftext|>