<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package http

import (
	"net/http"
	"testing"
	"time"

	//	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/mocks"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	errutil "github.com/TheThingsNetwork/ttn/utils/errors/checks"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
)

func TestHTTPAckNacker(t *testing.T) {
	{
		Desc(t, "Ack a nil packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		err := an.Ack(nil)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusOK,
			Content:    nil,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Ack on a nil chresp")

		// Build
		an := httpAckNacker{Chresp: nil}

		// Operate
		err := an.Ack(nil)

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}

	// --------------------

	{
		Desc(t, "Ack a valid packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}
		p := mocks.NewMockPacket()
		p.OutMarshalBinary = []byte{14, 14, 14}

		// Operate
		err := an.Ack(p)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusOK,
			Content:    p.OutMarshalBinary,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Ack an invalid packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}
		p := mocks.NewMockPacket()
		p.Failures["MarshalBinary"] = errors.New(errors.Structural, "Mock Error")

		// Operate
		err := an.Ack(p)

		// Check
		errutil.CheckErrors(t, pointer.String(string(errors.Structural)), err)
		CheckResps(t, nil, chresp)
	}

	// --------------------

	{
		Desc(t, "Don't consume chresp")

		// Build
		chresp := make(chan MsgRes)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		cherr := make(chan error)
		go func() {
			cherr <- an.Ack(nil)
		}()

		// Check
		var err error
		select {
		case err = <-cherr:
		case <-time.After(time.Millisecond * 100):
		}
		errutil.CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}

	// --------------------

	{
		Desc(t, "Nack")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		err := an.Nack()

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusNotFound,
			Content:    []byte(errors.Structural),
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Nack on a nil chresp")

		// Build
		an := httpAckNacker{Chresp: nil}

		// Operate
		err := an.Nack()

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}
}
<commit_msg>[test/http-adapter] Add tests for reg Acknacker<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package http

import (
	"net/http"
	"testing"
	"time"

	//	"github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/mocks"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	errutil "github.com/TheThingsNetwork/ttn/utils/errors/checks"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
)

func TestHTTPAckNacker(t *testing.T) {
	{
		Desc(t, "Ack a nil packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		err := an.Ack(nil)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusOK,
			Content:    nil,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Ack on a nil chresp")

		// Build
		an := httpAckNacker{Chresp: nil}

		// Operate
		err := an.Ack(nil)

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}

	// --------------------

	{
		Desc(t, "Ack a valid packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}
		p := mocks.NewMockPacket()
		p.OutMarshalBinary = []byte{14, 14, 14}

		// Operate
		err := an.Ack(p)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusOK,
			Content:    p.OutMarshalBinary,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Ack an invalid packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}
		p := mocks.NewMockPacket()
		p.Failures["MarshalBinary"] = errors.New(errors.Structural, "Mock Error")

		// Operate
		err := an.Ack(p)

		// Check
		errutil.CheckErrors(t, pointer.String(string(errors.Structural)), err)
		CheckResps(t, nil, chresp)
	}

	// --------------------

	{
		Desc(t, "Don't consume chresp on Ack")

		// Build
		chresp := make(chan MsgRes)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		cherr := make(chan error)
		go func() {
			cherr <- an.Ack(nil)
		}()

		// Check
		var err error
		select {
		case err = <-cherr:
		case <-time.After(time.Millisecond * 100):
		}
		errutil.CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}

	// --------------------

	{
		Desc(t, "Nack")

		// Build
		chresp := make(chan MsgRes, 1)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		err := an.Nack()

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusNotFound,
			Content:    []byte(errors.Structural),
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Nack on a nil chresp")

		// Build
		an := httpAckNacker{Chresp: nil}

		// Operate
		err := an.Nack()

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}

	// --------------------

	{
		Desc(t, "Don't consume chresp on Nack")

		// Build
		chresp := make(chan MsgRes)
		an := httpAckNacker{Chresp: chresp}

		// Operate
		cherr := make(chan error)
		go func() {
			cherr <- an.Nack()
		}()

		// Check
		var err error
		select {
		case err = <-cherr:
		case <-time.After(time.Millisecond * 100):
		}
		errutil.CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}
}

func TestRegAckNacker(t *testing.T) {
	{
		Desc(t, "Ack a nil packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := regAckNacker{Chresp: chresp}

		// Operate
		err := an.Ack(nil)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusAccepted,
			Content:    nil,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Ack on a nil chresp")

		// Build
		an := regAckNacker{Chresp: nil}

		// Operate
		err := an.Ack(nil)

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}

	// --------------------

	{
		Desc(t, "Ack a valid packet")

		// Build
		chresp := make(chan MsgRes, 1)
		an := regAckNacker{Chresp: chresp}
		p := mocks.NewMockPacket()
		p.OutMarshalBinary = []byte{14, 14, 14}

		// Operate
		err := an.Ack(p)

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusAccepted,
			Content:    nil,
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Don't consume chresp on Ack")

		// Build
		chresp := make(chan MsgRes)
		an := regAckNacker{Chresp: chresp}

		// Operate
		cherr := make(chan error)
		go func() {
			cherr <- an.Ack(nil)
		}()

		// Check
		var err error
		select {
		case err = <-cherr:
		case <-time.After(time.Millisecond * 100):
		}
		errutil.CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}

	// --------------------

	{
		Desc(t, "Nack")

		// Build
		chresp := make(chan MsgRes, 1)
		an := regAckNacker{Chresp: chresp}

		// Operate
		err := an.Nack()

		// Expectation
		want := &MsgRes{
			StatusCode: http.StatusConflict,
			Content:    []byte(errors.Structural),
		}

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, want, chresp)
	}

	// --------------------

	{
		Desc(t, "Nack on a nil chresp")

		// Build
		an := regAckNacker{Chresp: nil}

		// Operate
		err := an.Nack()

		// Check
		errutil.CheckErrors(t, nil, err)
		CheckResps(t, nil, nil)
	}

	// --------------------

	{
		Desc(t, "Don't consume chresp on Nack")

		// Build
		chresp := make(chan MsgRes)
		an := regAckNacker{Chresp: chresp}

		// Operate
		cherr := make(chan error)
		go func() {
			cherr <- an.Nack()
		}()

		// Check
		var err error
		select {
		case err = <-cherr:
		case <-time.After(time.Millisecond * 100):
		}
		errutil.CheckErrors(t, pointer.String(string(errors.Operational)), err)
	}
}
<|endoftext|>


<commit_before>package repository

import (
	"bufio"
	"database/sql"
	"fmt"
	"io"
	"time"

	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util/file"
)

var exampleNotFoundError = model.NotFoundError("example")

// データが存在しなければ追加
// データが存在する場合は、以下の場合にのみ更新する
// - ラベルが正例か負例に変更された
// - クロール対象のサイトが一時的に200以外のステータスで前回データが取得できなかった
func (r *repository) UpdateOrCreateExample(e *model.Example) error {
	now := time.Now()
	e.UpdatedAt = now
	_, err := r.db.NamedExec(`
INSERT INTO example
( url,  final_url,  title,  description,  og_description,  og_type,  og_image,  body,  score,  is_new,  status_code,  favicon,  label,  created_at,  updated_at)
VALUES
(:url, :final_url, :title, :description, :og_description, :og_type, :og_image, :body, :score, :is_new, :status_code, :favicon, :label, :created_at, :updated_at)
ON CONFLICT (url)
DO UPDATE SET
url = :url, final_url = :final_url, title = :title,
description = :description, og_description = :og_description, og_type = :og_type, og_image = :og_image,
body = :body, score = :score, is_new = :is_new, status_code = :status_code, favicon = :favicon,
label = :label, created_at = :created_at, updated_at = :updated_at
WHERE
((EXCLUDED.label != 0) AND (example.label != EXCLUDED.label)) OR
((example.status_code != 200) AND (EXCLUDED.status_code = 200))
;`, e)
	if err != nil {
		return err
	}
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	e.Id = tmp.Id
	return nil
}

func (r *repository) UpdateScore(e *model.Example) error {
	if _, err := r.FindExampleByUlr(e.Url); err != nil {
		return err
	}
	if _, err := r.db.Exec(`UPDATE example SET score = $1, updated_at = $2 WHERE url = $3;`, e.Score, time.Now(), e.Url); err != nil {
		return err
	}
	return nil
}

func (r *repository) IncErrorCount(e *model.Example) error {
	errorCount, err := r.GetErrorCount(e)
	if err != nil {
		return err
	}
	if _, err := r.db.Exec(`UPDATE example SET error_count = $1, updated_at = $2 WHERE url = $3;`, errorCount+1, time.Now(), e.Url); err != nil {
		return err
	}
	return nil
}

func (r *repository) GetErrorCount(e *model.Example) (int, error) {
	example, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		if err == exampleNotFoundError {
			return 0, nil
		}
		return 0, err
	}
	return example.ErrorCount, nil
}

func (r *repository) UpdateFeatureVector(e *model.Example) error {
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id
	if _, err = r.db.Exec(`DELETE FROM feature WHERE example_id = $1;`, id); err != nil {
		return err
	}
	_, err = r.db.Exec(`INSERT INTO feature (example_id, feature) VALUES ($1, unnest(cast($2 AS TEXT[])));`, id, pq.Array(e.Fv))
	return err
}

func (r *repository) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	line := scanner.Text()
	e, err := file.ParseLine(line)
	if err != nil {
		return nil, err
	}
	err = r.UpdateOrCreateExample(e)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (r *repository) InsertExamplesFromReader(reader io.Reader) error {
	scanner := bufio.NewScanner(reader)

	for scanner.Scan() {
		_, err := r.InsertExampleFromScanner(scanner)
		if err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func (r *repository) searchExamples(query string, args ...interface{}) (model.Examples, error) {
	examples := model.Examples{}
	err := r.db.Select(&examples, query, args...)
	if err != nil {
		return nil, err
	}
	return examples, nil
}

func (r *repository) findExample(query string, args ...interface{}) (*model.Example, error) {
	e := model.Example{}

	err := r.db.Get(&e, query, args...)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, exampleNotFoundError
		}
		return nil, err
	}
	return &e, nil
}

func (r *repository) SearchExamples() (model.Examples, error) {
	query := `SELECT * FROM example;`
	return r.searchExamples(query)
}

func (r *repository) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE created_at > $1 ORDER BY updated_at DESC LIMIT $2;`
	return r.searchExamples(query, from, limit)
}

func (r *repository) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE final_url like $1 || '%' AND created_at > $2 ORDER BY updated_at DESC LIMIT $3;`
	return r.searchExamples(query, host, from, limit)
}

func (r *repository) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE label = $1 ORDER BY updated_at DESC LIMIT $2;`
	return r.searchExamples(query, label, limit)
}

func (r *repository) SearchLabeledExamples(limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE label != 0 ORDER BY updated_at DESC LIMIT $1;`
	return r.searchExamples(query, limit)
}

func (r *repository) SearchPositiveExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.POSITIVE, limit)
}

func (r *repository) SearchNegativeExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.NEGATIVE, limit)
}

func (r *repository) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.UNLABELED, limit)
}

func (r *repository) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE score > 0 ORDER BY updated_at DESC LIMIT $1;`
	return r.searchExamples(query, limit)
}

func (r *repository) FindExampleByUlr(url string) (*model.Example, error) {
	query := `SELECT * FROM example WHERE url = $1;`
	return r.findExample(query, url)
}

// bodyなどは極めて長くなりえるので、DB側で絞って返すことができるようにする
func buildSelectQuery(useTruncatedField bool) string {
	title := "title"
	description := "description"
	ogDescription := "og_description"
	body := "body"

	if useTruncatedField {
		title = "LEFT(title, 200) AS title"
		description = "LEFT(description, 1000) AS description"
		ogDescription = "LEFT(og_description, 1000) AS og_description"
		body = "LEFT(body, 1000) AS body"
	}
	return fmt.Sprintf("SELECT id, label, url, final_url, %s, %s, %s, og_type, og_image, %s, score, is_new, status_code, favicon, error_count, created_at, updated_at", title, description, ogDescription, body)
}

func (r *repository) FindExampleById(id int) (*model.Example, error) {
	query := fmt.Sprintf(`%s FROM example WHERE id = $1;`, buildSelectQuery(true))
	return r.findExample(query, id)
}

func (r *repository) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	// ref: https://godoc.org/github.com/lib/pq#Array
	query := `SELECT * FROM example WHERE url = ANY($1);`
	return r.searchExamples(query, pq.Array(urls))
}

func (r *repository) SearchExamplesByIds(ids []int) (model.Examples, error) {
	query := fmt.Sprintf(`%s FROM example WHERE id = ANY($1);`, buildSelectQuery(true))
	return r.searchExamples(query, pq.Array(ids))
}

func (r *repository) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	if len(keywords) == 0 {
		return model.Examples{}, nil
	}
	regexList := make([]string, 0)
	for _, w := range keywords {
		regexList = append(regexList, fmt.Sprintf(`.*%s.*`, w))
	}
	query := fmt.Sprintf(`%s FROM example WHERE title ~* %s($1) AND label != -1 ORDER BY (label, score) DESC LIMIT $2;`, buildSelectQuery(true), aggregator)
	return r.searchExamples(query, pq.Array(regexList), limit)
}

func (r *repository) countExamplesByLabel(label model.LabelType) (int, error) {
	cnt := 0
	err := r.db.Get(&cnt, `SELECT COUNT(*) FROM example WHERE label = $1`, label)
	if err != nil {
		return 0, err
	}
	return cnt, nil
}

func (r *repository) CountPositiveExamples() (int, error) {
	return r.countExamplesByLabel(model.POSITIVE)
}

func (r *repository) CountNegativeExamples() (int, error) {
	return r.countExamplesByLabel(model.NEGATIVE)
}

func (r *repository) CountUnlabeledExamples() (int, error) {
	return r.countExamplesByLabel(model.UNLABELED)
}

func (r *repository) FindFeatureVector(e *model.Example) (feature.FeatureVector, error) {
	fv := feature.FeatureVector{}
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return fv, err
	}
	id := tmp.Id
	query := `SELECT feature FROM feature WHERE example_id = $1;`
	err = r.db.Select(&fv, query, id)
	if err != nil {
		return fv, err
	}
	return fv, nil
}

func (r *repository) SearchFeatureVector(examples model.Examples) (map[int]feature.FeatureVector, error) {
	type Pair struct {
		ExampleId int    `db:"example_id"`
		Feature   string `db:"feature"`
	}

	fvById := make(map[int]feature.FeatureVector)
	urls := make([]string, 0)
	for _, e := range examples {
		urls = append(urls, e.Url)
	}

	tmp, err := r.SearchExamplesByUlrs(urls)
	if err != nil {
		return fvById, err
	}
	ids := make([]int, 0)
	for _, e := range tmp {
		ids = append(ids, e.Id)
	}

	query := `SELECT example_id, feature FROM feature WHERE example_id = ANY($1);`
	pairs := make([]Pair, 0)
	err = r.db.Select(&pairs, query, pq.Array(ids))
	if err != nil {
		return fvById, err
	}

	for _, pair := range pairs {
		fvById[pair.ExampleId] = append(fvById[pair.ExampleId], pair.Feature)
	}
	return fvById, nil
}

func (r *repository) DeleteAllExamples() error {
	_, err := r.db.Exec(`DELETE FROM example;`)
	return err
}
<commit_msg>idsが空の場合はdbにクエリを飛ばさない<commit_after>package repository

import (
	"bufio"
	"database/sql"
	"fmt"
	"io"
	"time"

	"github.com/lib/pq"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/model"
	"github.com/syou6162/go-active-learning/lib/util/file"
)

var exampleNotFoundError = model.NotFoundError("example")

// データが存在しなければ追加
// データが存在する場合は、以下の場合にのみ更新する
// - ラベルが正例か負例に変更された
// - クロール対象のサイトが一時的に200以外のステータスで前回データが取得できなかった
func (r *repository) UpdateOrCreateExample(e *model.Example) error {
	now := time.Now()
	e.UpdatedAt = now
	_, err := r.db.NamedExec(`
INSERT INTO example
( url,  final_url,  title,  description,  og_description,  og_type,  og_image,  body,  score,  is_new,  status_code,  favicon,  label,  created_at,  updated_at)
VALUES
(:url, :final_url, :title, :description, :og_description, :og_type, :og_image, :body, :score, :is_new, :status_code, :favicon, :label, :created_at, :updated_at)
ON CONFLICT (url)
DO UPDATE SET
url = :url, final_url = :final_url, title = :title,
description = :description, og_description = :og_description, og_type = :og_type, og_image = :og_image,
body = :body, score = :score, is_new = :is_new, status_code = :status_code, favicon = :favicon,
label = :label, created_at = :created_at, updated_at = :updated_at
WHERE
((EXCLUDED.label != 0) AND (example.label != EXCLUDED.label)) OR
((example.status_code != 200) AND (EXCLUDED.status_code = 200))
;`, e)
	if err != nil {
		return err
	}
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	e.Id = tmp.Id
	return nil
}

func (r *repository) UpdateScore(e *model.Example) error {
	if _, err := r.FindExampleByUlr(e.Url); err != nil {
		return err
	}
	if _, err := r.db.Exec(`UPDATE example SET score = $1, updated_at = $2 WHERE url = $3;`, e.Score, time.Now(), e.Url); err != nil {
		return err
	}
	return nil
}

func (r *repository) IncErrorCount(e *model.Example) error {
	errorCount, err := r.GetErrorCount(e)
	if err != nil {
		return err
	}
	if _, err := r.db.Exec(`UPDATE example SET error_count = $1, updated_at = $2 WHERE url = $3;`, errorCount+1, time.Now(), e.Url); err != nil {
		return err
	}
	return nil
}

func (r *repository) GetErrorCount(e *model.Example) (int, error) {
	example, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		if err == exampleNotFoundError {
			return 0, nil
		}
		return 0, err
	}
	return example.ErrorCount, nil
}

func (r *repository) UpdateFeatureVector(e *model.Example) error {
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return err
	}
	id := tmp.Id
	if _, err = r.db.Exec(`DELETE FROM feature WHERE example_id = $1;`, id); err != nil {
		return err
	}
	_, err = r.db.Exec(`INSERT INTO feature (example_id, feature) VALUES ($1, unnest(cast($2 AS TEXT[])));`, id, pq.Array(e.Fv))
	return err
}

func (r *repository) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	line := scanner.Text()
	e, err := file.ParseLine(line)
	if err != nil {
		return nil, err
	}
	err = r.UpdateOrCreateExample(e)
	if err != nil {
		return nil, err
	}
	return e, nil
}

func (r *repository) InsertExamplesFromReader(reader io.Reader) error {
	scanner := bufio.NewScanner(reader)

	for scanner.Scan() {
		_, err := r.InsertExampleFromScanner(scanner)
		if err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func (r *repository) searchExamples(query string, args ...interface{}) (model.Examples, error) {
	examples := model.Examples{}
	err := r.db.Select(&examples, query, args...)
	if err != nil {
		return nil, err
	}
	return examples, nil
}

func (r *repository) findExample(query string, args ...interface{}) (*model.Example, error) {
	e := model.Example{}

	err := r.db.Get(&e, query, args...)
	if err != nil {
		if err == sql.ErrNoRows {
			return nil, exampleNotFoundError
		}
		return nil, err
	}
	return &e, nil
}

func (r *repository) SearchExamples() (model.Examples, error) {
	query := `SELECT * FROM example;`
	return r.searchExamples(query)
}

func (r *repository) SearchRecentExamples(from time.Time, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE created_at > $1 ORDER BY updated_at DESC LIMIT $2;`
	return r.searchExamples(query, from, limit)
}

func (r *repository) SearchRecentExamplesByHost(host string, from time.Time, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE final_url like $1 || '%' AND created_at > $2 ORDER BY updated_at DESC LIMIT $3;`
	return r.searchExamples(query, host, from, limit)
}

func (r *repository) SearchExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE label = $1 ORDER BY updated_at DESC LIMIT $2;`
	return r.searchExamples(query, label, limit)
}

func (r *repository) SearchLabeledExamples(limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE label != 0 ORDER BY updated_at DESC LIMIT $1;`
	return r.searchExamples(query, limit)
}

func (r *repository) SearchPositiveExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.POSITIVE, limit)
}

func (r *repository) SearchNegativeExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.NEGATIVE, limit)
}

func (r *repository) SearchUnlabeledExamples(limit int) (model.Examples, error) {
	return r.SearchExamplesByLabel(model.UNLABELED, limit)
}

func (r *repository) SearchPositiveScoredExamples(limit int) (model.Examples, error) {
	query := `SELECT * FROM example WHERE score > 0 ORDER BY updated_at DESC LIMIT $1;`
	return r.searchExamples(query, limit)
}

func (r *repository) FindExampleByUlr(url string) (*model.Example, error) {
	query := `SELECT * FROM example WHERE url = $1;`
	return r.findExample(query, url)
}

// bodyなどは極めて長くなりえるので、DB側で絞って返すことができるようにする
func buildSelectQuery(useTruncatedField bool) string {
	title := "title"
	description := "description"
	ogDescription := "og_description"
	body := "body"

	if useTruncatedField {
		title = "LEFT(title, 200) AS title"
		description = "LEFT(description, 1000) AS description"
		ogDescription = "LEFT(og_description, 1000) AS og_description"
		body = "LEFT(body, 1000) AS body"
	}
	return fmt.Sprintf("SELECT id, label, url, final_url, %s, %s, %s, og_type, og_image, %s, score, is_new, status_code, favicon, error_count, created_at, updated_at", title, description, ogDescription, body)
}

func (r *repository) FindExampleById(id int) (*model.Example, error) {
	query := fmt.Sprintf(`%s FROM example WHERE id = $1;`, buildSelectQuery(true))
	return r.findExample(query, id)
}

func (r *repository) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	// ref: https://godoc.org/github.com/lib/pq#Array
	query := `SELECT * FROM example WHERE url = ANY($1);`
	return r.searchExamples(query, pq.Array(urls))
}

func (r *repository) SearchExamplesByIds(ids []int) (model.Examples, error) {
	if len(ids) == 0 {
		return model.Examples{}, nil
	}
	query := fmt.Sprintf(`%s FROM example WHERE id = ANY($1);`, buildSelectQuery(true))
	return r.searchExamples(query, pq.Array(ids))
}

func (r *repository) SearchExamplesByKeywords(keywords []string, aggregator string, limit int) (model.Examples, error) {
	if len(keywords) == 0 {
		return model.Examples{}, nil
	}
	regexList := make([]string, 0)
	for _, w := range keywords {
		regexList = append(regexList, fmt.Sprintf(`.*%s.*`, w))
	}
	query := fmt.Sprintf(`%s FROM example WHERE title ~* %s($1) AND label != -1 ORDER BY (label, score) DESC LIMIT $2;`, buildSelectQuery(true), aggregator)
	return r.searchExamples(query, pq.Array(regexList), limit)
}

func (r *repository) countExamplesByLabel(label model.LabelType) (int, error) {
	cnt := 0
	err := r.db.Get(&cnt, `SELECT COUNT(*) FROM example WHERE label = $1`, label)
	if err != nil {
		return 0, err
	}
	return cnt, nil
}

func (r *repository) CountPositiveExamples() (int, error) {
	return r.countExamplesByLabel(model.POSITIVE)
}

func (r *repository) CountNegativeExamples() (int, error) {
	return r.countExamplesByLabel(model.NEGATIVE)
}

func (r *repository) CountUnlabeledExamples() (int, error) {
	return r.countExamplesByLabel(model.UNLABELED)
}

func (r *repository) FindFeatureVector(e *model.Example) (feature.FeatureVector, error) {
	fv := feature.FeatureVector{}
	tmp, err := r.FindExampleByUlr(e.Url)
	if err != nil {
		return fv, err
	}
	id := tmp.Id
	query := `SELECT feature FROM feature WHERE example_id = $1;`
	err = r.db.Select(&fv, query, id)
	if err != nil {
		return fv, err
	}
	return fv, nil
}

func (r *repository) SearchFeatureVector(examples model.Examples) (map[int]feature.FeatureVector, error) {
	type Pair struct {
		ExampleId int    `db:"example_id"`
		Feature   string `db:"feature"`
	}

	fvById := make(map[int]feature.FeatureVector)
	urls := make([]string, 0)
	for _, e := range examples {
		urls = append(urls, e.Url)
	}

	tmp, err := r.SearchExamplesByUlrs(urls)
	if err != nil {
		return fvById, err
	}
	ids := make([]int, 0)
	for _, e := range tmp {
		ids = append(ids, e.Id)
	}

	query := `SELECT example_id, feature FROM feature WHERE example_id = ANY($1);`
	pairs := make([]Pair, 0)
	err = r.db.Select(&pairs, query, pq.Array(ids))
	if err != nil {
		return fvById, err
	}

	for _, pair := range pairs {
		fvById[pair.ExampleId] = append(fvById[pair.ExampleId], pair.Feature)
	}
	return fvById, nil
}

func (r *repository) DeleteAllExamples() error {
	_, err := r.db.Exec(`DELETE FROM example;`)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type Validator struct {
	root string // work dir
	code []Code // list of submissions
}

func NewValidator(dir string, train bool) (Validator, error) {
	var err error
	v := Validator{
		root: dir,
		code: make([]Code, 0, 2),
	}

	// find all 'higgsml-pred' executables
	exes := make([]string, 0)
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), runscript) {
			exes = append(exes, path)
		}
		return err
	})

	if len(exes) <= 0 {
		return v, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if len(exes) > 2 {
		return v, fmt.Errorf("hml: too many higgsml-pred executables (got=%d, max=%d)", len(exes), 2)
	}

	for _, exe := range exes {
		dir := filepath.Dir(exe)
		code, err := NewCode(dir, train)
		if err != nil {
			return v, err
		}
		code.Name = dir[len(v.root):]
		if code.Name[0] == '/' {
			code.Name = code.Name[1:]
		}
		v.code = append(v.code, code)
	}

	return v, err
}

func (v Validator) Run() error {
	var err error

	for i, code := range v.code {
		start := time.Now()
		printf("\n=== code submission #%d/%d (%s)...\n", i+1, len(v.code), code.Name)
		err = code.run()
		if err != nil {
			return err
		}
		printf("=== code submission #%d/%d (%s)... [ok] (delta=%v)\n",
			i+1, len(v.code), code.Name,
			time.Since(start),
		)
	}

	return err
}

type Code struct {
	Root    string // directory containing sources/binaries
	Name    string // name of this code submission (e.g. team/code)
	Train   string // path to training executable
	Pred    string // path to prediction executable
	Trained string // path to trained data parameters

	License string // path to LICENSE file
	Readme  string // path to README file

	DoTraining bool

	trainfile string // path to training.csv file
	testfile  string // path to test.csv file
}

func NewCode(dir string, train bool) (Code, error) {
	var err error

	code := Code{
		Root:       dir,
		Name:       filepath.Base(dir),
		Trained:    "trained.dat",
		DoTraining: train,
		trainfile:  trainfile,
		testfile:   testfile,
	}

	// FIXME: handle multiple-submissions zipfiles
	//        presumably: 1 directory per submission.

	exes := make([]string, 0)
	// find executables
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), "readme") {
			code.Readme = path
		}

		if strings.Contains(strings.ToLower(path), "license") {
			code.License = path
		}

		if strings.Contains(strings.ToLower(path), "trained.dat") {
			code.Trained = path
		}

		// FIXME: better way ?
		if !strings.Contains(fi.Mode().String(), "x") {
			return nil
		}
		exes = append(exes, path)
		// printf(">>> %s\n", path)
		if strings.Contains(strings.ToLower(path), trainscript) {
			code.Train = path
		}
		if strings.Contains(strings.ToLower(path), runscript) {
			code.Pred = path
		}
		return err
	})

	if len(exes) <= 0 {
		return code, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if code.Train == "" && code.Pred == "" {
		// take first one
		code.Train = exes[0]
		code.Pred = exes[0]
	}

	if code.Train == "" && code.Pred != "" {
		code.Train = code.Pred
	}

	if code.Train != "" && code.Pred == "" {
		code.Pred = code.Train
	}

	if code.License == "" {
		return code, fmt.Errorf("hml: could not find a LICENSE file under [%s]", code.Root)
	}

	return code, err
}

func (code Code) run() error {
	var err error

	dir := filepath.Join(code.Root, ".higgsml-work")
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	if code.DoTraining {
		err = code.run_training(dir)
		if err != nil {
			return err
		}
		printf("\n")
	}

	err = code.run_pred(dir)
	if err != nil {
		return err
	}

	return err
}

func (code Code) run_training(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run training...\n")

	cmd := exec.Command(code.Train, code.trainfile, pdir(dir, "trained.dat"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: training timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run training... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run training... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func (code Code) run_pred(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run prediction...\n")

	trained := code.Trained
	if code.DoTraining {
		// if we ran the training, then use that file instead.
		trained = pdir(dir, "trained.dat")
	}

	cmd := exec.Command(code.Pred, code.testfile, trained, pdir(dir, "scores_test.csv"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: prediction timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run prediction... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run prediction... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func pdir(dirs ...string) string {
	return filepath.Join(dirs...)
}
<commit_msg>hml-validate: exact match for higgsm-xyz scripts<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type Validator struct {
	root string // work dir
	code []Code // list of submissions
}

func NewValidator(dir string, train bool) (Validator, error) {
	var err error
	v := Validator{
		root: dir,
		code: make([]Code, 0, 2),
	}

	// find all 'higgsml-pred' executables
	exes := make([]string, 0)
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), runscript) {
			exes = append(exes, path)
		}
		return err
	})

	if len(exes) <= 0 {
		return v, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if len(exes) > 2 {
		return v, fmt.Errorf("hml: too many higgsml-pred executables (got=%d, max=%d)", len(exes), 2)
	}

	for _, exe := range exes {
		dir := filepath.Dir(exe)
		code, err := NewCode(dir, train)
		if err != nil {
			return v, err
		}
		code.Name = dir[len(v.root):]
		if code.Name[0] == '/' {
			code.Name = code.Name[1:]
		}
		v.code = append(v.code, code)
	}

	return v, err
}

func (v Validator) Run() error {
	var err error

	for i, code := range v.code {
		start := time.Now()
		printf("\n=== code submission #%d/%d (%s)...\n", i+1, len(v.code), code.Name)
		err = code.run()
		if err != nil {
			return err
		}
		printf("=== code submission #%d/%d (%s)... [ok] (delta=%v)\n",
			i+1, len(v.code), code.Name,
			time.Since(start),
		)
	}

	return err
}

type Code struct {
	Root    string // directory containing sources/binaries
	Name    string // name of this code submission (e.g. team/code)
	Train   string // path to training executable
	Pred    string // path to prediction executable
	Trained string // path to trained data parameters

	License string // path to LICENSE file
	Readme  string // path to README file

	DoTraining bool

	trainfile string // path to training.csv file
	testfile  string // path to test.csv file
}

func NewCode(dir string, train bool) (Code, error) {
	var err error

	code := Code{
		Root:       dir,
		Name:       filepath.Base(dir),
		Trained:    "trained.dat",
		DoTraining: train,
		trainfile:  trainfile,
		testfile:   testfile,
	}

	// FIXME: handle multiple-submissions zipfiles
	//        presumably: 1 directory per submission.

	exes := make([]string, 0)
	// find executables
	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if fi.IsDir() {
			return nil
		}
		if strings.Contains(strings.ToLower(path), "readme") {
			code.Readme = path
		}

		if strings.Contains(strings.ToLower(path), "license") {
			code.License = path
		}

		if strings.Contains(strings.ToLower(path), "trained.dat") {
			code.Trained = path
		}

		// FIXME: better way ?
		if !strings.Contains(fi.Mode().String(), "x") {
			return nil
		}
		exes = append(exes, path)
		// printf(">>> %s\n", path)
		if strings.HasSuffix(strings.ToLower(path), trainscript) {
			code.Train = path
		}
		if strings.HasSuffix(strings.ToLower(path), runscript) {
			code.Pred = path
		}
		return err
	})

	if len(exes) <= 0 {
		return code, fmt.Errorf("hml: could not find any suitable executable in zip-file")
	}

	if code.Train == "" && code.Pred == "" {
		// take first one
		code.Train = exes[0]
		code.Pred = exes[0]
	}

	if code.Train == "" && code.Pred != "" {
		code.Train = code.Pred
	}

	if code.Train != "" && code.Pred == "" {
		code.Pred = code.Train
	}

	if code.License == "" {
		return code, fmt.Errorf("hml: could not find a LICENSE file under [%s]", code.Root)
	}

	return code, err
}

func (code Code) run() error {
	var err error

	dir := filepath.Join(code.Root, ".higgsml-work")
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	if code.DoTraining {
		err = code.run_training(dir)
		if err != nil {
			return err
		}
		printf("\n")
	}

	err = code.run_pred(dir)
	if err != nil {
		return err
	}

	return err
}

func (code Code) run_training(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run training...\n")

	cmd := exec.Command(code.Train, code.trainfile, pdir(dir, "trained.dat"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: training timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run training... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run training... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func (code Code) run_pred(dir string) error {
	var err error
	errch := make(chan error)

	printf("::: run prediction...\n")

	trained := code.Trained
	if code.DoTraining {
		// if we ran the training, then use that file instead.
		trained = pdir(dir, "trained.dat")
	}

	cmd := exec.Command(code.Pred, code.testfile, trained, pdir(dir, "scores_test.csv"))
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin
	cmd.Dir = code.Root

	start := time.Now()
	go func() {
		err = cmd.Start()
		if err != nil {
			errch <- err
			return
		}
		errch <- cmd.Wait()
	}()

	duration := 1 * time.Hour
	select {
	case <-time.After(duration):
		cmd.Process.Kill()
		return fmt.Errorf("hml: prediction timed out (%v)\n", duration)
	case err = <-errch:
		break
	}

	if err != nil {
		printf("::: run prediction... [ERR] (delta=%v)\n", time.Since(start))
		return err
	}

	printf("::: run prediction... [ok] (delta=%v)\n", time.Since(start))
	return err
}

func pdir(dirs ...string) string {
	return filepath.Join(dirs...)
}
<|endoftext|>


<commit_before>package hook

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/buildkite/agent/v3/bootstrap/shell"
	"github.com/buildkite/agent/v3/env"
	"github.com/buildkite/agent/v3/utils"
)

const (
	hookExitStatusEnv = `BUILDKITE_HOOK_EXIT_STATUS`
	hookWorkingDirEnv = `BUILDKITE_HOOK_WORKING_DIR`
)

// Hooks get "sourced" into the bootstrap in the sense that they get the
// environment set for them and then we capture any extra environment variables
// that are exported in the script.

// The tricky thing is that it's impossible to grab the ENV of a child process
// before it finishes, so we've got an awesome (ugly) hack to get around this.
// We write the ENV to file, run the hook and then write the ENV back to another file.
// Then we can use the diff of the two to figure out what changes to make to the
// bootstrap. Horrible, but effective.

// ScriptWrapper wraps a hook script with env collection and then provides
// a way to get the difference between the environment before the hook is run and
// after it
type ScriptWrapper struct {
	hookPath      string
	scriptFile    *os.File
	beforeEnvFile *os.File
	afterEnvFile  *os.File
	beforeWd      string
}

type hookScriptChanges struct {
	Env *env.Environment
	Dir string
}

// CreateScriptWrapper creates and configures a ScriptWrapper.
// Writes temporary files to the filesystem.
func CreateScriptWrapper(hookPath string) (*ScriptWrapper, error) {
	var wrap = &ScriptWrapper{
		hookPath: hookPath,
	}

	var err error
	var scriptFileName string = `buildkite-agent-bootstrap-hook-runner`
	var isBashHook bool
	var isPwshHook bool
	var isWindows = runtime.GOOS == "windows"

	// we use bash hooks for scripts with no extension, otherwise on windows
	// we probably need a .bat extension
	if filepath.Ext(hookPath) == ".ps1" {
		isPwshHook = true
		scriptFileName += ".ps1"
	} else if filepath.Ext(hookPath) == "" {
		isBashHook = true
	} else if isWindows {
		scriptFileName += ".bat"
	}

	// Create a temporary file that we'll put the hook runner code in
	wrap.scriptFile, err = shell.TempFileWithExtension(scriptFileName)
	if err != nil {
		return nil, err
	}
	defer wrap.scriptFile.Close()

	// We'll pump the ENV before the hook into this temp file
	wrap.beforeEnvFile, err = shell.TempFileWithExtension(
		`buildkite-agent-bootstrap-hook-env-before`,
	)
	if err != nil {
		return nil, err
	}
	wrap.beforeEnvFile.Close()

	// We'll then pump the ENV _after_ the hook into this temp file
	wrap.afterEnvFile, err = shell.TempFileWithExtension(
		`buildkite-agent-bootstrap-hook-env-after`,
	)
	if err != nil {
		return nil, err
	}
	wrap.afterEnvFile.Close()

	absolutePathToHook, err := filepath.Abs(wrap.hookPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to find absolute path to \"%s\" (%s)", wrap.hookPath, err)
	}

	wrap.beforeWd, err = os.Getwd()
	if err != nil {
		return nil, err
	}

	// Create the hook runner code
	var script string
	if isWindows && !isBashHook && !isPwshHook {
		script = "@echo off\n" +
			"SETLOCAL ENABLEDELAYEDEXPANSION\n" +
			"SET > \"" + wrap.beforeEnvFile.Name() + "\"\n" +
			"CALL \"" + absolutePathToHook + "\"\n" +
			"SET " + hookExitStatusEnv + "=!ERRORLEVEL!\n" +
			"SET " + hookWorkingDirEnv + "=%CD%\n" +
			"SET > \"" + wrap.afterEnvFile.Name() + "\"\n" +
			"EXIT %" + hookExitStatusEnv + "%"
	} else if isWindows && isPwshHook {
		script = `$ErrorActionPreference = "STOP"\n` +
			`Get-ChildItem Env: | Foreach-Object {$($_.Name)=$($_.Value)"} | Set-Content "` + wrap.beforeEnvFile.Name() + `\n` +
			absolutePathToHook + `\n` +
			`if ($LASTEXITCODE -eq $null) {$Env:` + hookExitStatusEnv + ` = 0} else {$Env:` + hookExitStatusEnv + ` = $LASTEXITCODE}\n` +
			`$Env:` + hookWorkingDirEnv + ` = $PWD | Select-Object -ExpandProperty Path\n` +
			`Get-ChildItem Env: | Foreach-Object {"$($_.Name)=$($_.Value)"} | Set-Content "` + wrap.afterEnvFile.Name() + `"\n` +
			`exit $Env:` + hookExitStatusEnv
	} else {
		script = "export -p > \"" + filepath.ToSlash(wrap.beforeEnvFile.Name()) + "\"\n" +
			". \"" + filepath.ToSlash(absolutePathToHook) + "\"\n" +
			"export " + hookExitStatusEnv + "=$?\n" +
			"export " + hookWorkingDirEnv + "=$PWD\n" +
			"export -p > \"" + filepath.ToSlash(wrap.afterEnvFile.Name()) + "\"\n" +
			"exit $" + hookExitStatusEnv
	}

	// Write the hook script to the runner then close the file so we can run it
	_, err = wrap.scriptFile.WriteString(script)
	if err != nil {
		return nil, err
	}

	// Make script executable
	err = utils.ChmodExecutable(wrap.scriptFile.Name())
	if err != nil {
		return wrap, err
	}

	return wrap, nil
}

// Path returns the path to the wrapper script, this is the one that should be executed
func (wrap *ScriptWrapper) Path() string {
	return wrap.scriptFile.Name()
}

// Close cleans up the wrapper script and the environment files
func (wrap *ScriptWrapper) Close() {
	os.Remove(wrap.scriptFile.Name())
	os.Remove(wrap.beforeEnvFile.Name())
	os.Remove(wrap.afterEnvFile.Name())
}

// Changes returns the changes in the environment and working dir after the hook script runs
func (wrap *ScriptWrapper) Changes() (hookScriptChanges, error) {
	beforeEnvContents, err := ioutil.ReadFile(wrap.beforeEnvFile.Name())
	if err != nil {
		return hookScriptChanges{}, fmt.Errorf("Failed to read \"%s\" (%s)", wrap.beforeEnvFile.Name(), err)
	}

	afterEnvContents, err := ioutil.ReadFile(wrap.afterEnvFile.Name())
	if err != nil {
		return hookScriptChanges{}, fmt.Errorf("Failed to read \"%s\" (%s)", wrap.afterEnvFile.Name(), err)
	}

	beforeEnv := env.FromExport(string(beforeEnvContents))
	afterEnv := env.FromExport(string(afterEnvContents))
	diff := afterEnv.Diff(beforeEnv)
	wd, _ := diff.Get(hookWorkingDirEnv)

	diff.Remove(hookExitStatusEnv)
	diff.Remove(hookWorkingDirEnv)

	return hookScriptChanges{Env: diff, Dir: wd}, nil
}
<commit_msg>Update scriptwrapper to use the env.Diff type in Changes()<commit_after>package hook

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"

	"github.com/buildkite/agent/v3/bootstrap/shell"
	"github.com/buildkite/agent/v3/env"
	"github.com/buildkite/agent/v3/utils"
)

const (
	hookExitStatusEnv = `BUILDKITE_HOOK_EXIT_STATUS`
	hookWorkingDirEnv = `BUILDKITE_HOOK_WORKING_DIR`
)

// Hooks get "sourced" into the bootstrap in the sense that they get the
// environment set for them and then we capture any extra environment variables
// that are exported in the script.

// The tricky thing is that it's impossible to grab the ENV of a child process
// before it finishes, so we've got an awesome (ugly) hack to get around this.
// We write the ENV to file, run the hook and then write the ENV back to another file.
// Then we can use the diff of the two to figure out what changes to make to the
// bootstrap. Horrible, but effective.

// ScriptWrapper wraps a hook script with env collection and then provides
// a way to get the difference between the environment before the hook is run and
// after it
type ScriptWrapper struct {
	hookPath      string
	scriptFile    *os.File
	beforeEnvFile *os.File
	afterEnvFile  *os.File
	beforeWd      string
}

type hookScriptChanges struct {
	Diff env.Diff
	Dir string
}

// CreateScriptWrapper creates and configures a ScriptWrapper.
// Writes temporary files to the filesystem.
func CreateScriptWrapper(hookPath string) (*ScriptWrapper, error) {
	var wrap = &ScriptWrapper{
		hookPath: hookPath,
	}

	var err error
	var scriptFileName string = `buildkite-agent-bootstrap-hook-runner`
	var isBashHook bool
	var isPwshHook bool
	var isWindows = runtime.GOOS == "windows"

	// we use bash hooks for scripts with no extension, otherwise on windows
	// we probably need a .bat extension
	if filepath.Ext(hookPath) == ".ps1" {
		isPwshHook = true
		scriptFileName += ".ps1"
	} else if filepath.Ext(hookPath) == "" {
		isBashHook = true
	} else if isWindows {
		scriptFileName += ".bat"
	}

	// Create a temporary file that we'll put the hook runner code in
	wrap.scriptFile, err = shell.TempFileWithExtension(scriptFileName)
	if err != nil {
		return nil, err
	}
	defer wrap.scriptFile.Close()

	// We'll pump the ENV before the hook into this temp file
	wrap.beforeEnvFile, err = shell.TempFileWithExtension(
		`buildkite-agent-bootstrap-hook-env-before`,
	)
	if err != nil {
		return nil, err
	}
	wrap.beforeEnvFile.Close()

	// We'll then pump the ENV _after_ the hook into this temp file
	wrap.afterEnvFile, err = shell.TempFileWithExtension(
		`buildkite-agent-bootstrap-hook-env-after`,
	)
	if err != nil {
		return nil, err
	}
	wrap.afterEnvFile.Close()

	absolutePathToHook, err := filepath.Abs(wrap.hookPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to find absolute path to \"%s\" (%s)", wrap.hookPath, err)
	}

	wrap.beforeWd, err = os.Getwd()
	if err != nil {
		return nil, err
	}

	// Create the hook runner code
	var script string
	if isWindows && !isBashHook && !isPwshHook {
		script = "@echo off\n" +
			"SETLOCAL ENABLEDELAYEDEXPANSION\n" +
			"SET > \"" + wrap.beforeEnvFile.Name() + "\"\n" +
			"CALL \"" + absolutePathToHook + "\"\n" +
			"SET " + hookExitStatusEnv + "=!ERRORLEVEL!\n" +
			"SET " + hookWorkingDirEnv + "=%CD%\n" +
			"SET > \"" + wrap.afterEnvFile.Name() + "\"\n" +
			"EXIT %" + hookExitStatusEnv + "%"
	} else if isWindows && isPwshHook {
		script = `$ErrorActionPreference = "STOP"\n` +
			`Get-ChildItem Env: | Foreach-Object {$($_.Name)=$($_.Value)"} | Set-Content "` + wrap.beforeEnvFile.Name() + `\n` +
			absolutePathToHook + `\n` +
			`if ($LASTEXITCODE -eq $null) {$Env:` + hookExitStatusEnv + ` = 0} else {$Env:` + hookExitStatusEnv + ` = $LASTEXITCODE}\n` +
			`$Env:` + hookWorkingDirEnv + ` = $PWD | Select-Object -ExpandProperty Path\n` +
			`Get-ChildItem Env: | Foreach-Object {"$($_.Name)=$($_.Value)"} | Set-Content "` + wrap.afterEnvFile.Name() + `"\n` +
			`exit $Env:` + hookExitStatusEnv
	} else {
		script = "export -p > \"" + filepath.ToSlash(wrap.beforeEnvFile.Name()) + "\"\n" +
			". \"" + filepath.ToSlash(absolutePathToHook) + "\"\n" +
			"export " + hookExitStatusEnv + "=$?\n" +
			"export " + hookWorkingDirEnv + "=$PWD\n" +
			"export -p > \"" + filepath.ToSlash(wrap.afterEnvFile.Name()) + "\"\n" +
			"exit $" + hookExitStatusEnv
	}

	// Write the hook script to the runner then close the file so we can run it
	_, err = wrap.scriptFile.WriteString(script)
	if err != nil {
		return nil, err
	}

	// Make script executable
	err = utils.ChmodExecutable(wrap.scriptFile.Name())
	if err != nil {
		return wrap, err
	}

	return wrap, nil
}

// Path returns the path to the wrapper script, this is the one that should be executed
func (wrap *ScriptWrapper) Path() string {
	return wrap.scriptFile.Name()
}

// Close cleans up the wrapper script and the environment files
func (wrap *ScriptWrapper) Close() {
	os.Remove(wrap.scriptFile.Name())
	os.Remove(wrap.beforeEnvFile.Name())
	os.Remove(wrap.afterEnvFile.Name())
}

// Changes returns the changes in the environment and working dir after the hook script runs
func (wrap *ScriptWrapper) Changes() (hookScriptChanges, error) {
	beforeEnvContents, err := ioutil.ReadFile(wrap.beforeEnvFile.Name())
	if err != nil {
		return hookScriptChanges{}, fmt.Errorf("Failed to read \"%s\" (%s)", wrap.beforeEnvFile.Name(), err)
	}

	afterEnvContents, err := ioutil.ReadFile(wrap.afterEnvFile.Name())
	if err != nil {
		return hookScriptChanges{}, fmt.Errorf("Failed to read \"%s\" (%s)", wrap.afterEnvFile.Name(), err)
	}

	beforeEnv := env.FromExport(string(beforeEnvContents))
	afterEnv := env.FromExport(string(afterEnvContents))
	diff := afterEnv.Diff(beforeEnv)

	wd := wrap.getAfterWd(diff)

	diff.Remove(hookExitStatusEnv)
	diff.Remove(hookWorkingDirEnv)

	return hookScriptChanges{Diff: diff, Dir: wd}, nil
}

func (wrap *ScriptWrapper) getAfterWd(diff env.Diff) string {
	addedVar, ok := diff.Added[hookWorkingDirEnv]
	if ok {
		return addedVar
	}

	changedVar, ok := diff.Changed[hookWorkingDirEnv]
	if ok {
		return changedVar.New
	}

	return wrap.beforeWd
}
<|endoftext|>


<commit_before>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum)

		secondAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum)

		//Quick hack to make sure that firstAccumulator and secondAccumulator have same number of generations
		//( even if they don't really)

		for len(firstAccumulator) < len(secondAccumulator) {
			firstAccumulator = firstAccumulator.addGeneration()
		}

		for len(secondAccumulator) < len(firstAccumulator) {
			secondAccumulator = secondAccumulator.addGeneration()
		}

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		for generation := 0; generation < len(firstAccumulator); generation++ {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

		//TODO: figure out why the printed output does not return the same steps (different order is fine).
		//Does this mean there's some unexpected non-deterministic behavior in chainSearcher? Perhaps in the precise moment we
		//notice an inconsistency and stop searching more?
		//The problem seems to be down the eventually-inconsistent branch. Depending on order we visit, we get either 8 or 9
		//cells seen. Probably the right answer is to throw out the entire last generation if any inconsitency found.

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map (length " + strconv.Itoa(len(c)) + ")\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

func (c chainSearcherAccumulator) addGeneration() chainSearcherAccumulator {
	newGeneration := make(chainSearcherGenerationDetails)
	result := append(c, newGeneration)
	if len(result) > 1 {
		oldGeneration := result[len(result)-2]
		//Accumulate forward old generation
		for key, val := range oldGeneration {
			newGeneration[key] = val
		}
	}
	return result
}

func chainSearcher(maxGeneration int, cell *Cell, numToApply int) chainSearcherAccumulator {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	var result chainSearcherAccumulator

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		0,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation > maxGeneration {
			break
		}

		for len(result) < step.generation+1 {
			result = result.addGeneration()
		}

		generationDetails := result[step.generation]

		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		if currentVal, ok := generationDetails[step.cell.ref()]; ok {
			if currentVal != step.numToApply {
				//Found a contradiction! We can bail from processing any more because this branch leads inexorably
				//to a contradiction.
				return result
			}
		}
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation + 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

	return result

}
<commit_msg>TESTS FAIL. Fixed the bug where we got non-deterministic results on the same puzzle by throwing out the whole last generation when we find an inconsistency. Now we return the same list of steps (even though it still isn't perfect)<commit_after>package sudoku

import (
	"container/list"
	"fmt"
	"log"
	"strconv"
)

type forcingChainsTechnique struct {
	*basicSolveTechnique
}

func (self *forcingChainsTechnique) HumanLikelihood() float64 {
	//TODO: figure out what the baseDifficulty should be
	return self.difficultyHelper(200.0)
}

func (self *forcingChainsTechnique) Description(step *SolveStep) string {
	//TODO: implement this
	return "ERROR: NOT IMPLEMENTED"
}

func (self *forcingChainsTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {
	//TODO: test that this will find multiple if they exist.
	//TODO: Implement this.

	getter := grid.queue().DefaultGetter()

	_MAX_IMPLICATION_STEPS := 6

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		candidate := getter.GetSmallerThan(3)

		if candidate == nil {
			break
		}

		candidateCell := candidate.(*Cell)

		if len(candidateCell.Possibilities()) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		firstPossibilityNum := candidateCell.Possibilities()[0]
		secondPossibilityNum := candidateCell.Possibilities()[1]

		firstGrid := grid.Copy()
		secondGrid := grid.Copy()

		//Check that the neighbor isn't just already having a single possibility, because then this technique is overkill.

		firstAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(firstGrid),
			firstPossibilityNum)

		secondAccumulator := chainSearcher(_MAX_IMPLICATION_STEPS,
			candidateCell.InGrid(secondGrid),
			secondPossibilityNum)

		//Quick hack to make sure that firstAccumulator and secondAccumulator have same number of generations
		//( even if they don't really)

		for len(firstAccumulator) < len(secondAccumulator) {
			firstAccumulator = firstAccumulator.addGeneration()
		}

		for len(secondAccumulator) < len(firstAccumulator) {
			secondAccumulator = secondAccumulator.addGeneration()
		}

		//TODO:Check if the sets overlap.

		doPrint := candidateCell.Row() == 1 && candidateCell.Col() == 0

		//For these debugging purposes, only print out the candidateCell we know to be interesting in the test case.
		if doPrint {
			log.Println(firstAccumulator)
			log.Println(secondAccumulator)
		}

		//See if either branch, at some generation, has the same cell forced to the same number in either generation.

		for generation := 0; generation < len(firstAccumulator); generation++ {

			//Check for any overlap at the last generation
			firstAffectedCells := firstAccumulator[generation]
			secondAffectedCells := secondAccumulator[generation]

			for key, val := range firstAffectedCells {

				//Skip the candidateCell, because that's not a meaningful overlap--we set that one as a way of branching!
				if key == candidateCell.ref() {
					continue
				}

				if num, ok := secondAffectedCells[key]; ok {
					//Found cell overlap! ... is the forced number the same?
					if val == num {
						//Yup, seems like we've found a cell that is forced to the same value on either branch.
						step := &SolveStep{self,
							CellSlice{key.Cell(grid)},
							IntSlice{val},
							CellSlice{candidateCell},
							candidateCell.Possibilities(),
						}

						if doPrint {
							log.Println(step)
							log.Println("Candidate Cell", candidateCell.ref())
						}

						if step.IsUseful(grid) {
							if doPrint {
								log.Println("Found solution on generation: ", generation)
							}
							select {
							case results <- step:
							case <-done:
								return
							}
						}
					}
				}
			}
		}

		//TODO: figure out why the tests are coming back with different answers, even when only looking at the key cell
		//that should work from the example.
		//TODO: we should prefer solutions where the total implications on both branches are minimized.
		//For example, if only one implication is requried on left, but 4 are on right, that's preferable to one where
		//three implications are required on both sides.
		//TODO: figure out a way to only compute a generation if required on each branch (don't compute all the way to _MAX_IMPLICATIONS to start)

	}
}

type chainSearcherGenerationDetails map[cellRef]int

func (c chainSearcherGenerationDetails) String() string {
	result := "Begin map (length " + strconv.Itoa(len(c)) + ")\n"
	for cell, num := range c {
		result += "\t" + cell.String() + " : " + strconv.Itoa(num) + "\n"
	}
	result += "End map\n"
	return result
}

type chainSearcherAccumulator []chainSearcherGenerationDetails

func (c chainSearcherAccumulator) String() string {
	result := "Accumulator[\n"
	for _, rec := range c {
		result += fmt.Sprintf("%s\n", rec)
	}
	result += "]\n"
	return result
}

func (c chainSearcherAccumulator) addGeneration() chainSearcherAccumulator {
	newGeneration := make(chainSearcherGenerationDetails)
	result := append(c, newGeneration)
	if len(result) > 1 {
		oldGeneration := result[len(result)-2]
		//Accumulate forward old generation
		for key, val := range oldGeneration {
			newGeneration[key] = val
		}
	}
	return result
}

func chainSearcher(maxGeneration int, cell *Cell, numToApply int) chainSearcherAccumulator {

	//Chainsearcher implements a BFS over implications forward given the starting point.
	//It collects its results in the provided chainSearcherAccumulator.

	//the first time we cross over into a new generation, we should do a one-time copy of the old generation
	//into the new.
	//At any write, if we notice that we'd be overwriting to a different value, we can bail out (how would
	//we mark that we bailed early), since we've run into an inconsistency down this branch and following
	//it further is not useful.

	type modificationToMake struct {
		generation int
		cell       *Cell
		numToApply int
	}

	var result chainSearcherAccumulator

	workSteps := list.New()

	//Add the first workstep.
	workSteps.PushBack(modificationToMake{
		0,
		cell,
		numToApply,
	})

	var step modificationToMake

	e := workSteps.Front()

	for e != nil {

		workSteps.Remove(e)

		switch t := e.Value.(type) {
		case modificationToMake:
			step = t
		default:
			panic("Found unexpected type in workSteps list")
		}

		if step.generation > maxGeneration {
			break
		}

		for len(result) < step.generation+1 {
			result = result.addGeneration()
		}

		generationDetails := result[step.generation]

		cellsToVisit := step.cell.Neighbors().FilterByPossible(step.numToApply).FilterByNumPossibilities(2)

		step.cell.SetNumber(step.numToApply)

		if currentVal, ok := generationDetails[step.cell.ref()]; ok {
			if currentVal != step.numToApply {
				//Found a contradiction! We can bail from processing any more because this branch leads inexorably
				//to a contradiction.

				//However, this last generation--the one we found the inconsistency in--needs to be thrown out.

				return result[:len(result)-1]
			}
		}
		generationDetails[step.cell.ref()] = step.numToApply

		for _, cellToVisit := range cellsToVisit {
			possibilities := cellToVisit.Possibilities()

			if len(possibilities) != 1 {
				panic("Expected the cell to have one possibility")
			}

			forcedNum := possibilities[0]

			//Each branch modifies the grid, so create a new copy
			newGrid := cellToVisit.grid.Copy()
			cellToVisit = cellToVisit.InGrid(newGrid)

			workSteps.PushBack(modificationToMake{
				step.generation + 1,
				cellToVisit,
				forcedNum,
			})

		}

		e = workSteps.Front()

	}

	return result

}
<|endoftext|>


<commit_before>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		r.Header.Add("X-Forwarded-For", xff(r))
		r.Header.Add("X-Real-IP", xff(r))
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	s.mu.RLock()
	defer s.mu.RUnlock()
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.Get(h)
		if f == "" {
			return nil
		}
		s.proxy[h] = append(s.proxy[h], Proxy{0, f, makeHandler(f)})
	}
	return s.Next(h)
}

func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		s.mu.Lock()
		s.mu.Unlock()
		time.Sleep(probe)
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<commit_msg>playing with n+1<commit_after>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		r.Header.Add("X-Forwarded-For", xff(r))
		r.Header.Add("X-Real-IP", xff(r))
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	s.mu.RLock()
	defer s.mu.RUnlock()
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		f, _ := rc.Get(h)
		if f == "" {
			return nil
		}
		s.proxy[h] = append(s.proxy[h], Proxy{0, f, makeHandler(f)})
	}
	return s.Next(h)
}

/* TODO: Implement at least one decent loadbalance algorithm
Using n+1 to see it working*/
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	s.backend[h]++
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		s.mu.Lock()
		s.mu.Unlock()
		time.Sleep(probe)
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"errors"
	"fmt"
	"github.com/fiorix/go-redis/redis"
	"github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	proxy   map[string][]Proxy
	backend map[string]int
	vcount  map[string]map[string]int
}

type Proxy struct {
	//	last    time.Time
	Backend string
	handler http.Handler
}

func Listen(addr string) net.Listener {
	var l net.Listener
	var err error

	l, err = net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	s.vcount = make(map[string]map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	vhost := req.Host
	if i := strings.Index(vhost, ":"); i >= 0 {
		vhost = vhost[:i]
	}

	s.mu.RLock()
	_, ok := s.proxy[vhost]
	if !ok {
		err := s.populate_proxies(vhost, false)
		if err != nil {
			utils.Log(fmt.Sprintf("%s for vhost %s", err, vhost))
			return nil
		}
	}
	s.mu.RUnlock()
	return s.Next(vhost)
}

func (s *Server) populate_proxies(vhost string, rebalance bool) (err error) {
	f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", vhost), 0, -1, true)
	if len(f) == 0 {
		if len(s.proxy[vhost]) > 0 {
			delete(s.proxy, vhost)
			delete(s.backend, vhost)
			delete(s.vcount, vhost)
		}
		return errors.New("Backend list is empty")
	}

	var url string
	for _, be := range f {
		count, err := strconv.Atoi(be)
		if err != nil {
			url = be
			continue
		}
		backend := fmt.Sprintf("http://%s", url)
		for r := 1; r <= count; r++ {
			if rebalance == true && r <= s.vcount[vhost][backend] {
				continue
			}
			s.proxy[vhost] = append(s.proxy[vhost], Proxy{backend, makeHandler(url)})
			v, ready := s.vcount[vhost]
			if !ready {
				v = make(map[string]int)
				s.vcount[vhost] = v
			}
			s.vcount[vhost][backend]++
			utils.Log("Backend %s with %s handlers for %s", backend, s.vcount[vhost][backend], vhost)
		}
	}
	return
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(vhost string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[vhost]++
	total := len(s.proxy[vhost])
	if s.backend[vhost] >= total {
		s.backend[vhost] = 0
	}
	utils.Log(fmt.Sprintf(
		"Using backend: %s Url: %s", s.proxy[vhost][s.backend[vhost]].Backend, vhost))
	return s.proxy[vhost][s.backend[vhost]].handler
}

/* TODO: Implement more probes */
func (s *Server) probe_backends(probe time.Duration) {
	transport := http.Transport{Dial: dialTimeout}
	client := &http.Client{
		Transport: &transport,
	}

	for {
		time.Sleep(probe)
		for vhost, backends := range s.proxy {
			err := s.populate_proxies(vhost, true)
			if err != nil {
				utils.Log(fmt.Sprintf("Cleaned entries from vhost: %s", vhost))
				continue
			}
			is_dead := make(map[string]bool)
			removed := 0
			for backend := range backends {
				backend = backend - removed
				utils.Log(fmt.Sprintf(
					"vhost: %s backends: %s", vhost, s.proxy[vhost][backend].Backend))
				if is_dead[s.proxy[vhost][backend].Backend] == true {
					utils.Log(fmt.Sprintf("Removing dead backend: %s", s.proxy[vhost][backend].Backend))
					s.mu.Lock()
					s.proxy[vhost] = s.proxy[vhost][:backend+copy(s.proxy[vhost][backend:], s.proxy[vhost][backend+1:])]
					s.vcount[vhost][s.proxy[vhost][backend].Backend]--
					s.mu.Unlock()
					removed++
					continue
				}

				_, err := client.Get(s.proxy[vhost][backend].Backend)
				if err != nil {
					utils.Log(fmt.Sprintf("Removing dead backend: %s with error %s", s.proxy[vhost][backend].Backend, err))
					is_dead[s.proxy[vhost][backend].Backend] = true
					s.mu.Lock()
					s.proxy[vhost] = s.proxy[vhost][:backend+copy(s.proxy[vhost][backend:], s.proxy[vhost][backend+1:])]
					s.vcount[vhost][s.proxy[vhost][backend].Backend]--
					s.mu.Unlock()
					removed++
				} else {
					utils.Log(fmt.Sprintf("Alive: %s", s.proxy[vhost][backend].Backend))

				}
			}

		}

	}
}

func dialTimeout(network, addr string) (net.Conn, error) {
	timeout := time.Duration(2 * time.Second)
	return net.DialTimeout(network, addr, timeout)
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<commit_msg>missing fmt<commit_after>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"errors"
	"fmt"
	"github.com/fiorix/go-redis/redis"
	"github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	proxy   map[string][]Proxy
	backend map[string]int
	vcount  map[string]map[string]int
}

type Proxy struct {
	//	last    time.Time
	Backend string
	handler http.Handler
}

func Listen(addr string) net.Listener {
	var l net.Listener
	var err error

	l, err = net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	s.vcount = make(map[string]map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	vhost := req.Host
	if i := strings.Index(vhost, ":"); i >= 0 {
		vhost = vhost[:i]
	}

	s.mu.RLock()
	_, ok := s.proxy[vhost]
	if !ok {
		err := s.populate_proxies(vhost, false)
		if err != nil {
			utils.Log(fmt.Sprintf("%s for vhost %s", err, vhost))
			return nil
		}
	}
	s.mu.RUnlock()
	return s.Next(vhost)
}

func (s *Server) populate_proxies(vhost string, rebalance bool) (err error) {
	f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", vhost), 0, -1, true)
	if len(f) == 0 {
		if len(s.proxy[vhost]) > 0 {
			delete(s.proxy, vhost)
			delete(s.backend, vhost)
			delete(s.vcount, vhost)
		}
		return errors.New("Backend list is empty")
	}

	var url string
	for _, be := range f {
		count, err := strconv.Atoi(be)
		if err != nil {
			url = be
			continue
		}
		backend := fmt.Sprintf("http://%s", url)
		for r := 1; r <= count; r++ {
			if rebalance == true && r <= s.vcount[vhost][backend] {
				continue
			}
			s.proxy[vhost] = append(s.proxy[vhost], Proxy{backend, makeHandler(url)})
			v, ready := s.vcount[vhost]
			if !ready {
				v = make(map[string]int)
				s.vcount[vhost] = v
			}
			s.vcount[vhost][backend]++
			utils.Log(fmt.Sprintf("Backend %s with %s handlers for %s", backend, s.vcount[vhost][backend], vhost))
		}
	}
	return
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(vhost string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[vhost]++
	total := len(s.proxy[vhost])
	if s.backend[vhost] >= total {
		s.backend[vhost] = 0
	}
	utils.Log(fmt.Sprintf(
		"Using backend: %s Url: %s", s.proxy[vhost][s.backend[vhost]].Backend, vhost))
	return s.proxy[vhost][s.backend[vhost]].handler
}

/* TODO: Implement more probes */
func (s *Server) probe_backends(probe time.Duration) {
	transport := http.Transport{Dial: dialTimeout}
	client := &http.Client{
		Transport: &transport,
	}

	for {
		time.Sleep(probe)
		for vhost, backends := range s.proxy {
			err := s.populate_proxies(vhost, true)
			if err != nil {
				utils.Log(fmt.Sprintf("Cleaned entries from vhost: %s", vhost))
				continue
			}
			is_dead := make(map[string]bool)
			removed := 0
			for backend := range backends {
				backend = backend - removed
				utils.Log(fmt.Sprintf(
					"vhost: %s backends: %s", vhost, s.proxy[vhost][backend].Backend))
				if is_dead[s.proxy[vhost][backend].Backend] == true {
					utils.Log(fmt.Sprintf("Removing dead backend: %s", s.proxy[vhost][backend].Backend))
					s.mu.Lock()
					s.proxy[vhost] = s.proxy[vhost][:backend+copy(s.proxy[vhost][backend:], s.proxy[vhost][backend+1:])]
					s.vcount[vhost][s.proxy[vhost][backend].Backend]--
					s.mu.Unlock()
					removed++
					continue
				}

				_, err := client.Get(s.proxy[vhost][backend].Backend)
				if err != nil {
					utils.Log(fmt.Sprintf("Removing dead backend: %s with error %s", s.proxy[vhost][backend].Backend, err))
					is_dead[s.proxy[vhost][backend].Backend] = true
					s.mu.Lock()
					s.proxy[vhost] = s.proxy[vhost][:backend+copy(s.proxy[vhost][backend:], s.proxy[vhost][backend+1:])]
					s.vcount[vhost][s.proxy[vhost][backend].Backend]--
					s.mu.Unlock()
					removed++
				} else {
					utils.Log(fmt.Sprintf("Alive: %s", s.proxy[vhost][backend].Backend))

				}
			}

		}

	}
}

func dialTimeout(network, addr string) (net.Conn, error) {
	timeout := time.Duration(2 * time.Second)
	return net.DialTimeout(network, addr, timeout)
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"log"

	"encoding/json"
	"fmt"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/miolini/bankgo/rpc/client"
)

type BalanceEntry struct {
	UserID int64
	Value  int64
}

type App struct {
	router *echo.Echo
	client *client.BalanceStorageClient
}

func (app *App) Init(rpcAddr string) error {
	log.Printf("httpapi init")
	client, err := client.Connect(rpcAddr)
	if err != nil {
		return err
	}
	app.client = client
	app.router = echo.New()
	app.router.Use(middleware.Logger(), middleware.Recover())
	app.router.Use(echoJsonCheckErrorMW())
	app.router.Get("/balances", app.handleGetBalances)
	app.router.Post("/transaction", app.handlePostTransaction)
	return nil
}

func (app *App) Run(addr string) {
	app.router.Run(addr)
}

func (app *App) handleGetBalances(ctx *echo.Context) error {
	records, err := app.client.AllBalances()
	if err != nil {
		return err
	}
	core.EchoReplyJson(ctx, records)
	return nil
}

func (app *App) handlePostTransaction(ctx *echo.Context) error {
	var balance BalanceEntry
	err := json.NewDecoder(ctx.Request().Body).Decode(&balance)
	if err != nil {
		return replyJsonError(ctx, fmt.Errorf("json parse error: %s", err))
	}
	balance.Value, err = app.client.SetValue(balance.UserID, balance.Value)
	if err != nil {
		return replyJsonError(ctx, err)
	}
	return replyJson(ctx, balance)
}

// EchoReplyJson reply with json standart structure with filed response
func replyJson(ctx *echo.Context, v interface{}) error {
	return ctx.JSON(200, map[string]interface{}{"response": v})
}

// EchoReplyJsonError reply with json standart error structure
func replyJsonError(ctx *echo.Context, err interface{}) error {
	return ctx.JSON(400, map[string]interface{}{"error": fmt.Sprintf("%s", err)})
}

func echoJsonCheckErrorMW() echo.MiddlewareFunc {
	return func(h echo.HandlerFunc) echo.HandlerFunc {
		return func(c *echo.Context) error {
			err := h(c)
			if err != nil {
				return replyJsonError(c, err)
			}
			return nil
		}
	}
}
<commit_msg>remove core package<commit_after>package common

import (
	"log"

	"encoding/json"
	"fmt"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/miolini/bankgo/rpc/client"
)

type BalanceEntry struct {
	UserID int64
	Value  int64
}

type App struct {
	router *echo.Echo
	client *client.BalanceStorageClient
}

func (app *App) Init(rpcAddr string) error {
	log.Printf("httpapi init")
	client, err := client.Connect(rpcAddr)
	if err != nil {
		return err
	}
	app.client = client
	app.router = echo.New()
	app.router.Use(middleware.Logger(), middleware.Recover())
	app.router.Use(echoJsonCheckErrorMW())
	app.router.Get("/balances", app.handleGetBalances)
	app.router.Post("/transaction", app.handlePostTransaction)
	return nil
}

func (app *App) Run(addr string) {
	app.router.Run(addr)
}

func (app *App) handleGetBalances(ctx *echo.Context) error {
	records, err := app.client.AllBalances()
	if err != nil {
		return err
	}
	replyJson(ctx, records)
	return nil
}

func (app *App) handlePostTransaction(ctx *echo.Context) error {
	var balance BalanceEntry
	err := json.NewDecoder(ctx.Request().Body).Decode(&balance)
	if err != nil {
		return replyJsonError(ctx, fmt.Errorf("json parse error: %s", err))
	}
	balance.Value, err = app.client.SetValue(balance.UserID, balance.Value)
	if err != nil {
		return replyJsonError(ctx, err)
	}
	return replyJson(ctx, balance)
}

func replyJson(ctx *echo.Context, v interface{}) error {
	return ctx.JSON(200, map[string]interface{}{"response": v})
}

func replyJsonError(ctx *echo.Context, err interface{}) error {
	return ctx.JSON(400, map[string]interface{}{"error": fmt.Sprintf("%s", err)})
}

func echoJsonCheckErrorMW() echo.MiddlewareFunc {
	return func(h echo.HandlerFunc) echo.HandlerFunc {
		return func(c *echo.Context) error {
			err := h(c)
			if err != nil {
				return replyJsonError(c, err)
			}
			return nil
		}
	}
}
<|endoftext|>