<commit_before>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"encoding/binary"
	"io"
)

/*

       DONUT v1 Spec
   **********************
   BlockStart      [4]byte             // Magic="MINI"
   VersionMajor    uint16
   VersionMinor    uint16
   VersionPatch    uint16
   VersionReserved uint16
   Reserved        uint64
   GobHeaderLen    uint32
   GobHeader       io.Reader           // matches length
   BlockData       [4]byte             // Magic="DATA"
   Data            io.Reader           // matches length
   BlockLen        uint64              // length to block start
   BlockEnd        [4]byte             // Magic="INIM"

*/

type DonutStructure struct {
	BlockStart      [4]byte // Magic="MINI"
	VersionMajor    uint16
	VersionMinor    uint16
	VersionPatch    uint16
	VersionReserved uint16
	Reserved        uint64
	GobHeaderLen    uint32
	GobHeader       GobHeader
	BlockData       [4]byte
	Data            io.Reader
	BlockLen        uint64
	BlockEnd        [4]byte
}

type DonutFooter struct {
	BlockLen uint64
	BlockEnd uint32 // Magic="INIM"
}

type Donut struct {
	file io.Writer
	// mutex
}

type GobHeader struct{}

func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error {
	// TODO mutex
	// Create bytes buffer representing the new object
	donutStructure := DonutStructure{
		BlockStart:      [4]byte{'M', 'I', 'N', 'I'},
		VersionMajor:    1,
		VersionMinor:    0,
		VersionPatch:    0,
		VersionReserved: 0,
		Reserved:        0,
		GobHeaderLen:    0,
		GobHeader:       gobHeader,
		BlockData:       [4]byte{'D', 'A', 'T', 'A'},
		Data:            object,
		BlockLen:        0,
		BlockEnd:        [4]byte{'I', 'N', 'I', 'M'},
	}
	if err := donut.WriteStructure(donut.file, donutStructure); err != nil {
		return err
	}
	return nil
}

func (donut *Donut) WriteStructure(target io.Writer, donutStructure DonutStructure) error {
	err := binary.Write(target, binary.LittleEndian, donutStructure.BlockStart)
	err = binary.Write(target, binary.LittleEndian, donutStructure.VersionMajor)
	err = binary.Write(target, binary.LittleEndian, donutStructure.VersionMinor)
	err = binary.Write(target, binary.LittleEndian, donutStructure.VersionPatch)
	err = binary.Write(target, binary.LittleEndian, donutStructure.VersionReserved)
	err = binary.Write(target, binary.LittleEndian, donutStructure.Reserved)
	err = binary.Write(target, binary.LittleEndian, donutStructure.GobHeaderLen)
	err = binary.Write(target, binary.LittleEndian, donutStructure.GobHeader)
	err = binary.Write(target, binary.LittleEndian, donutStructure.BlockData)
	err = binary.Write(target, binary.LittleEndian, donutStructure.Data)
	err = binary.Write(target, binary.LittleEndian, donutStructure.BlockLen)
	err = binary.Write(target, binary.LittleEndian, donutStructure.BlockEnd)

	return err
}
<commit_msg>Magic as numbers are faster to encode and compare<commit_after>/*
 * Mini Object Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package donut

import (
	"encoding/binary"
	"io"
)

/*

       DONUT v1 Spec
   **********************
   BlockStart      [4]byte             // Magic="MINI"=1229867341
   VersionMajor    uint16
   VersionMinor    uint16
   VersionPatch    uint16
   VersionReserved uint16
   Reserved        uint64
   GobHeaderLen    uint32
   GobHeader       io.Reader           // matches length
   BlockData       [4]byte             // Magic="DATA"=1096040772
   Data            io.Reader           // matches length
   BlockLen        uint64              // length to block start
   BlockEnd        [4]byte             // Magic="INIM"=1229867341

*/

var (
	MagicMINI = binary.LittleEndian.Uint32([]byte{'M', 'I', 'N', 'I'})
	MagicDATA = binary.LittleEndian.Uint32([]byte{'D', 'A', 'T', 'A'})
	MagicINIM = binary.LittleEndian.Uint32([]byte{'I', 'N', 'I', 'M'})
)

type DonutFormat struct {
	BlockStart      uint32 // Magic="MINI"=1229867341
	VersionMajor    uint16
	VersionMinor    uint16
	VersionPatch    uint16
	VersionReserved uint16
	Reserved        uint64
	GobHeaderLen    uint32
	GobHeader       GobHeader
	BlockData       uint32 // Magic="DATA"=1096040772
	Data            io.Reader
	BlockLen        uint64
	BlockEnd        uint32
}

type DonutFooter struct {
	BlockLen uint64
	BlockEnd uint32 // Magic="INIM"=1229867341
}

type Donut struct {
	file io.Writer
	// mutex
}

type GobHeader struct{}

func (donut *Donut) Write(gobHeader GobHeader, object io.Reader) error {
	// TODO mutex
	// Create bytes buffer representing the new object
	donutFormat := DonutFormat{
		BlockStart:      MagicMINI,
		VersionMajor:    1,
		VersionMinor:    0,
		VersionPatch:    0,
		VersionReserved: 0,
		Reserved:        0,
		GobHeaderLen:    0,
		GobHeader:       gobHeader,
		BlockData:       MagicDATA,
		Data:            object,
		BlockLen:        0,
		BlockEnd:        MagicINIM,
	}
	if err := donut.WriteFormat(donut.file, donutFormat); err != nil {
		return err
	}
	return nil
}

func (donut *Donut) WriteFormat(target io.Writer, donutFormat DonutFormat) error {
	err := binary.Write(target, binary.LittleEndian, donutFormat.BlockStart)
	err = binary.Write(target, binary.LittleEndian, donutFormat.VersionMajor)
	err = binary.Write(target, binary.LittleEndian, donutFormat.VersionMinor)
	err = binary.Write(target, binary.LittleEndian, donutFormat.VersionPatch)
	err = binary.Write(target, binary.LittleEndian, donutFormat.VersionReserved)
	err = binary.Write(target, binary.LittleEndian, donutFormat.Reserved)
	err = binary.Write(target, binary.LittleEndian, donutFormat.GobHeaderLen)
	err = binary.Write(target, binary.LittleEndian, donutFormat.GobHeader)
	err = binary.Write(target, binary.LittleEndian, donutFormat.BlockData)
	err = binary.Write(target, binary.LittleEndian, donutFormat.Data)
	err = binary.Write(target, binary.LittleEndian, donutFormat.BlockLen)
	err = binary.Write(target, binary.LittleEndian, donutFormat.BlockEnd)

	return err
}
<|endoftext|>


<commit_before>package true_git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func SyncDevBranchWithStagedFiles(ctx context.Context, gitDir, workTreeCacheDir, commit string) (string, error) {
	var resCommit string

	if err := withWorkTreeCacheLock(ctx, workTreeCacheDir, func() error {
		var err error
		if gitDir, err = filepath.Abs(gitDir); err != nil {
			return fmt.Errorf("bad git dir %s: %s", gitDir, err)
		}

		if workTreeCacheDir, err = filepath.Abs(workTreeCacheDir); err != nil {
			return fmt.Errorf("bad work tree cache dir %s: %s", workTreeCacheDir, err)
		}

		if err := checkSubmoduleConstraint(); err != nil {
			return err
		}

		workTreeDir, err := prepareWorkTree(ctx, gitDir, workTreeCacheDir, commit, true)
		if err != nil {
			return fmt.Errorf("unable to prepare worktree for commit %v: %s", commit, err)
		}

		currentCommitPath := filepath.Join(workTreeCacheDir, "current_commit")
		if err := os.RemoveAll(currentCommitPath); err != nil {
			return fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
		}

		devBranchName := fmt.Sprintf("werf-dev-%s", commit)
		var isDevBranchExist bool
		if output, err := runGitCmd(ctx, []string{"branch", "--list", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
			return err
		} else {
			isDevBranchExist = output.Len() != 0
		}

		var devHeadCommit string
		if isDevBranchExist {
			if _, err := runGitCmd(ctx, []string{"checkout", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			if output, err := runGitCmd(ctx, []string{"rev-parse", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			} else {
				devHeadCommit = strings.TrimSpace(output.String())
			}
		} else {
			if _, err := runGitCmd(ctx, []string{"checkout", "-b", devBranchName, commit}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			devHeadCommit = commit
		}

		gitDiffArgs := []string{
			"-c", "diff.renames=false",
			"-c", "core.quotePath=false",
			"diff",
			"--full-index",
			"--binary",
			"--cached",
			devHeadCommit,
		}
		if diffOutput, err := runGitCmd(ctx, gitDiffArgs, gitDir, runGitCmdOptions{}); err != nil {
			return err
		} else if len(diffOutput.Bytes()) == 0 {
			resCommit = devHeadCommit
		} else {
			if _, err := runGitCmd(ctx, []string{"apply", "--binary", "--index"}, workTreeDir, runGitCmdOptions{stdin: diffOutput}); err != nil {
				return err
			}

			gitArgs := []string{"-c", "user.email=werf@werf.io", "-c", "user.name=werf", "commit", "-m", time.Now().String()}
			if _, err := runGitCmd(ctx, gitArgs, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			if output, err := runGitCmd(ctx, []string{"rev-parse", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			} else {
				newDevCommit := strings.TrimSpace(output.String())
				resCommit = newDevCommit
			}
		}

		if _, err := runGitCmd(ctx, []string{"checkout", "--force", "--detach", resCommit}, workTreeDir, runGitCmdOptions{}); err != nil {
			return err
		}

		if err := ioutil.WriteFile(currentCommitPath, []byte(resCommit+"\n"), 0644); err != nil {
			return fmt.Errorf("unable to write %s: %s", currentCommitPath, err)
		}

		return nil
	}); err != nil {
		return "", err
	}

	return resCommit, nil
}

type runGitCmdOptions struct {
	stdin io.Reader
}

func runGitCmd(ctx context.Context, args []string, dir string, opts runGitCmdOptions) (*bytes.Buffer, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = dir

	if opts.stdin != nil {
		cmd.Stdin = opts.stdin
	}

	output := setCommandRecordingLiveOutput(ctx, cmd)

	err := cmd.Run()

	cmdWithArgs := strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " ")
	if debug() {
		fmt.Printf("[DEBUG] %s\n%s\n", cmdWithArgs, output)
	}

	if err != nil {
		return nil, fmt.Errorf("git command %s failed: %s\n%s", cmdWithArgs, err, output)
	}

	return output, err
}

func debug() bool {
	return os.Getenv("WERF_DEBUG_TRUE_GIT") == "1"
}
<commit_msg>[true git] Fix SyncDevBranchWithStagedFiles handles submodules changes improperly<commit_after>package true_git

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

func SyncDevBranchWithStagedFiles(ctx context.Context, gitDir, workTreeCacheDir, commit string) (string, error) {
	var resCommit string

	if err := withWorkTreeCacheLock(ctx, workTreeCacheDir, func() error {
		var err error
		if gitDir, err = filepath.Abs(gitDir); err != nil {
			return fmt.Errorf("bad git dir %s: %s", gitDir, err)
		}

		if workTreeCacheDir, err = filepath.Abs(workTreeCacheDir); err != nil {
			return fmt.Errorf("bad work tree cache dir %s: %s", workTreeCacheDir, err)
		}

		if err := checkSubmoduleConstraint(); err != nil {
			return err
		}

		workTreeDir, err := prepareWorkTree(ctx, gitDir, workTreeCacheDir, commit, true)
		if err != nil {
			return fmt.Errorf("unable to prepare worktree for commit %v: %s", commit, err)
		}

		currentCommitPath := filepath.Join(workTreeCacheDir, "current_commit")
		if err := os.RemoveAll(currentCommitPath); err != nil {
			return fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
		}

		devBranchName := fmt.Sprintf("werf-dev-%s", commit)
		var isDevBranchExist bool
		if output, err := runGitCmd(ctx, []string{"branch", "--list", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
			return err
		} else {
			isDevBranchExist = output.Len() != 0
		}

		var devHeadCommit string
		if isDevBranchExist {
			if _, err := runGitCmd(ctx, []string{"checkout", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			if output, err := runGitCmd(ctx, []string{"rev-parse", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			} else {
				devHeadCommit = strings.TrimSpace(output.String())
			}
		} else {
			if _, err := runGitCmd(ctx, []string{"checkout", "-b", devBranchName, commit}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			devHeadCommit = commit
		}

		gitDiffArgs := []string{
			"-c", "diff.renames=false",
			"-c", "core.quotePath=false",
			"diff",
			"--full-index",
			"--binary",
			"--cached",
			devHeadCommit,
		}
		if diffOutput, err := runGitCmd(ctx, gitDiffArgs, gitDir, runGitCmdOptions{}); err != nil {
			return err
		} else if len(diffOutput.Bytes()) == 0 {
			resCommit = devHeadCommit
		} else {
			if _, err := runGitCmd(ctx, []string{"apply", "--binary", "--index"}, workTreeDir, runGitCmdOptions{stdin: diffOutput}); err != nil {
				return err
			}

			gitArgs := []string{"-c", "user.email=werf@werf.io", "-c", "user.name=werf", "commit", "-m", time.Now().String()}
			if _, err := runGitCmd(ctx, gitArgs, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			}

			if output, err := runGitCmd(ctx, []string{"rev-parse", devBranchName}, workTreeDir, runGitCmdOptions{}); err != nil {
				return err
			} else {
				newDevCommit := strings.TrimSpace(output.String())
				resCommit = newDevCommit
			}
		}

		if _, err := runGitCmd(ctx, []string{"checkout", "--force", "--detach", resCommit}, workTreeDir, runGitCmdOptions{}); err != nil {
			return err
		}

		return nil
	}); err != nil {
		return "", err
	}

	return resCommit, nil
}

type runGitCmdOptions struct {
	stdin io.Reader
}

func runGitCmd(ctx context.Context, args []string, dir string, opts runGitCmdOptions) (*bytes.Buffer, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = dir

	if opts.stdin != nil {
		cmd.Stdin = opts.stdin
	}

	output := setCommandRecordingLiveOutput(ctx, cmd)

	err := cmd.Run()

	cmdWithArgs := strings.Join(append([]string{cmd.Path}, cmd.Args[1:]...), " ")
	if debug() {
		fmt.Printf("[DEBUG] %s\n%s\n", cmdWithArgs, output)
	}

	if err != nil {
		return nil, fmt.Errorf("git command %s failed: %s\n%s", cmdWithArgs, err, output)
	}

	return output, err
}

func debug() bool {
	return os.Getenv("WERF_DEBUG_TRUE_GIT") == "1"
}
<|endoftext|>


<commit_before>package actor

import (
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
)

type XfceSetter int

func (XfceSetter) Set(filename string) error {
	path, err := filepath.Abs(filename)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"xfconf-query",
		"-c",
		"xfce4-desktop",
		"-l",
	)
	bs, err := cmd.Output()
	if err != nil {
		logrus.Errorf("cannot read xfce4 desktop settings: %s", string(bs))
	}

	for _, line := range strings.Split(string(bs), "\n") {
		logrus.Debugf("xfce-config output %v", line)
		if strings.HasSuffix(line, "last-image") {
			if err = setWithCommand(
				"xfconf-query",
				"-c",
				"xfce4-desktop",
				"-p",
				line,
				"-s",
				path,
			); err != nil {
				logrus.Errorf(
					"error during setting %s to %s: %s", line, path, err,
				)
				return err
			}
		} else if strings.HasSuffix(line, "image-style") {
			if err = setWithCommand(
				"xfconf-query",
				"-c",
				"xfce4-desktop",
				"-p",
				line,
				"-s",
				"5",
			); err != nil {
				logrus.Errorf("error during setting %s to 5: %s", line, err)
				return err
			}
		}
	}

	return nil
}

func init() {
	Setters.Register("xfce", XfceSetter(0))
}
<commit_msg>build xfcesetter on linux exclusively<commit_after>// +build linux

package actor

import (
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/Sirupsen/logrus"
)

type XfceSetter int

func (XfceSetter) Set(filename string) error {
	path, err := filepath.Abs(filename)
	if err != nil {
		return err
	}

	cmd := exec.Command(
		"xfconf-query",
		"-c",
		"xfce4-desktop",
		"-l",
	)
	bs, err := cmd.Output()
	if err != nil {
		logrus.Errorf("cannot read xfce4 desktop settings: %s", string(bs))
	}

	for _, line := range strings.Split(string(bs), "\n") {
		logrus.Debugf("xfce-config output %v", line)
		if strings.HasSuffix(line, "last-image") {
			if err = setWithCommand(
				"xfconf-query",
				"-c",
				"xfce4-desktop",
				"-p",
				line,
				"-s",
				path,
			); err != nil {
				logrus.Errorf(
					"error during setting %s to %s: %s", line, path, err,
				)
				return err
			}
		} else if strings.HasSuffix(line, "image-style") {
			if err = setWithCommand(
				"xfconf-query",
				"-c",
				"xfce4-desktop",
				"-p",
				line,
				"-s",
				"5",
			); err != nil {
				logrus.Errorf("error during setting %s to 5: %s", line, err)
				return err
			}
		}
	}

	return nil
}

func init() {
	Setters.Register("xfce", XfceSetter(0))
}
<|endoftext|>


<commit_before>package raw

import (
	"bytes"
	"errors"
	"log"
	"net"
	"os"
	"reflect"
	"text/template"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewRawAdapter, "raw")
}

// NewRawAdapter returns a configured raw.Adapter
func NewRawAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("bad transport: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}
	tmplStr := "{{.Data}}\n"
	if os.Getenv("RAW_FORMAT") != "" {
		tmplStr = os.Getenv("RAW_FORMAT")
	}
	tmpl, err := template.New("raw").Parse(tmplStr)
	if err != nil {
		return nil, err
	}
	return &Adapter{
		route: route,
		conn:  conn,
		tmpl:  tmpl,
	}, nil
}

// Adapter is a simple adapter that streams log output to a connection without any templating
type Adapter struct {
	conn  net.Conn
	route *router.Route
	tmpl  *template.Template
}

// Stream sends log data to a connection
func (a *Adapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		buf := new(bytes.Buffer)
		err := a.tmpl.Execute(buf, message)
		if err != nil {
			log.Println("raw:", err)
			return
		}
		//log.Println("debug:", buf.String())
		_, err = a.conn.Write(buf.Bytes())
		if err != nil {
			log.Println("raw:", err)
			if reflect.TypeOf(a.conn).String() != "*net.UDPConn" {
				return
			}
		}
	}
}
<commit_msg>toJSON on raw<commit_after>package raw

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"net"
	"os"
	"reflect"
	"text/template"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewRawAdapter, "raw")
}

var funcs = template.FuncMap{
	"toJSON": func(value interface{}) string {
		bytes, err := json.Marshal(value)
		if err != nil {
			log.Println("raw:", err)
			return "\"\""
		}
		return string(bytes)

	},
}

// NewRawAdapter returns a configured raw.Adapter
func NewRawAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("bad transport: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}
	tmplStr := "{{.Data}}\n"
	if os.Getenv("RAW_FORMAT") != "" {
		tmplStr = os.Getenv("RAW_FORMAT")
	}
	tmpl, err := template.New("raw").Funcs(funcs).Parse(tmplStr)
	if err != nil {
		return nil, err
	}
	return &Adapter{
		route: route,
		conn:  conn,
		tmpl:  tmpl,
	}, nil
}

// Adapter is a simple adapter that streams log output to a connection without any templating
type Adapter struct {
	conn  net.Conn
	route *router.Route
	tmpl  *template.Template
}

// Stream sends log data to a connection
func (a *Adapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		buf := new(bytes.Buffer)
		err := a.tmpl.Execute(buf, message)
		if err != nil {
			log.Println("raw:", err)
			return
		}
		//log.Println("debug:", buf.String())
		_, err = a.conn.Write(buf.Bytes())
		if err != nil {
			log.Println("raw:", err)
			if reflect.TypeOf(a.conn).String() != "*net.UDPConn" {
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package ParquetType

import (
	"fmt"
	"log"
	"reflect"
)

//base type
type BOOLEAN bool
type INT32 int32
type INT64 int64
type INT96 string // length=96
type FLOAT float32
type DOUBLE float64
type BYTE_ARRAY string
type FIXED_LEN_BYTE_ARRAY string

//logical type
type UTF8 string
type INT_8 int32
type INT_16 int32
type INT_32 int32
type INT_64 int64
type UINT_8 uint32
type UINT_16 uint32
type UINT_32 uint32
type UINT_64 uint64
type DATE int32
type TIME_MILLIS int32
type TIME_MICROS int64
type TIMESTAMP_MILLIS int64
type TIMESTAMP_MICROS int64
type INTERVAL string // length=12
type DECIMAL string

func ParquetTypeToGoType(value interface{}) interface{} {
	typeName := reflect.TypeOf(value).Name()
	switch typeName {
	case "BOOLEAN":
		return bool(value.(BOOLEAN))
	case "INT32":
		return int32(value.(INT32))
	case "INT64":
		return int64(value.(INT64))
	case "INT96":
		return string(value.(INT96))
	case "FLOAT":
		return float32(value.(FLOAT))
	case "DOUBLE":
		return float64(value.(DOUBLE))
	case "BYTE_ARRAY":
		return string(value.(BYTE_ARRAY))
	case "FIXED_LEN_BYTE_ARRAY":
		return string(value.(FIXED_LEN_BYTE_ARRAY))
	case "UTF8":
		return string(value.(UTF8))
	case "INT_8":
		return int32(value.(INT_8))
	case "INT_16":
		return int32(value.(INT_16))
	case "INT_32":
		return int32(value.(INT_32))
	case "INT_64":
		return int64(value.(INT_64))
	case "UINT_8":
		return uint32(value.(UINT_8))
	case "UINT_16":
		return uint32(value.(UINT_16))
	case "UINT_32":
		return uint32(value.(UINT_32))
	case "UINT_64":
		return uint64(value.(UINT_64))
	case "DATE":
		return int32(value.(DATE))
	case "TIME_MILLIS":
		return int32(value.(TIME_MILLIS))
	case "TIME_MICROS":
		return int64(value.(TIME_MICROS))
	case "TIMESTAMP_MILLIS":
		return int64(value.(TIMESTAMP_MILLIS))
	case "TIMESTAMP_MICROS":
		return int64(value.(TIMESTAMP_MICROS))
	case "INTERVAL":
		return string(value.(INTERVAL))
	case "DECIMAL":
		return string(value.(DECIMAL))
	}
	return nil

}

//Scan a string to parquet value
func StrToParquetType(s string, typeName string) interface{} {
	if typeName == "BOOLEAN" {
		var v BOOLEAN
		fmt.Sscanf(s, "%t", &v)
		return v
	} else if typeName == "INT32" {
		var v INT32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT64" {
		var v INT64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT96" {
		var v INT96
		v = INT96(s)
		return v
	} else if typeName == "FLOAT" {
		var v FLOAT
		fmt.Sscanf(s, "%f", &v)
		return v
	} else if typeName == "DOUBLE" {
		var v DOUBLE
		fmt.Sscanf(s, "%f", &v)
		return v
	} else if typeName == "BYTE_ARRAY" {
		var v BYTE_ARRAY
		v = BYTE_ARRAY(s)
		return v
	} else if typeName == "FIXED_LEN_BYTE_ARRAY" {
		var v FIXED_LEN_BYTE_ARRAY
		v = FIXED_LEN_BYTE_ARRAY(s)
		return v
	} else if typeName == "UTF8" {
		var v UTF8
		v = UTF8(s)
		return v
	} else if typeName == "INT_8" {
		var v INT_8
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_16" {
		var v INT_16
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_32" {
		var v INT_32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_64" {
		var v INT_64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_8" {
		var v UINT_8
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_16" {
		var v UINT_16
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_32" {
		var v UINT_32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_64" {
		var v UINT_64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "DATE" {
		var v DATE
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIME_MILLIS" {
		var v TIME_MILLIS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIME_MICROS" {
		var v TIME_MICROS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIMESTAMP_MILLIS" {
		var v TIMESTAMP_MILLIS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIMESTAMP_MICROS" {
		var v TIMESTAMP_MICROS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INTERVAL" {
		var v INTERVAL
		v = INTERVAL(s)
		return v
	} else if typeName == "DECIMAL" {
		var v DECIMAL
		v = DECIMAL(s)
		return v
	} else {
		log.Printf("Type Error: %v ", typeName)
		return nil
	}

}
<commit_msg>fix parquet type to go type nil bug<commit_after>package ParquetType

import (
	"fmt"
	"log"
	"reflect"
)

//base type
type BOOLEAN bool
type INT32 int32
type INT64 int64
type INT96 string // length=96
type FLOAT float32
type DOUBLE float64
type BYTE_ARRAY string
type FIXED_LEN_BYTE_ARRAY string

//logical type
type UTF8 string
type INT_8 int32
type INT_16 int32
type INT_32 int32
type INT_64 int64
type UINT_8 uint32
type UINT_16 uint32
type UINT_32 uint32
type UINT_64 uint64
type DATE int32
type TIME_MILLIS int32
type TIME_MICROS int64
type TIMESTAMP_MILLIS int64
type TIMESTAMP_MICROS int64
type INTERVAL string // length=12
type DECIMAL string

func ParquetTypeToGoType(value interface{}) interface{} {
	if value == nil {
		return nil
	}
	typeName := reflect.TypeOf(value).Name()
	switch typeName {
	case "BOOLEAN":
		return bool(value.(BOOLEAN))
	case "INT32":
		return int32(value.(INT32))
	case "INT64":
		return int64(value.(INT64))
	case "INT96":
		return string(value.(INT96))
	case "FLOAT":
		return float32(value.(FLOAT))
	case "DOUBLE":
		return float64(value.(DOUBLE))
	case "BYTE_ARRAY":
		return string(value.(BYTE_ARRAY))
	case "FIXED_LEN_BYTE_ARRAY":
		return string(value.(FIXED_LEN_BYTE_ARRAY))
	case "UTF8":
		return string(value.(UTF8))
	case "INT_8":
		return int32(value.(INT_8))
	case "INT_16":
		return int32(value.(INT_16))
	case "INT_32":
		return int32(value.(INT_32))
	case "INT_64":
		return int64(value.(INT_64))
	case "UINT_8":
		return uint32(value.(UINT_8))
	case "UINT_16":
		return uint32(value.(UINT_16))
	case "UINT_32":
		return uint32(value.(UINT_32))
	case "UINT_64":
		return uint64(value.(UINT_64))
	case "DATE":
		return int32(value.(DATE))
	case "TIME_MILLIS":
		return int32(value.(TIME_MILLIS))
	case "TIME_MICROS":
		return int64(value.(TIME_MICROS))
	case "TIMESTAMP_MILLIS":
		return int64(value.(TIMESTAMP_MILLIS))
	case "TIMESTAMP_MICROS":
		return int64(value.(TIMESTAMP_MICROS))
	case "INTERVAL":
		return string(value.(INTERVAL))
	case "DECIMAL":
		return string(value.(DECIMAL))
	}
	return nil

}

//Scan a string to parquet value
func StrToParquetType(s string, typeName string) interface{} {
	if typeName == "BOOLEAN" {
		var v BOOLEAN
		fmt.Sscanf(s, "%t", &v)
		return v
	} else if typeName == "INT32" {
		var v INT32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT64" {
		var v INT64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT96" {
		var v INT96
		v = INT96(s)
		return v
	} else if typeName == "FLOAT" {
		var v FLOAT
		fmt.Sscanf(s, "%f", &v)
		return v
	} else if typeName == "DOUBLE" {
		var v DOUBLE
		fmt.Sscanf(s, "%f", &v)
		return v
	} else if typeName == "BYTE_ARRAY" {
		var v BYTE_ARRAY
		v = BYTE_ARRAY(s)
		return v
	} else if typeName == "FIXED_LEN_BYTE_ARRAY" {
		var v FIXED_LEN_BYTE_ARRAY
		v = FIXED_LEN_BYTE_ARRAY(s)
		return v
	} else if typeName == "UTF8" {
		var v UTF8
		v = UTF8(s)
		return v
	} else if typeName == "INT_8" {
		var v INT_8
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_16" {
		var v INT_16
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_32" {
		var v INT_32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INT_64" {
		var v INT_64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_8" {
		var v UINT_8
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_16" {
		var v UINT_16
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_32" {
		var v UINT_32
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "UINT_64" {
		var v UINT_64
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "DATE" {
		var v DATE
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIME_MILLIS" {
		var v TIME_MILLIS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIME_MICROS" {
		var v TIME_MICROS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIMESTAMP_MILLIS" {
		var v TIMESTAMP_MILLIS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "TIMESTAMP_MICROS" {
		var v TIMESTAMP_MICROS
		fmt.Sscanf(s, "%d", &v)
		return v
	} else if typeName == "INTERVAL" {
		var v INTERVAL
		v = INTERVAL(s)
		return v
	} else if typeName == "DECIMAL" {
		var v DECIMAL
		v = DECIMAL(s)
		return v
	} else {
		log.Printf("Type Error: %v ", typeName)
		return nil
	}

}
<|endoftext|>


<commit_before>package cache

import (
	//"bytes"
	//"encoding/gob"
	"time"

	"github.com/garyburd/redigo/redis"
)

//Cache storage implementation using redis as key/value storage
type RedisCacheStorage struct {
	redisPool      	redis.Pool
	ttlReadTimeout 	int
	cacheArea      	string
	enableTTL	 	bool
	Serializer     	Serializer // usually SerializerGOB implementation
}

var _=SerializerGOB{} // this is the usual serializer used above!!

//recover all cacheregistries of keys
func (s RedisCacheStorage) GetValuesMap(cacheKeys ...string) (map[string]CacheRegistry, error) {

	ttlMapChan := make(chan map[string]int, 1)
	if (s.enableTTL) {
		go func() {
			defer func() {
				if r := recover(); r != nil {
					log.Critical("Error trying to get ttl for registries %v!", cacheKeys)

					//in case of error, retur an empty map
					ttlMapChan <- make(map[string]int, 0)
				}
			}()

			//put result on channel
			ttlMapChan <- s.GetTTLMap(cacheKeys)
		}()
	}

	mapCacheRegistry := make(map[string]CacheRegistry)

	if len(cacheKeys) <= 0 {
		log.Debug("Nenhuma chave informada para busca. len(arrKeys)=0!")
		return mapCacheRegistry, nil //empty map
	}

	conn := s.redisPool.Get()
	defer conn.Close()
	var err error = nil

	//log.Debug(cacheKeys)

	replyMget, err := conn.Do("MGET", (s.getKeys(cacheKeys))...)
	if err != nil || replyMget == nil {
		log.Error("Error trying to get values from cache %v", err)
		log.Error("Returning an empty registry!")

		return mapCacheRegistry, err // error trying to search cache keys
	}

	arrResults, isArray := replyMget.([]interface{}) //try to convert the returned value to array

	if !isArray {
		log.Error("Value returned by a MGET query is not array for keys %v! No error will be returned!", cacheKeys) //formal check
		return make(map[string]CacheRegistry), nil
	}

	for _, cacheRegistryNotBytes := range arrResults {
		if cacheRegistryNotBytes != nil {

/*
			cacheRegistryBytes, isByteArr := cacheRegistryNotBytes.(string)
			if(isByteArr){
				log.Error("error trying to deserialize! not a byte array")
				return mapCacheRegistry, errors.New("not byte array!")
			}
*/

			cacheRegistryBytes, errBytes := redis.Bytes(cacheRegistryNotBytes, err)
			if errBytes != nil || replyMget == nil {
				return mapCacheRegistry, errBytes
			}

			cacheRegistry := CacheRegistry{}

			interfaceResp, _, errUnm := s.Serializer.UnmarshalMsg(cacheRegistry,cacheRegistryBytes)
			if errUnm!=nil {
				log.Error("error trying to deserialize!",errUnm)
				return mapCacheRegistry, errUnm
			}

			cacheRegistry, isCR := interfaceResp.(CacheRegistry)
			if(!isCR){
				log.Error("error trying to deserialize! object is not a CacheRegistry object type!")
				return mapCacheRegistry, nil
			}

			if err != nil {
				log.Error("Warning!! Error trying to recover data from redis!", err)
			} else {
				if cacheRegistry.Payload == nil {
					log.Error("ATENCAO! NENHUM PAYLOAD FOI RETORNADO DO REDIS!")
				}
				//Everything is alright
				mapCacheRegistry[cacheRegistry.CacheKey] = cacheRegistry
			}
		}
	}

	if (s.enableTTL) {
		select {
		//wait for ttl channel
		case ttlMap := <-ttlMapChan:
			mapCacheRegistry = s.zipTTL(mapCacheRegistry, ttlMap)
		//in case of timeout, returt an empty map
		case <-time.After(time.Duration(s.ttlReadTimeout) * time.Millisecond):
			log.Warning("Retrieve TTL for cachekeys %v from redis timeout after %dms, continuing without it.", cacheKeys, s.ttlReadTimeout)
			mapCacheRegistry = s.zipTTL(mapCacheRegistry, make(map[string]int, 0))
		}
	}

	return mapCacheRegistry, nil // err=nil by default, if everything is alright
}

//Recover current ttl information about registry
func (s RedisCacheStorage) GetTTL(key string) (int, error) {
	oneItemMap := make(map[string]CacheRegistry, 1)

	oneItemMap[key] = CacheRegistry{key, "", -2 /*not found*/, true, ""}

	respMap, errTTL := s.GetActualTTL(oneItemMap)
	return respMap[key].Ttl, errTTL

}

//Recover current ttl information about registries
func (s RedisCacheStorage) zipTTL(mapCacheRegistry map[string]CacheRegistry, ttlMap map[string]int) map[string]CacheRegistry {
	//prepare a keyval pair array
	for key, cacheRegistry := range mapCacheRegistry {
		if ttl, hasTtl := ttlMap[key]; hasTtl {
			cacheRegistry.Ttl = ttl
		} else {
			cacheRegistry.Ttl = -1
		}
		mapCacheRegistry[key] = cacheRegistry
	}

	return mapCacheRegistry
}

//Recover current ttl information about registries
func (s RedisCacheStorage) GetActualTTL(mapCacheRegistry map[string]CacheRegistry) (map[string]CacheRegistry, error) {

	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for keyMap, cacheRegistry := range mapCacheRegistry {

		respTtl, err := conn.Do("ttl", s.getKey(keyMap))
		log.Debug("TTL %v that came from redis %v", keyMap, respTtl)

		if err != nil {
			log.Error("Error trying to retrieve ttl of key " + keyMap, err)
			cacheRegistry.Ttl = -2
			return mapCacheRegistry, err

		} else {
			intResp, _ := respTtl.(int64)
			cacheRegistry.Ttl = int(intResp)
		}

		mapCacheRegistry[keyMap] = setTTLToPayload(&cacheRegistry)
	}

	return mapCacheRegistry, nil
}

//Recover current ttl information about registries
func (s RedisCacheStorage) GetTTLMap(keys []string) map[string]int {

	ttlMap := make(map[string]int, len(keys))

	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for _, key := range keys {

		respTtl, err := conn.Do("ttl", s.getKey(key))
		log.Debug("TTL %v that came from redis %v", key, respTtl)

		if err != nil {
			log.Error("Error trying to retrieve ttl of key " + key, err)
			ttlMap[key] = -2

		} else {
			intResp, _ := respTtl.(int64)
			ttlMap[key] = int(intResp)
		}

	}

	return ttlMap
}

//transfer the ttl information from cacheRegistry to paylaod interface, if it is ExposeTTL
func setTTLToPayload(cacheRegistry *CacheRegistry) CacheRegistry {

	payload := cacheRegistry.Payload

	exposeTTL, hasTtl := payload.(ExposeTTL)

	if hasTtl {
		log.Debug("Transfering ttl from redis (%d seconds) registry to ttl attribute of object %s", cacheRegistry.Ttl, cacheRegistry.CacheKey)
		payload = exposeTTL.SetTtl(cacheRegistry.Ttl) // assure the same type, from set ttl
		cacheRegistry.Payload = payload
		log.Debug("Setting ttl to %v, ttl value %v", cacheRegistry.CacheKey, exposeTTL.GetTtl())
	} else {
		log.Debug("Payload doesn't ExposeTTL %v", cacheRegistry.CacheKey)
	}

	return *cacheRegistry
}

//save informed registries on redis
func (s RedisCacheStorage) SetValues(registries ...CacheRegistry) error {

	var cacheRegistry CacheRegistry
	var index int

	defer func(cacheRegistry *CacheRegistry) {
		if r := recover(); r != nil {
			log.Error("Error trying to save cacheRegistry! recover= %v", r)
		}
	}(&cacheRegistry)

	conn := s.redisPool.Get()
	defer conn.Close()

	keyValPairs := make([]interface{}, 2 * len(registries))

	//prepare a keyval pair array
	for index, cacheRegistry = range registries {

		if len(cacheRegistry.CacheKey) == 0 {
			log.Error("cache key vazio !!!")
			//panic(errors.New("cache key vazio"))
		}

		var bytes = []byte{}
		bytes, err := s.Serializer.MarshalMsg(cacheRegistry,bytes)
		if(err!=nil){
			return err
		}

		if len(bytes) == 0 {
			log.Error("Error trying to decode value for key %v", cacheRegistry.CacheKey)
		}

		keyValPairs[(index * 2)] = s.getKey(cacheRegistry.CacheKey)
		keyValPairs[(index * 2) + 1] = bytes

	}

	_, errDo := conn.Do("MSET", keyValPairs...)
	if errDo != nil {
		log.Error("Error trying to save registry! %v %v", s.getKey(cacheRegistry.CacheKey), errDo)
		return errDo
	} else {
		log.Debug("Updating cache reg key %v ", s.getKey(cacheRegistry.CacheKey))
	}

	errF := conn.Flush()
	if errF != nil {
		log.Error("Error trying to flush connection! %v", errF)
		return errF
	}
	s.SetExpireTTL(registries...)
	return nil
}

//set defined ttl to the cache registries
func (s RedisCacheStorage) SetExpireTTL(cacheRegistries ...CacheRegistry) {
	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for _, cacheRegistry := range cacheRegistries {
		if cacheRegistry.GetTTL() > 0 {
			//log.Debug("Setting ttl to key %s ", cacheRegistry.CacheKey)
			_, err := conn.Do("expire", s.getKey(cacheRegistry.CacheKey), cacheRegistry.GetTTL())
			if err != nil {
				log.Error("Error trying to save cache registry w! %v", err)
				return
			}

		} else {
			log.Debug("TTL for %s, ttl=%d will not be setted! ", s.getKey(cacheRegistry.CacheKey), cacheRegistry.GetTTL())
		}
	}

	err := conn.Flush()
	if err != nil {
		log.Error("Error trying to save cache registry z! %v", err)
		return
	}
}

//delete values from redis
func (s RedisCacheStorage) DeleteValues(cacheKeys ...string) error {

	c := s.redisPool.Get()
	defer func() {
		c.Close()
	}()

	//apply a prefix to cache area
	keys := s.getKeys(cacheKeys)

	reply, err := c.Do("DEL", keys...)
	if err != nil {
		log.Error("Erro ao tentar invalidar registro no cache!", err, reply)
		return err
	}

	return nil
}

//apply a prefix to cache area
func (s RedisCacheStorage) getKey(key string) string {
	var newKey string

	var serPredix = s.Serializer.GetPrefix()

	if len(s.cacheArea) > 0 {
		newKey = s.cacheArea + ":"+serPredix+":" + key
	} else {
		newKey = key
	}

	return newKey
}

//apply a prefix to cachearea
func (s RedisCacheStorage) getKeys(keys []string) []interface{} {

	newKeys := make([]interface{}, len(keys))

	for index, key := range keys {
		newKey := s.getKey(key)
		newKeys[index] = newKey
	}

	return newKeys
}

//instantiate a new cachestorage redis
func NewRedisCacheStorage(hostPort string, password string, maxIdle int, readTimeout int, ttlReadTimeout int, cacheArea string, serializer Serializer, enableTTL bool) RedisCacheStorage {

	redisCacheStorage := RedisCacheStorage{
		*newPoolRedis(hostPort, password, maxIdle, readTimeout),
		ttlReadTimeout,
		cacheArea,
		enableTTL,
		serializer,
	}

	return redisCacheStorage
}

//create a redis connection pool
func newPoolRedis(server, password string, maxIdle int, readTimeout int) *redis.Pool {

	return &redis.Pool{
		MaxIdle:     maxIdle,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {

			c, err := redis.Dial("tcp", server, redis.DialReadTimeout(time.Duration(readTimeout) * time.Millisecond))
			if err != nil {
				log.Error("Erro ao tentar se conectar ao redis! ", err)
				return nil, err
			}

			return c, err
		},

		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<commit_msg>optimizing prefix<commit_after>package cache

import (
	//"bytes"
	//"encoding/gob"
	"time"

	"github.com/garyburd/redigo/redis"
)

//Cache storage implementation using redis as key/value storage
type RedisCacheStorage struct {
	redisPool      	redis.Pool
	ttlReadTimeout 	int
	cacheArea      	string
	enableTTL	 	bool
	Serializer     	Serializer // usually SerializerGOB implementation
}

var _=SerializerGOB{} // this is the usual serializer used above!!

//recover all cacheregistries of keys
func (s RedisCacheStorage) GetValuesMap(cacheKeys ...string) (map[string]CacheRegistry, error) {

	ttlMapChan := make(chan map[string]int, 1)
	if (s.enableTTL) {
		go func() {
			defer func() {
				if r := recover(); r != nil {
					log.Critical("Error trying to get ttl for registries %v!", cacheKeys)

					//in case of error, retur an empty map
					ttlMapChan <- make(map[string]int, 0)
				}
			}()

			//put result on channel
			ttlMapChan <- s.GetTTLMap(cacheKeys)
		}()
	}

	mapCacheRegistry := make(map[string]CacheRegistry)

	if len(cacheKeys) <= 0 {
		log.Debug("Nenhuma chave informada para busca. len(arrKeys)=0!")
		return mapCacheRegistry, nil //empty map
	}

	conn := s.redisPool.Get()
	defer conn.Close()
	var err error = nil

	//log.Debug(cacheKeys)

	replyMget, err := conn.Do("MGET", (s.getKeys(cacheKeys))...)
	if err != nil || replyMget == nil {
		log.Error("Error trying to get values from cache %v", err)
		log.Error("Returning an empty registry!")

		return mapCacheRegistry, err // error trying to search cache keys
	}

	arrResults, isArray := replyMget.([]interface{}) //try to convert the returned value to array

	if !isArray {
		log.Error("Value returned by a MGET query is not array for keys %v! No error will be returned!", cacheKeys) //formal check
		return make(map[string]CacheRegistry), nil
	}

	for _, cacheRegistryNotBytes := range arrResults {
		if cacheRegistryNotBytes != nil {

/*
			cacheRegistryBytes, isByteArr := cacheRegistryNotBytes.(string)
			if(isByteArr){
				log.Error("error trying to deserialize! not a byte array")
				return mapCacheRegistry, errors.New("not byte array!")
			}
*/

			cacheRegistryBytes, errBytes := redis.Bytes(cacheRegistryNotBytes, err)
			if errBytes != nil || replyMget == nil {
				return mapCacheRegistry, errBytes
			}

			cacheRegistry := CacheRegistry{}

			interfaceResp, _, errUnm := s.Serializer.UnmarshalMsg(cacheRegistry,cacheRegistryBytes)
			if errUnm!=nil {
				log.Error("error trying to deserialize!",errUnm)
				return mapCacheRegistry, errUnm
			}

			cacheRegistry, isCR := interfaceResp.(CacheRegistry)
			if(!isCR){
				log.Error("error trying to deserialize! object is not a CacheRegistry object type!")
				return mapCacheRegistry, nil
			}

			if err != nil {
				log.Error("Warning!! Error trying to recover data from redis!", err)
			} else {
				if cacheRegistry.Payload == nil {
					log.Error("ATENCAO! NENHUM PAYLOAD FOI RETORNADO DO REDIS!")
				}
				//Everything is alright
				mapCacheRegistry[cacheRegistry.CacheKey] = cacheRegistry
			}
		}
	}

	if (s.enableTTL) {
		select {
		//wait for ttl channel
		case ttlMap := <-ttlMapChan:
			mapCacheRegistry = s.zipTTL(mapCacheRegistry, ttlMap)
		//in case of timeout, returt an empty map
		case <-time.After(time.Duration(s.ttlReadTimeout) * time.Millisecond):
			log.Warning("Retrieve TTL for cachekeys %v from redis timeout after %dms, continuing without it.", cacheKeys, s.ttlReadTimeout)
			mapCacheRegistry = s.zipTTL(mapCacheRegistry, make(map[string]int, 0))
		}
	}

	return mapCacheRegistry, nil // err=nil by default, if everything is alright
}

//Recover current ttl information about registry
func (s RedisCacheStorage) GetTTL(key string) (int, error) {
	oneItemMap := make(map[string]CacheRegistry, 1)

	oneItemMap[key] = CacheRegistry{key, "", -2 /*not found*/, true, ""}

	respMap, errTTL := s.GetActualTTL(oneItemMap)
	return respMap[key].Ttl, errTTL

}

//Recover current ttl information about registries
func (s RedisCacheStorage) zipTTL(mapCacheRegistry map[string]CacheRegistry, ttlMap map[string]int) map[string]CacheRegistry {
	//prepare a keyval pair array
	for key, cacheRegistry := range mapCacheRegistry {
		if ttl, hasTtl := ttlMap[key]; hasTtl {
			cacheRegistry.Ttl = ttl
		} else {
			cacheRegistry.Ttl = -1
		}
		mapCacheRegistry[key] = cacheRegistry
	}

	return mapCacheRegistry
}

//Recover current ttl information about registries
func (s RedisCacheStorage) GetActualTTL(mapCacheRegistry map[string]CacheRegistry) (map[string]CacheRegistry, error) {

	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for keyMap, cacheRegistry := range mapCacheRegistry {

		respTtl, err := conn.Do("ttl", s.getKey(keyMap))
		log.Debug("TTL %v that came from redis %v", keyMap, respTtl)

		if err != nil {
			log.Error("Error trying to retrieve ttl of key " + keyMap, err)
			cacheRegistry.Ttl = -2
			return mapCacheRegistry, err

		} else {
			intResp, _ := respTtl.(int64)
			cacheRegistry.Ttl = int(intResp)
		}

		mapCacheRegistry[keyMap] = setTTLToPayload(&cacheRegistry)
	}

	return mapCacheRegistry, nil
}

//Recover current ttl information about registries
func (s RedisCacheStorage) GetTTLMap(keys []string) map[string]int {

	ttlMap := make(map[string]int, len(keys))

	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for _, key := range keys {

		respTtl, err := conn.Do("ttl", s.getKey(key))
		log.Debug("TTL %v that came from redis %v", key, respTtl)

		if err != nil {
			log.Error("Error trying to retrieve ttl of key " + key, err)
			ttlMap[key] = -2

		} else {
			intResp, _ := respTtl.(int64)
			ttlMap[key] = int(intResp)
		}

	}

	return ttlMap
}

//transfer the ttl information from cacheRegistry to paylaod interface, if it is ExposeTTL
func setTTLToPayload(cacheRegistry *CacheRegistry) CacheRegistry {

	payload := cacheRegistry.Payload

	exposeTTL, hasTtl := payload.(ExposeTTL)

	if hasTtl {
		log.Debug("Transfering ttl from redis (%d seconds) registry to ttl attribute of object %s", cacheRegistry.Ttl, cacheRegistry.CacheKey)
		payload = exposeTTL.SetTtl(cacheRegistry.Ttl) // assure the same type, from set ttl
		cacheRegistry.Payload = payload
		log.Debug("Setting ttl to %v, ttl value %v", cacheRegistry.CacheKey, exposeTTL.GetTtl())
	} else {
		log.Debug("Payload doesn't ExposeTTL %v", cacheRegistry.CacheKey)
	}

	return *cacheRegistry
}

//save informed registries on redis
func (s RedisCacheStorage) SetValues(registries ...CacheRegistry) error {

	var cacheRegistry CacheRegistry
	var index int

	defer func(cacheRegistry *CacheRegistry) {
		if r := recover(); r != nil {
			log.Error("Error trying to save cacheRegistry! recover= %v", r)
		}
	}(&cacheRegistry)

	conn := s.redisPool.Get()
	defer conn.Close()

	keyValPairs := make([]interface{}, 2 * len(registries))

	//prepare a keyval pair array
	for index, cacheRegistry = range registries {

		if len(cacheRegistry.CacheKey) == 0 {
			log.Error("cache key vazio !!!")
			//panic(errors.New("cache key vazio"))
		}

		var bytes = []byte{}
		bytes, err := s.Serializer.MarshalMsg(cacheRegistry,bytes)
		if(err!=nil){
			return err
		}

		if len(bytes) == 0 {
			log.Error("Error trying to decode value for key %v", cacheRegistry.CacheKey)
		}

		keyValPairs[(index * 2)] = s.getKey(cacheRegistry.CacheKey)
		keyValPairs[(index * 2) + 1] = bytes

	}

	_, errDo := conn.Do("MSET", keyValPairs...)
	if errDo != nil {
		log.Error("Error trying to save registry! %v %v", s.getKey(cacheRegistry.CacheKey), errDo)
		return errDo
	} else {
		log.Debug("Updating cache reg key %v ", s.getKey(cacheRegistry.CacheKey))
	}

	errF := conn.Flush()
	if errF != nil {
		log.Error("Error trying to flush connection! %v", errF)
		return errF
	}
	s.SetExpireTTL(registries...)
	return nil
}

//set defined ttl to the cache registries
func (s RedisCacheStorage) SetExpireTTL(cacheRegistries ...CacheRegistry) {
	conn := s.redisPool.Get()
	defer conn.Close()

	//prepare a keyval pair array
	for _, cacheRegistry := range cacheRegistries {
		if cacheRegistry.GetTTL() > 0 {
			//log.Debug("Setting ttl to key %s ", cacheRegistry.CacheKey)
			_, err := conn.Do("expire", s.getKey(cacheRegistry.CacheKey), cacheRegistry.GetTTL())
			if err != nil {
				log.Error("Error trying to save cache registry w! %v", err)
				return
			}

		} else {
			log.Debug("TTL for %s, ttl=%d will not be setted! ", s.getKey(cacheRegistry.CacheKey), cacheRegistry.GetTTL())
		}
	}

	err := conn.Flush()
	if err != nil {
		log.Error("Error trying to save cache registry z! %v", err)
		return
	}
}

//delete values from redis
func (s RedisCacheStorage) DeleteValues(cacheKeys ...string) error {

	c := s.redisPool.Get()
	defer func() {
		c.Close()
	}()

	//apply a prefix to cache area
	keys := s.getKeys(cacheKeys)

	reply, err := c.Do("DEL", keys...)
	if err != nil {
		log.Error("Erro ao tentar invalidar registro no cache!", err, reply)
		return err
	}

	return nil
}

//apply a prefix to cache area
func (s RedisCacheStorage) getKey(key string) string {
	var newKey string

	var serPredix = s.Serializer.GetPrefix()

	if len(s.cacheArea) > 0 {
		newKey = s.cacheArea + serPredix + key
	} else {
		newKey = key
	}

	return newKey
}

//apply a prefix to cachearea
func (s RedisCacheStorage) getKeys(keys []string) []interface{} {

	newKeys := make([]interface{}, len(keys))

	for index, key := range keys {
		newKey := s.getKey(key)
		newKeys[index] = newKey
	}

	return newKeys
}

//instantiate a new cachestorage redis
func NewRedisCacheStorage(hostPort string, password string, maxIdle int, readTimeout int, ttlReadTimeout int, cacheArea string, serializer Serializer, enableTTL bool) RedisCacheStorage {

	redisCacheStorage := RedisCacheStorage{
		*newPoolRedis(hostPort, password, maxIdle, readTimeout),
		ttlReadTimeout,
		cacheArea,
		enableTTL,
		serializer,
	}

	return redisCacheStorage
}

//create a redis connection pool
func newPoolRedis(server, password string, maxIdle int, readTimeout int) *redis.Pool {

	return &redis.Pool{
		MaxIdle:     maxIdle,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {

			c, err := redis.Dial("tcp", server, redis.DialReadTimeout(time.Duration(readTimeout) * time.Millisecond))
			if err != nil {
				log.Error("Erro ao tentar se conectar ao redis! ", err)
				return nil, err
			}

			return c, err
		},

		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<|endoftext|>


<commit_before>//
// grump reader parses a country specific file and generate a config file of bodies
//
// for each cell of the country specifc file, this program generate bodies per cells according to
// the population count in the cell
//
// the arrangement of circle in each cell is taken from a outsite source (csq something) up to 200 circles
//
// usage grump-reader -country=xxx where xxx isthe 3 small letter ISO 3166 code for the country (for instance "fra")
//
package main

import "flag"
import "math"
// import "math/rand"
import "fmt"
import "os"
import "log"
import "bufio"
import "path/filepath"
import "github.com/thomaspeugeot/tkv/barnes-hut"
import "github.com/thomaspeugeot/tkv/quadtree"

// store country code
type country struct {
	Name string
	NCols, NRows, XllCorner, YllCorner int
}

// coordinates of arrangement of circle packing in a square
type circleCoord struct {
	x,y float64
}

var targetMaxBodies = 400000

var maxCirclePerCell = 500

// storage of circle arrangement per number of circle in the square
type arrangementsStore [][]circleCoord

//
// on the PC
//  go run grump-reader.go -tkvdata="C:\Users\peugeot\tkv-data"
func main() {

	// flag "country"
	countryPtr := flag.String("country","fra","iso 3166 country code")

	// get the directory containing tkv data through the flag "tkvdata"
	dirTKVDataPtr := flag.String("tkvdata","/Users/thomaspeugeot/the-mapping-data/%s_grumpv1_pcount_00_ascii_30/","directory containing input tkv data")

	var country country

	flag.Parse()
	fmt.Println( "country to parse", *countryPtr)
	country.Name = *countryPtr
	fmt.Println( "directory containing tkv data", *dirTKVDataPtr)
	dirTKVData := *dirTKVDataPtr

	// create the path to the agragate country count
	grumpFilePath := fmt.Sprintf( "%s/%s_grumpv1_pcount_00_ascii_30/%sup00ag.asc", dirTKVData, *countryPtr, *countryPtr )
	fmt.Println("relative path ", filepath.Clean( grumpFilePath))
	var grumpFile *os.File
	var err error
	grumpFile, err = os.Open( filepath.Clean( grumpFilePath))
	if err != nil {
		log.Fatal(err)
	}

	// parse the grump
	var word int
	scanner := bufio.NewScanner( grumpFile)
	scanner.Split(bufio.ScanWords)

	// scan 8 first lines
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NCols)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NRows)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.XllCorner)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.YllCorner)

	fmt.Println( country )

	// scan the reamining header
	for word < 4 {
		scanner.Scan()
		word++
		fmt.Println( fmt.Sprintf("item %d : %s", word, scanner.Text()))
	}
	rowLatWidth := 0.0083333333333
	colLngWidth := 0.0083333333333

	// prepare the count matrix
	countMatrix := make([]float64, country.NRows * country.NCols)

	popTotal := 0.0
	// scan the file and store result in countMatrix
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			scanner.Scan()
			// lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			var count float64
			fmt.Sscanf( scanner.Text(), "%f", &count)
			popTotal += count

			countMatrix[ (country.NRows-row-1)*country.NCols + col ] = count
		}
		fmt.Printf("\rrow %5d lat %2.3f total %f", row, lat, popTotal)
	}
	fmt.Println("")
	grumpFile.Close()

	// get the arrangement
	arrangements := make( arrangementsStore, maxCirclePerCell)
	for nbCircles := 1; nbCircles < maxCirclePerCell; nbCircles++ {

		fmt.Printf("\rgetting arrangement for %3d circles", nbCircles)

		arrangements[nbCircles] = make( []circleCoord, nbCircles)

		// open the reference file
		circlePackingFilePath := fmt.Sprintf( "%s/csq_coords/csq%d.txt", dirTKVData, nbCircles )
		var circlePackingFile *os.File
		var errCirclePackingFile error
		circlePackingFile, errCirclePackingFile = os.Open( filepath.Clean( circlePackingFilePath))
		if errCirclePackingFile != nil {
			log.Fatal(err)
		}

		// prepare scanner
		scannerCircle := bufio.NewScanner( circlePackingFile)
		scannerCircle.Split(bufio.ScanWords)

		// one line per circle
		for circle := 0; circle < nbCircles; circle++ {

			// scan the id of the circle
			scannerCircle.Scan();

			// scan X coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].x))
			// scan Y coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].y))
			// fmt.Printf("getting arrangement for %d circle %f %f\n", nbCircles, arrangements[nbCircles][circle].x, arrangements[nbCircles][circle].y)
		}
		circlePackingFile.Close()
	}

	// prepare the output density file
	var bodies []quadtree.Body
	bodiesInCellMax := 0

	cumulativePopTotal := 0.0
	bodiesNb :=0
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			// compute relative coordinate of the cell
			relX := (lng - float64(country.XllCorner)) / (float64(country.NCols) * colLngWidth)
			relY := (lat - float64(country.YllCorner)) / (float64(country.NRows) * rowLatWidth)

			// fetch count of the cell
			count := countMatrix[ row*country.NCols + col ]

			// how many bodies ? it is maxBodies *( count / country.PCount)
			bodiesInCell := int( math.Floor( float64( targetMaxBodies) * (count/popTotal)))
			if bodiesInCell > bodiesInCellMax { bodiesInCellMax = bodiesInCell}

			// initiate the bodies
			for i :=0; i<bodiesInCell; i++ {
				var body quadtree.Body
				// angle := float64(i) * 2.0 * math.Pi / float64(bodiesInCell)
				body.X = relX + (1.0/float64(country.NCols))*(0.5 + arrangements[bodiesInCell][i].x)
				body.Y = relY + (1.0/float64(country.NRows))*(0.5 + arrangements[bodiesInCell][i].y)
				body.M = count/float64(bodiesInCell)
				bodies = append( bodies,  body)
			}
			cumulativePopTotal += count
			bodiesNb += bodiesInCell
		}
	}

	// var quadtree quadtree.Quadtree
	// quadtree.Init( &bodies)
	fmt.Println("bodies in cell max ", bodiesInCellMax)
	fmt.Println("cumulative pop ", cumulativePopTotal)
	fmt.Println("nb of bodies ", bodiesNb)

	var run barnes_hut.Run
	run.Init( & bodies)

	run.CaptureConfigCountry( country.Name)
}<commit_msg>generate body with unique MASS instead of variable mass (to avoid comet's tail effect)<commit_after>//
// grump reader parses a country specific file and generate a config file of bodies
//
// for each cell of the country specifc file, this program generate bodies per cells according to
// the population count in the cell
//
// the arrangement of circle in each cell is taken from a outsite source (csq something) up to 200 circles
//
// usage grump-reader -country=xxx where xxx isthe 3 small letter ISO 3166 code for the country (for instance "fra")
//
package main

import "flag"
import "math"
// import "math/rand"
import "fmt"
import "os"
import "log"
import "bufio"
import "path/filepath"
import "github.com/thomaspeugeot/tkv/barnes-hut"
import "github.com/thomaspeugeot/tkv/quadtree"

// store country code
type country struct {
	Name string
	NCols, NRows, XllCorner, YllCorner int
}

// coordinates of arrangement of circle packing in a square
type circleCoord struct {
	x,y float64
}

var targetMaxBodies = 400000

var maxCirclePerCell = 500

// storage of circle arrangement per number of circle in the square
type arrangementsStore [][]circleCoord

//
// on the PC
//  go run grump-reader.go -tkvdata="C:\Users\peugeot\tkv-data"
func main() {

	// flag "country"
	countryPtr := flag.String("country","fra","iso 3166 country code")

	// get the directory containing tkv data through the flag "tkvdata"
	dirTKVDataPtr := flag.String("tkvdata","/Users/thomaspeugeot/the-mapping-data/%s_grumpv1_pcount_00_ascii_30/","directory containing input tkv data")

	var country country

	flag.Parse()
	fmt.Println( "country to parse", *countryPtr)
	country.Name = *countryPtr
	fmt.Println( "directory containing tkv data", *dirTKVDataPtr)
	dirTKVData := *dirTKVDataPtr

	// create the path to the agragate country count
	grumpFilePath := fmt.Sprintf( "%s/%s_grumpv1_pcount_00_ascii_30/%sup00ag.asc", dirTKVData, *countryPtr, *countryPtr )
	fmt.Println("relative path ", filepath.Clean( grumpFilePath))
	var grumpFile *os.File
	var err error
	grumpFile, err = os.Open( filepath.Clean( grumpFilePath))
	if err != nil {
		log.Fatal(err)
	}

	// parse the grump
	var word int
	scanner := bufio.NewScanner( grumpFile)
	scanner.Split(bufio.ScanWords)

	// scan 8 first lines
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NCols)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.NRows)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.XllCorner)
	scanner.Scan(); scanner.Scan()
	fmt.Sscanf( scanner.Text(), "%d", & country.YllCorner)

	fmt.Println( country )

	// scan the reamining header
	for word < 4 {
		scanner.Scan()
		word++
		fmt.Println( fmt.Sprintf("item %d : %s", word, scanner.Text()))
	}
	rowLatWidth := 0.0083333333333
	colLngWidth := 0.0083333333333

	// prepare the count matrix
	countMatrix := make([]float64, country.NRows * country.NCols)

	popTotal := 0.0
	// scan the file and store result in countMatrix
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			scanner.Scan()
			// lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			var count float64
			fmt.Sscanf( scanner.Text(), "%f", &count)
			popTotal += count

			countMatrix[ (country.NRows-row-1)*country.NCols + col ] = count
		}
		fmt.Printf("\rrow %5d lat %2.3f total %f", row, lat, popTotal)
	}
	fmt.Println("")
	grumpFile.Close()

	// get the arrangement
	arrangements := make( arrangementsStore, maxCirclePerCell)
	for nbCircles := 1; nbCircles < maxCirclePerCell; nbCircles++ {

		fmt.Printf("\rgetting arrangement for %3d circles", nbCircles)

		arrangements[nbCircles] = make( []circleCoord, nbCircles)

		// open the reference file
		circlePackingFilePath := fmt.Sprintf( "%s/csq_coords/csq%d.txt", dirTKVData, nbCircles )
		var circlePackingFile *os.File
		var errCirclePackingFile error
		circlePackingFile, errCirclePackingFile = os.Open( filepath.Clean( circlePackingFilePath))
		if errCirclePackingFile != nil {
			log.Fatal(err)
		}

		// prepare scanner
		scannerCircle := bufio.NewScanner( circlePackingFile)
		scannerCircle.Split(bufio.ScanWords)

		// one line per circle
		for circle := 0; circle < nbCircles; circle++ {

			// scan the id of the circle
			scannerCircle.Scan();

			// scan X coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].x))
			// scan Y coordinate
			scannerCircle.Scan()
			fmt.Sscanf( scannerCircle.Text(), "%f", & (arrangements[nbCircles][circle].y))
			// fmt.Printf("getting arrangement for %d circle %f %f\n", nbCircles, arrangements[nbCircles][circle].x, arrangements[nbCircles][circle].y)
		}
		circlePackingFile.Close()
	}

	// prepare the output density file
	var bodies []quadtree.Body
	bodiesInCellMax := 0

	cumulativePopTotal := 0.0
	bodiesNb :=0
	for row :=0; row < country.NRows; row++ {
		lat := float64( country.YllCorner) + (float64( country.NRows - row)*rowLatWidth)
		for col :=0; col < country.NCols ; col++ {
			lng := float64(country.XllCorner) + (float64(col)*colLngWidth)

			// compute relative coordinate of the cell
			relX := (lng - float64(country.XllCorner)) / (float64(country.NCols) * colLngWidth)
			relY := (lat - float64(country.YllCorner)) / (float64(country.NRows) * rowLatWidth)

			// fetch count of the cell
			count := countMatrix[ row*country.NCols + col ]

			// how many bodies ? it is maxBodies *( count / country.PCount)
			bodiesInCell := int( math.Floor( float64( targetMaxBodies) * (count/popTotal)))
			if bodiesInCell > bodiesInCellMax { bodiesInCellMax = bodiesInCell}

			// initiate the bodies
			for i :=0; i<bodiesInCell; i++ {
				var body quadtree.Body
				// angle := float64(i) * 2.0 * math.Pi / float64(bodiesInCell)
				body.X = relX + (1.0/float64(country.NCols))*(0.5 + arrangements[bodiesInCell][i].x)
				body.Y = relY + (1.0/float64(country.NRows))*(0.5 + arrangements[bodiesInCell][i].y)
				// body.M = count/float64(bodiesInCell)
				body.M = float64(targetMaxBodies)
				bodies = append( bodies,  body)
			}
			cumulativePopTotal += count
			bodiesNb += bodiesInCell
		}
	}

	// var quadtree quadtree.Quadtree
	// quadtree.Init( &bodies)
	fmt.Println("bodies in cell max ", bodiesInCellMax)
	fmt.Println("cumulative pop ", cumulativePopTotal)
	fmt.Println("nb of bodies ", bodiesNb)

	var run barnes_hut.Run
	run.Init( & bodies)

	run.CaptureConfigCountry( country.Name)
}<|endoftext|>


<commit_before>package main

import (
	"github.com/nsf/tulib"
)

//----------------------------------------------------------------------------
// view_tree
//----------------------------------------------------------------------------

type view_tree struct {
	// At the same time only one of these groups can be valid:
	// 1) 'left', 'right' and 'split'
	// 2) 'top', 'bottom' and 'split'
	// 3) 'leaf'
	parent     *view_tree
	left       *view_tree
	top        *view_tree
	right      *view_tree
	bottom     *view_tree
	leaf       *view
	split      float32
	tulib.Rect // updated with 'resize' call
}

func new_view_tree_leaf(parent *view_tree, v *view) *view_tree {
	return &view_tree{
		parent: parent,
		leaf:   v,
	}
}

func (v *view_tree) split_vertically() {
	top := v.leaf
	bottom := new_view(top.ctx, top.buf)
	*v = view_tree{
		parent: v.parent,
		top:    new_view_tree_leaf(v, top),
		bottom: new_view_tree_leaf(v, bottom),
		split:  0.5,
	}
}

func (v *view_tree) split_horizontally() {
	left := v.leaf
	right := new_view(left.ctx, left.buf)
	*v = view_tree{
		parent: v.parent,
		left:   new_view_tree_leaf(v, left),
		right:  new_view_tree_leaf(v, right),
		split:  0.5,
	}
}

func (v *view_tree) draw() {
	if v.leaf != nil {
		v.leaf.draw()
		return
	}

	if v.left != nil {
		v.left.draw()
		v.right.draw()
	} else {
		v.top.draw()
		v.bottom.draw()
	}
}

func (v *view_tree) resize(pos tulib.Rect) {
	v.Rect = pos
	if v.leaf != nil {
		v.leaf.resize(pos.Width, pos.Height)
		return
	}

	if v.left != nil {
		// horizontal split, use 'w'
		w := pos.Width
		if w > 0 {
			// reserve one line for splitter, if we have one line
			w--
		}
		lw := int(float32(w) * v.split)
		rw := w - lw
		v.left.resize(tulib.Rect{pos.X, pos.Y, lw, pos.Height})
		v.right.resize(tulib.Rect{pos.X + lw + 1, pos.Y, rw, pos.Height})
	} else {
		// vertical split, use 'h', no need to reserve one line for
		// splitter, because splitters are part of the buffer's output
		// (their status bars act like a splitter)
		h := pos.Height
		th := int(float32(h) * v.split)
		bh := h - th
		v.top.resize(tulib.Rect{pos.X, pos.Y, pos.Width, th})
		v.bottom.resize(tulib.Rect{pos.X, pos.Y + th, pos.Width, bh})
	}
}

func (v *view_tree) traverse(cb func(*view_tree)) {
	if v.leaf != nil {
		cb(v)
		return
	}

	if v.left != nil {
		v.left.traverse(cb)
		v.right.traverse(cb)
	} else if v.top != nil {
		v.top.traverse(cb)
		v.bottom.traverse(cb)
	}
}

func (v *view_tree) nearest_vsplit() *view_tree {
	v = v.parent
	for v != nil {
		if v.top != nil {
			return v
		}
		v = v.parent
	}
	return nil
}

func (v *view_tree) nearest_hsplit() *view_tree {
	v = v.parent
	for v != nil {
		if v.left != nil {
			return v
		}
		v = v.parent
	}
	return nil
}

func (v *view_tree) one_step() float32 {
	if v.top != nil {
		return 1.0 / float32(v.Height)
	} else if v.left != nil {
		return 1.0 / float32(v.Width-1)
	}
	return 0.0
}

func (v *view_tree) normalize_split() {
	var off int
	if v.top != nil {
		off = int(float32(v.Height) * v.split)
	} else {
		off = int(float32(v.Width-1) * v.split)
	}
	v.split = float32(off) * v.one_step()
}

func (v *view_tree) step_resize(n int) {
	if v.Width <= 1 || v.Height <= 0 {
		// avoid division by zero, result is really bad
		return
	}

	one := v.one_step()
	v.normalize_split()
	v.split += one*float32(n) + (one * 0.5)
	if v.split > 1.0 {
		v.split = 1.0
	}
	if v.split < 0.0 {
		v.split = 0.0
	}
	v.resize(v.Rect)
}

func (v *view_tree) reparent() {
	if v.left != nil {
		v.left.parent = v
		v.right.parent = v
	} else if v.top != nil {
		v.top.parent = v
		v.bottom.parent = v
	}
}

func (v *view_tree) sibling() *view_tree {
	p := v.parent
	if p == nil {
		return nil
	}
	switch {
	case v == p.left:
		return p.right
	case v == p.right:
		return p.left
	case v == p.top:
		return p.bottom
	case v == p.bottom:
		return p.top
	}
	panic("unreachable")
}

func (v *view_tree) first_leaf_node() *view_tree {
	if v.left != nil {
		return v.left.first_leaf_node()
	} else if v.top != nil {
		return v.top.first_leaf_node()
	} else if v.leaf != nil {
		return v
	}
	panic("unreachable")
}
<commit_msg>On view split, use the view_location from the view we're splitting.<commit_after>package main

import (
	"github.com/nsf/tulib"
)

//----------------------------------------------------------------------------
// view_tree
//----------------------------------------------------------------------------

type view_tree struct {
	// At the same time only one of these groups can be valid:
	// 1) 'left', 'right' and 'split'
	// 2) 'top', 'bottom' and 'split'
	// 3) 'leaf'
	parent     *view_tree
	left       *view_tree
	top        *view_tree
	right      *view_tree
	bottom     *view_tree
	leaf       *view
	split      float32
	tulib.Rect // updated with 'resize' call
}

func new_view_tree_leaf(parent *view_tree, v *view) *view_tree {
	return &view_tree{
		parent: parent,
		leaf:   v,
	}
}

func (v *view_tree) split_vertically() {
	top := v.leaf
	bottom := new_view(top.ctx, top.buf)
	bottom.view_location = top.view_location
	*v = view_tree{
		parent: v.parent,
		top:    new_view_tree_leaf(v, top),
		bottom: new_view_tree_leaf(v, bottom),
		split:  0.5,
	}
}

func (v *view_tree) split_horizontally() {
	left := v.leaf
	right := new_view(left.ctx, left.buf)
	right.view_location = left.view_location
	*v = view_tree{
		parent: v.parent,
		left:   new_view_tree_leaf(v, left),
		right:  new_view_tree_leaf(v, right),
		split:  0.5,
	}
}

func (v *view_tree) draw() {
	if v.leaf != nil {
		v.leaf.draw()
		return
	}

	if v.left != nil {
		v.left.draw()
		v.right.draw()
	} else {
		v.top.draw()
		v.bottom.draw()
	}
}

func (v *view_tree) resize(pos tulib.Rect) {
	v.Rect = pos
	if v.leaf != nil {
		v.leaf.resize(pos.Width, pos.Height)
		return
	}

	if v.left != nil {
		// horizontal split, use 'w'
		w := pos.Width
		if w > 0 {
			// reserve one line for splitter, if we have one line
			w--
		}
		lw := int(float32(w) * v.split)
		rw := w - lw
		v.left.resize(tulib.Rect{pos.X, pos.Y, lw, pos.Height})
		v.right.resize(tulib.Rect{pos.X + lw + 1, pos.Y, rw, pos.Height})
	} else {
		// vertical split, use 'h', no need to reserve one line for
		// splitter, because splitters are part of the buffer's output
		// (their status bars act like a splitter)
		h := pos.Height
		th := int(float32(h) * v.split)
		bh := h - th
		v.top.resize(tulib.Rect{pos.X, pos.Y, pos.Width, th})
		v.bottom.resize(tulib.Rect{pos.X, pos.Y + th, pos.Width, bh})
	}
}

func (v *view_tree) traverse(cb func(*view_tree)) {
	if v.leaf != nil {
		cb(v)
		return
	}

	if v.left != nil {
		v.left.traverse(cb)
		v.right.traverse(cb)
	} else if v.top != nil {
		v.top.traverse(cb)
		v.bottom.traverse(cb)
	}
}

func (v *view_tree) nearest_vsplit() *view_tree {
	v = v.parent
	for v != nil {
		if v.top != nil {
			return v
		}
		v = v.parent
	}
	return nil
}

func (v *view_tree) nearest_hsplit() *view_tree {
	v = v.parent
	for v != nil {
		if v.left != nil {
			return v
		}
		v = v.parent
	}
	return nil
}

func (v *view_tree) one_step() float32 {
	if v.top != nil {
		return 1.0 / float32(v.Height)
	} else if v.left != nil {
		return 1.0 / float32(v.Width-1)
	}
	return 0.0
}

func (v *view_tree) normalize_split() {
	var off int
	if v.top != nil {
		off = int(float32(v.Height) * v.split)
	} else {
		off = int(float32(v.Width-1) * v.split)
	}
	v.split = float32(off) * v.one_step()
}

func (v *view_tree) step_resize(n int) {
	if v.Width <= 1 || v.Height <= 0 {
		// avoid division by zero, result is really bad
		return
	}

	one := v.one_step()
	v.normalize_split()
	v.split += one*float32(n) + (one * 0.5)
	if v.split > 1.0 {
		v.split = 1.0
	}
	if v.split < 0.0 {
		v.split = 0.0
	}
	v.resize(v.Rect)
}

func (v *view_tree) reparent() {
	if v.left != nil {
		v.left.parent = v
		v.right.parent = v
	} else if v.top != nil {
		v.top.parent = v
		v.bottom.parent = v
	}
}

func (v *view_tree) sibling() *view_tree {
	p := v.parent
	if p == nil {
		return nil
	}
	switch {
	case v == p.left:
		return p.right
	case v == p.right:
		return p.left
	case v == p.top:
		return p.bottom
	case v == p.bottom:
		return p.top
	}
	panic("unreachable")
}

func (v *view_tree) first_leaf_node() *view_tree {
	if v.left != nil {
		return v.left.first_leaf_node()
	} else if v.top != nil {
		return v.top.first_leaf_node()
	} else if v.leaf != nil {
		return v
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>package utron

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

var baseApp *App

func init() {
	baseApp = NewApp()
}

// App is the main utron application
type App struct {
	router     *Router
	cfg        *Config
	view       View
	log        Logger
	model      *Model
	configPath string
	isInit     bool
}

// NewApp creates a new bare-bone utron application. To use MVC components, you should call
// Init method before serving requests.
func NewApp() *App {
	app := &App{}
	app.Set(logThis)
	r := NewRouter(app)
	app.Set(r)
	app.Set(NewModel())
	return app
}

// NewMVC creates a new MVC utron app, if cfg is passed, it should be a directory to look for
// configuration file. The App returned is initialized.
func NewMVC(cfg ...string) (*App, error) {
	app := NewApp()
	if len(cfg) > 0 {
		app.SetConfigPath(cfg[0])
	}
	if err := app.Init(); err != nil {
		return nil, err
	}
	return app, nil
}

// Init initializes MVC App
func (a *App) Init() error {
	if a.configPath == "" {
		a.SetConfigPath("config")
	}
	return a.init()
}

// SetConfigPath sets dir as a path to search for config files
func (a *App) SetConfigPath(dir string) {
	a.configPath = dir
}

// init initializes values to the app components.
func (a *App) init() error {
	appConfig, err := loadConfig(a.configPath)
	if err != nil {
		return err
	}

	views, err := NewSimpleView(appConfig.ViewsDir)
	if err != nil {
		return err
	}
	if a.model != nil && !a.model.IsOpen() {
		oerr := a.model.OpenWithConfig(appConfig)
		if oerr != nil {
			return oerr
		}
	} else {
		model, err := NewModelWithConfig(appConfig)
		if err != nil {
			return err
		}
		a.Set(model)
	}
	a.router.loadRoutes(a.configPath) // load routes file if any
	a.Set(appConfig)
	a.Set(views)
	a.isInit = true

	// Case the StaticDir is specified in the Config fille, register
	// a handler serving contents of the directory under the PathPrefix /static/
	if appConfig.StaticDir != "" {
		static, err := getAbsolutePath(appConfig.StaticDir)
		if err != nil {
			logThis.Errors(err)
		}
		if static != "" {
			a.router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(static))))
		}

	}
	return nil
}

//getAbsolutePath returns absolute path to dir, if dir is relative then we add current working directory.
// Checks are made to ensure the directory exist.In case of any error, and empty string is returned.
func getAbsolutePath(dir string) (string, error) {
	info, err := os.Stat(dir)
	if err != nil {
		return "", err
	}
	if !info.IsDir() {
		return "", fmt.Errorf("untron: %s is not a directory", dir)
	}

	if filepath.IsAbs(dir) { // dir is already absolute, return it
		return dir, nil
	}
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	absDir := filepath.Join(wd, dir)
	_, err = os.Stat(absDir)
	if err != nil {
		return "", err
	}
	return absDir, nil
}

// loadConfig loads configuration file, if cfg is provided then it is used as the directory
// for searching configuration file else defaults to directory named config in the current
// working directory.
func loadConfig(cfg ...string) (*Config, error) {
	cfgDir := "config"
	if len(cfg) > 0 {
		cfgDir = cfg[0]
	}

	// load configurations
	cfgFile, err := findConfigFile(cfgDir, "app")
	if err != nil {
		return nil, err
	}
	return NewConfig(cfgFile)
}

//findConfigFile finds the configuration file name, in the directory dir
func findConfigFile(dir string, name string) (file string, err error) {
	extensions := []string{".json", ".toml", ".yml"}

	for _, ext := range extensions {
		file = filepath.Join(dir, name)
		if info, serr := os.Stat(file); serr == nil && !info.IsDir() {
			return
		}
		file = file + ext
		if info, serr := os.Stat(file); serr == nil && !info.IsDir() {
			return
		}
	}
	return "", fmt.Errorf("utron: can't find configuration file %s in %s", name, dir)
}

// AddController registers ctrl, and middlewares if provided
func (a *App) AddController(ctrl Controller, middlewares ...interface{}) {
	a.router.Add(ctrl, middlewares...)
}

// Set assigns value to *App components. The following can be set
//	Logger by passing Logger
//	View by passing View
//	Router by passing *Router
//	Config by passing *Config
//	Model by passing *Model
func (a *App) Set(value interface{}) {
	switch value.(type) {
	case Logger:
		a.log = value.(Logger)
	case *Router:
		a.router = value.(*Router)
	case View:
		a.view = value.(View)
	case *Config:
		a.cfg = value.(*Config)
	case *Model:
		a.model = value.(*Model)
	}
}

// ServeHTTP serves http, it can be used with other http.Handler implementations
func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	a.router.ServeHTTP(w, r)
}

// SetConfigPath sets the path to look for configurations files in the
// global utron App.
func SetConfigPath(path string) {
	baseApp.SetConfigPath(path)
}

// RegisterModels registers models in the global utron App.
func RegisterModels(models ...interface{}) {
	baseApp.model.Register(models...)
}

// RegisterController register ctrl in the global utron App.
func RegisterController(ctrl Controller, middlewares ...interface{}) {
	baseApp.router.Add(ctrl, middlewares...)
}

// ServeHTTP serves request using global utron App
func ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !baseApp.isInit {
		if err := baseApp.Init(); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	baseApp.ServeHTTP(w, r)
}

// Migrate runs migrations on the global utron app.
func Migrate() {
	baseApp.model.AutoMigrateAll()
}

// Run runs a http server, serving the global utron App.
//
// By using this, you should make sure you followed MVC pattern,
func Run() {
	if err := baseApp.Init(); err != nil {
		logThis.Errors(err)
		os.Exit(1)
	}
	Migrate()
	port := baseApp.cfg.Port
	logThis.Info("starting server at ", baseApp.cfg.BaseURL)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), baseApp))
}
<commit_msg>Update utron.go - Fix docs<commit_after>package utron

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

var baseApp *App

func init() {
	baseApp = NewApp()
}

// App is the main utron application.
type App struct {
	router     *Router
	cfg        *Config
	view       View
	log        Logger
	model      *Model
	configPath string
	isInit     bool
}

// NewApp creates a new bare-bone utron application. To use the MVC components, you should call
// the Init method before serving requests.
func NewApp() *App {
	app := &App{}
	app.Set(logThis)
	r := NewRouter(app)
	app.Set(r)
	app.Set(NewModel())
	return app
}

// NewMVC creates a new MVC utron app. If cfg is passed, it should be a directory to look for
// the configuration files. The App returned is initialized.
func NewMVC(cfg ...string) (*App, error) {
	app := NewApp()
	if len(cfg) > 0 {
		app.SetConfigPath(cfg[0])
	}
	if err := app.Init(); err != nil {
		return nil, err
	}
	return app, nil
}

// Init initializes the MVC App.
func (a *App) Init() error {
	if a.configPath == "" {
		a.SetConfigPath("config")
	}
	return a.init()
}

// SetConfigPath sets the directory path to search for the config files.
func (a *App) SetConfigPath(dir string) {
	a.configPath = dir
}

// init initializes values to the app components.
func (a *App) init() error {
	appConfig, err := loadConfig(a.configPath)
	if err != nil {
		return err
	}

	views, err := NewSimpleView(appConfig.ViewsDir)
	if err != nil {
		return err
	}
	if a.model != nil && !a.model.IsOpen() {
		oerr := a.model.OpenWithConfig(appConfig)
		if oerr != nil {
			return oerr
		}
	} else {
		model, err := NewModelWithConfig(appConfig)
		if err != nil {
			return err
		}
		a.Set(model)
	}
	a.router.loadRoutes(a.configPath) // Load a routes file if available.
	a.Set(appConfig)
	a.Set(views)
	a.isInit = true

	// In case the StaticDir is specified in the Config file, register
	// a handler serving contents of that directory under the PathPrefix /static/.
	if appConfig.StaticDir != "" {
		static, err := getAbsolutePath(appConfig.StaticDir)
		if err != nil {
			logThis.Errors(err)
		}
		if static != "" {
			a.router.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(static))))
		}

	}
	return nil
}

// getAbsolutePath returns the absolute path to dir. If the dir is relative, then we add
// the current working directory. Checks are made to ensure the directory exist.
// In case of any error, an empty string is returned.
func getAbsolutePath(dir string) (string, error) {
	info, err := os.Stat(dir)
	if err != nil {
		return "", err
	}
	if !info.IsDir() {
		return "", fmt.Errorf("untron: %s is not a directory", dir)
	}

	if filepath.IsAbs(dir) { // If dir is already absolute, return it.
		return dir, nil
	}
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	absDir := filepath.Join(wd, dir)
	_, err = os.Stat(absDir)
	if err != nil {
		return "", err
	}
	return absDir, nil
}

// loadConfig loads the configuration file. If cfg is provided, then it is used as the directory
// for searching the configuration files. It defaults to the directory named config in the current
// working directory.
func loadConfig(cfg ...string) (*Config, error) {
	cfgDir := "config"
	if len(cfg) > 0 {
		cfgDir = cfg[0]
	}

	// Load configurations.
	cfgFile, err := findConfigFile(cfgDir, "app")
	if err != nil {
		return nil, err
	}
	return NewConfig(cfgFile)
}

// findConfigFile finds the configuration file name in the directory specified.
func findConfigFile(dir string, name string) (file string, err error) {
	extensions := []string{".json", ".toml", ".yml"}

	for _, ext := range extensions {
		file = filepath.Join(dir, name)
		if info, serr := os.Stat(file); serr == nil && !info.IsDir() {
			return
		}
		file = file + ext
		if info, serr := os.Stat(file); serr == nil && !info.IsDir() {
			return
		}
	}
	return "", fmt.Errorf("utron: can't find configuration file %s in %s", name, dir)
}

// AddController registers a controller, and middlewares if any is provided.
func (a *App) AddController(ctrl Controller, middlewares ...interface{}) {
	a.router.Add(ctrl, middlewares...)
}

// Set is for assigning a value to *App components. The following can be set:
//	Logger by passing Logger
//	View by passing View
//	Router by passing *Router
//	Config by passing *Config
//	Model by passing *Model
func (a *App) Set(value interface{}) {
	switch value.(type) {
	case Logger:
		a.log = value.(Logger)
	case *Router:
		a.router = value.(*Router)
	case View:
		a.view = value.(View)
	case *Config:
		a.cfg = value.(*Config)
	case *Model:
		a.model = value.(*Model)
	}
}

// ServeHTTP serves http requests. It can be used with other http.Handler implementations.
func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	a.router.ServeHTTP(w, r)
}

// SetConfigPath sets the path to look for the configuration file in the
// global utron App.
func SetConfigPath(path string) {
	baseApp.SetConfigPath(path)
}

// RegisterModels registers models in the global utron App.
func RegisterModels(models ...interface{}) {
	baseApp.model.Register(models...)
}

// RegisterController registers a controller in the global utron App.
func RegisterController(ctrl Controller, middlewares ...interface{}) {
	baseApp.router.Add(ctrl, middlewares...)
}

// ServeHTTP serves request using global utron App.
func ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !baseApp.isInit {
		if err := baseApp.Init(); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	baseApp.ServeHTTP(w, r)
}

// Migrate runs migrations on the global utron app.
func Migrate() {
	baseApp.model.AutoMigrateAll()
}

// Run runs a http server, serving the global utron App.
//
// By using this, you should make sure you followed the MVC pattern.
func Run() {
	if err := baseApp.Init(); err != nil {
		logThis.Errors(err)
		os.Exit(1)
	}
	Migrate()
	port := baseApp.cfg.Port
	logThis.Info("starting server at ", baseApp.cfg.BaseURL)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), baseApp))
}
<|endoftext|>


<commit_before>package commands

import "testing"

type TestOutput struct {
  Foo, Bar string
  Baz int
}

func TestMarshalling(t *testing.T) {
  req := NewRequest()

  res := Response{
    req: req,
    Value: TestOutput{ "beep", "boop", 1337 },
  }

  _, err := res.Marshal()
  if err == nil {
    t.Error("Should have failed (no encoding type specified in request)")
  }

  req.SetOption(globalOptions[0], Json)
  bytes, err := res.Marshal()
  if err != nil {
    t.Error("Should have passed")
  }
  output := string(bytes)
  if output != "{\"Foo\":\"beep\",\"Bar\":\"boop\",\"Baz\":1337}" {
    t.Error("Incorrect JSON output")
  }
}
<commit_msg>commands: Added test for Response error marshalling<commit_after>package commands

import (
  "testing"
  "fmt"
)

type TestOutput struct {
  Foo, Bar string
  Baz int
}

func TestMarshalling(t *testing.T) {
  req := NewRequest()

  res := Response{
    req: req,
    Value: TestOutput{ "beep", "boop", 1337 },
  }

  _, err := res.Marshal()
  if err == nil {
    t.Error("Should have failed (no encoding type specified in request)")
  }

  req.SetOption(globalOptions[0], Json)
  bytes, err := res.Marshal()
  if err != nil {
    t.Error("Should have passed")
  }
  output := string(bytes)
  if output != "{\"Foo\":\"beep\",\"Bar\":\"boop\",\"Baz\":1337}" {
    t.Error("Incorrect JSON output")
  }

  res.SetError(fmt.Errorf("You broke something!"), Client)
  bytes, err = res.Marshal()
  if err != nil {
    t.Error("Should have passed")
  }
  output = string(bytes)
  if output != "{\"Message\":\"You broke something!\",\"Code\":1}" {
    t.Error("Incorrect JSON output")
  }
}
<|endoftext|>


<commit_before>package agent

import (
	"net"
	"net/http"
	"net/url"
	"runtime"
	"time"

	"github.com/buildkite/agent/api"
)

var debug = false

type APIClient struct {
	Endpoint string
	Token    string
}

func APIClientEnableHTTPDebug() {
	debug = true
}

func (a APIClient) Create() *api.Client {
	// Create the transport used when making the Buildkite Agent API calls
	transport := &api.AuthenticatedTransport{
		Token: a.Token,
		Transport: &http.Transport{
			Proxy:              http.ProxyFromEnvironment,
			DisableKeepAlives:  false,
			DisableCompression: false,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 30 * time.Second,
		},
	}

	// From the transport, create the a http client
	httpClient := transport.Client()
	httpClient.Timeout = 10 * time.Second

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(a.Endpoint)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) UserAgent() string {
	return "buildkite-agent/" + Version() + "." + BuildVersion() + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
}
<commit_msg>Increase HTTP response timeout to 60 seconds<commit_after>package agent

import (
	"net"
	"net/http"
	"net/url"
	"runtime"
	"time"

	"github.com/buildkite/agent/api"
)

var debug = false

type APIClient struct {
	Endpoint string
	Token    string
}

func APIClientEnableHTTPDebug() {
	debug = true
}

func (a APIClient) Create() *api.Client {
	// Create the transport used when making the Buildkite Agent API calls
	transport := &api.AuthenticatedTransport{
		Token: a.Token,
		Transport: &http.Transport{
			Proxy:              http.ProxyFromEnvironment,
			DisableKeepAlives:  false,
			DisableCompression: false,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 30 * time.Second,
		},
	}

	// From the transport, create the a http client
	httpClient := transport.Client()
	httpClient.Timeout = 60 * time.Second

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(a.Endpoint)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) UserAgent() string {
	return "buildkite-agent/" + Version() + "." + BuildVersion() + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
}
<|endoftext|>


<commit_before>package buf_test

import (
	"bytes"
	"crypto/rand"
	"testing"

	"github.com/google/go-cmp/cmp"
	"v2ray.com/core/common"
	. "v2ray.com/core/common/buf"
)

func TestBufferClear(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	payload := "Bytes"
	buffer.Write([]byte(payload))
	if diff := cmp.Diff(buffer.Bytes(), payload); diff != "" {
		t.Error(diff)
	}

	buffer.Clear()
	if buffer.Len() != 0 {
		t.Error("expect 0 lenght, but got ", buffer.Len())
	}
}

func TestBufferIsEmpty(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	if buffer.IsEmpty() != true {
		t.Error("expect empty buffer, but not")
	}
}

func TestBufferString(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	const payload = "Test String"
	common.Must2(buffer.WriteString(payload))
	if buffer.String() != payload {
		t.Error("expect buffer content as ", payload, " but actually ", buffer.String())
	}
}

func TestBufferSlice(t *testing.T) {
	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesFrom(-2)
		if diff := cmp.Diff(bytes, []byte{'c', 'd'}); diff != "" {
			t.Error(diff)
		}
	}

	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesTo(-2)
		if diff := cmp.Diff(bytes, []byte{'a', 'b'}); diff != "" {
			t.Error(diff)
		}
	}

	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesRange(-3, -1)
		if diff := cmp.Diff(bytes, []byte{'b', 'c'}); diff != "" {
			t.Error(diff)
		}
	}
}

func TestBufferReadFullFrom(t *testing.T) {
	payload := make([]byte, 1024)
	common.Must2(rand.Read(payload))

	reader := bytes.NewReader(payload)
	b := New()
	n, err := b.ReadFullFrom(reader, 1024)
	common.Must(err)
	if n != 1024 {
		t.Error("expect reading 1024 bytes, but actually ", n)
	}

	if diff := cmp.Diff(payload, b.Bytes()); diff != "" {
		t.Error(diff)
	}
}

func BenchmarkNewBuffer(b *testing.B) {
	for i := 0; i < b.N; i++ {
		buffer := New()
		buffer.Release()
	}
}

func BenchmarkWrite2(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write([]byte{'a', 'b'})
		buffer.Clear()
	}
}

func BenchmarkWrite8(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write([]byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'})
		buffer.Clear()
	}
}

func BenchmarkWrite32(b *testing.B) {
	buffer := New()
	payload := make([]byte, 32)
	rand.Read(payload)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write(payload)
		buffer.Clear()
	}
}

func BenchmarkWriteByte2(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = buffer.WriteByte('a')
		_ = buffer.WriteByte('b')
		buffer.Clear()
	}
}

func BenchmarkWriteByte8(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = buffer.WriteByte('a')
		_ = buffer.WriteByte('b')
		_ = buffer.WriteByte('c')
		_ = buffer.WriteByte('d')
		_ = buffer.WriteByte('e')
		_ = buffer.WriteByte('f')
		_ = buffer.WriteByte('g')
		_ = buffer.WriteByte('h')
		buffer.Clear()
	}
}
<commit_msg>fix test break<commit_after>package buf_test

import (
	"bytes"
	"crypto/rand"
	"testing"

	"github.com/google/go-cmp/cmp"
	"v2ray.com/core/common"
	. "v2ray.com/core/common/buf"
)

func TestBufferClear(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	payload := "Bytes"
	buffer.Write([]byte(payload))
	if diff := cmp.Diff(buffer.Bytes(), []byte(payload)); diff != "" {
		t.Error(diff)
	}

	buffer.Clear()
	if buffer.Len() != 0 {
		t.Error("expect 0 lenght, but got ", buffer.Len())
	}
}

func TestBufferIsEmpty(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	if buffer.IsEmpty() != true {
		t.Error("expect empty buffer, but not")
	}
}

func TestBufferString(t *testing.T) {
	buffer := New()
	defer buffer.Release()

	const payload = "Test String"
	common.Must2(buffer.WriteString(payload))
	if buffer.String() != payload {
		t.Error("expect buffer content as ", payload, " but actually ", buffer.String())
	}
}

func TestBufferSlice(t *testing.T) {
	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesFrom(-2)
		if diff := cmp.Diff(bytes, []byte{'c', 'd'}); diff != "" {
			t.Error(diff)
		}
	}

	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesTo(-2)
		if diff := cmp.Diff(bytes, []byte{'a', 'b'}); diff != "" {
			t.Error(diff)
		}
	}

	{
		b := New()
		common.Must2(b.Write([]byte("abcd")))
		bytes := b.BytesRange(-3, -1)
		if diff := cmp.Diff(bytes, []byte{'b', 'c'}); diff != "" {
			t.Error(diff)
		}
	}
}

func TestBufferReadFullFrom(t *testing.T) {
	payload := make([]byte, 1024)
	common.Must2(rand.Read(payload))

	reader := bytes.NewReader(payload)
	b := New()
	n, err := b.ReadFullFrom(reader, 1024)
	common.Must(err)
	if n != 1024 {
		t.Error("expect reading 1024 bytes, but actually ", n)
	}

	if diff := cmp.Diff(payload, b.Bytes()); diff != "" {
		t.Error(diff)
	}
}

func BenchmarkNewBuffer(b *testing.B) {
	for i := 0; i < b.N; i++ {
		buffer := New()
		buffer.Release()
	}
}

func BenchmarkWrite2(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write([]byte{'a', 'b'})
		buffer.Clear()
	}
}

func BenchmarkWrite8(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write([]byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'})
		buffer.Clear()
	}
}

func BenchmarkWrite32(b *testing.B) {
	buffer := New()
	payload := make([]byte, 32)
	rand.Read(payload)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = buffer.Write(payload)
		buffer.Clear()
	}
}

func BenchmarkWriteByte2(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = buffer.WriteByte('a')
		_ = buffer.WriteByte('b')
		buffer.Clear()
	}
}

func BenchmarkWriteByte8(b *testing.B) {
	buffer := New()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_ = buffer.WriteByte('a')
		_ = buffer.WriteByte('b')
		_ = buffer.WriteByte('c')
		_ = buffer.WriteByte('d')
		_ = buffer.WriteByte('e')
		_ = buffer.WriteByte('f')
		_ = buffer.WriteByte('g')
		_ = buffer.WriteByte('h')
		buffer.Clear()
	}
}
<|endoftext|>


<commit_before>package agent

import (
	"testing"
)

// Assert that an existing conflict is triggered against the potential job name
func TestHasConflictExistingMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, name := state.HasConflict("b", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}
}

// Assert that a potential conflict is triggered against the existing job name
func TestHasConflictPotentialMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{})

	matched, name := state.HasConflict("b", []string{"a"})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}
}

// Assert that a existing jobs and potential jobs that do not conflict do not
// trigger a match
func TestHasConflictNoMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, _ := state.HasConflict("c", []string{"d"})
	if matched {
		t.Errorf("Expected no match")
	}
}

// Assert that our glob-parser can handle relatively-complex matching
func TestHasConflictComplexGlob(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"*.[1-9].service"})

	matched, name := state.HasConflict("web.2.service", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}

	matched, _ = state.HasConflict("app.99.service", []string{})
	if matched {
		t.Errorf("Expected no conflict")
	}
}

// Assert that a conflict is truly gone when DropJobConflicts is called
func TestHasConflictDropped(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, name := state.HasConflict("b", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}

	state.DropJobConflicts("a")
	matched, _ = state.HasConflict("b", []string{})
	if matched {
		t.Errorf("Expected no conflict")
	}
}
<commit_msg>test(agent): Add testing of peer tracking in AgentState<commit_after>package agent

import (
	"testing"
)

// Assert that an existing conflict is triggered against the potential job name
func TestHasConflictExistingMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, name := state.HasConflict("b", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}
}

// Assert that a potential conflict is triggered against the existing job name
func TestHasConflictPotentialMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{})

	matched, name := state.HasConflict("b", []string{"a"})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}
}

// Assert that a existing jobs and potential jobs that do not conflict do not
// trigger a match
func TestHasConflictNoMatch(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, _ := state.HasConflict("c", []string{"d"})
	if matched {
		t.Errorf("Expected no match")
	}
}

// Assert that our glob-parser can handle relatively-complex matching
func TestHasConflictComplexGlob(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"*.[1-9].service"})

	matched, name := state.HasConflict("web.2.service", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}

	matched, _ = state.HasConflict("app.99.service", []string{})
	if matched {
		t.Errorf("Expected no conflict")
	}
}

// Assert that a conflict is truly gone when DropJobConflicts is called
func TestHasConflictDropped(t *testing.T) {
	state := NewState()
	state.TrackJobConflicts("a", []string{"b"})

	matched, name := state.HasConflict("b", []string{})
	if !matched || name != "a" {
		t.Errorf("Expected conflict with 'a'")
	}

	state.DropJobConflicts("a")
	matched, _ = state.HasConflict("b", []string{})
	if matched {
		t.Errorf("Expected no conflict")
	}
}

// Assert that jobs and their peers are properly indexed
func TestGetJobsByPeer(t *testing.T) {
	state := NewState()
	state.TrackJobPeers("a", []string{"b", "c"})
	state.TrackJobPeers("d", []string{"c"})

	peers := state.GetJobsByPeer("b")
	if len(peers) != 1 || peers[0] != "a" {
		t.Fatalf("Unexpected index of job peers %v", peers)
	}

	peers = state.GetJobsByPeer("c")
	if len(peers) != 2 || peers[0] != "a" || peers[1] != "d" {
		t.Fatalf("Unexpected index of job peers %v", peers)
	}
}

// Assert that no jobs are returned for unknown peers
func TestGetJobsByPeerUnknown(t *testing.T) {
	state := NewState()
	state.TrackJobPeers("a", []string{"b"})

	peers := state.GetJobsByPeer("c")
	if len(peers) != 0 {
		t.Fatalf("Unexpected index of job peers %v", peers)
	}
}

// Assert that peers indexes are properly cleared after
// calling DropPeersJob
func TestDropPeersJob(t *testing.T) {
	state := NewState()
	state.TrackJobPeers("a", []string{"b", "c"})
	state.TrackJobPeers("d", []string{"c"})
	state.DropPeersJob("a")

	peers := state.GetJobsByPeer("c")
	if len(peers) != 1 || peers[0] != "d" {
		t.Fatalf("Unexpected index of job peers %v", peers)
	}
}
<|endoftext|>