<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package os

import (
	"runtime"
	"sync/atomic"
	"syscall"
)

// File represents an open file descriptor.
type File struct {
	*file
}

// file is the real representation of *File.
// The extra level of indirection ensures that no clients of os
// can overwrite this data, which could cause the finalizer
// to close the wrong file descriptor.
type file struct {
	fd      int
	name    string
	dirinfo *dirInfo // nil unless directory being read
	nepipe  int32    // number of consecutive EPIPE in Write
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (f *File) Fd() uintptr {
	if f == nil {
		return ^(uintptr(0))
	}
	return uintptr(f.fd)
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(fd uintptr, name string) *File {
	fdi := int(fd)
	if fdi < 0 {
		return nil
	}
	f := &File{&file{fd: fdi, name: name}}
	runtime.SetFinalizer(f.file, (*file).close)
	return f
}

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf  []byte // buffer for directory I/O
	nbuf int    // length of buf; return value from Getdirentries
	bufp int    // location of next record in buf.
}

func epipecheck(file *File, e error) {
	if e == syscall.EPIPE {
		if atomic.AddInt32(&file.nepipe, 1) >= 10 {
			sigpipe()
		}
	} else {
		atomic.StoreInt32(&file.nepipe, 0)
	}
}

// DevNull is the name of the operating system's ``null device.''
// On Unix-like systems, it is "/dev/null"; on Windows, "NUL".
const DevNull = "/dev/null"

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func OpenFile(name string, flag int, perm FileMode) (file *File, err error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm))
	if e != nil {
		return nil, &PathError{"open", name, e}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return NewFile(uintptr(r), name), nil
}

// Close closes the File, rendering it unusable for I/O.
// It returns an error, if any.
func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	return f.file.close()
}

func (file *file) close() error {
	if file == nil || file.fd < 0 {
		return syscall.EINVAL
	}
	var err error
	if e := syscall.Close(file.fd); e != nil {
		err = &PathError{"close", file.name, e}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (f *File) Stat() (fi FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	var stat syscall.Stat_t
	err = syscall.Fstat(f.fd, &stat)
	if err != nil {
		return nil, &PathError{"stat", f.name, err}
	}
	return fileInfoFromStat(&stat, f.name), nil
}

// Stat returns a FileInfo describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Stat(name, &stat)
	if err != nil {
		return nil, &PathError{"stat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

// Lstat returns a FileInfo describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link.  Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(name, &stat)
	if err != nil {
		return nil, &PathError{"lstat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	return fi, err
}

// Darwin and FreeBSD can't read or write 2GB+ at a time,
// even on 64-bit systems. See golang.org/issue/7812.
// Use 1GB instead of, say, 2GB-1, to keep subsequent
// reads aligned.
const (
	needsMaxRW = runtime.GOOS == "darwin" || runtime.GOOS == "freebsd"
	maxRW      = 1 << 30
)

// read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an error, if any.
func (f *File) read(b []byte) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Read(f.fd, b))
}

// pread reads len(b) bytes from the File starting at byte offset off.
// It returns the number of bytes read and the error, if any.
// EOF is signaled by a zero count with err set to nil.
func (f *File) pread(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pread(f.fd, b, off))
}

// write writes len(b) bytes to the File.
// It returns the number of bytes written and an error, if any.
func (f *File) write(b []byte) (n int, err error) {
	for {
		bcap := b
		if needsMaxRW && len(bcap) > maxRW {
			bcap = bcap[:maxRW]
		}
		m, err := fixCount(syscall.Write(f.fd, bcap))
		n += m

		// If the syscall wrote some data but not all (short write)
		// or it returned EINTR, then assume it stopped early for
		// reasons that are uninteresting to the caller, and try again.
		if 0 < m && m < len(bcap) || err == syscall.EINTR {
			b = b[m:]
			continue
		}

		if needsMaxRW && len(bcap) != len(b) && err == nil {
			b = b[m:]
			continue
		}

		return n, err
	}
}

// pwrite writes len(b) bytes to the File starting at byte offset off.
// It returns the number of bytes written and an error, if any.
func (f *File) pwrite(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pwrite(f.fd, b, off))
}

// seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an error, if any.
func (f *File) seek(offset int64, whence int) (ret int64, err error) {
	return fixCount(syscall.Seek(f.fd, offset, whence))
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
// If there is an error, it will be of type *PathError.
func Truncate(name string, size int64) error {
	if e := syscall.Truncate(name, size); e != nil {
		return &PathError{"truncate", name, e}
	}
	return nil
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func Remove(name string) error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	e := syscall.Unlink(name)
	if e == nil {
		return nil
	}
	e1 := syscall.Rmdir(name)
	if e1 == nil {
		return nil
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1
	}
	return &PathError{"remove", name, e}
}

// basename removes trailing slashes and the leading directory name from path name
func basename(name string) string {
	i := len(name) - 1
	// Remove trailing slashes
	for ; i > 0 && name[i] == '/'; i-- {
		name = name[:i]
	}
	// Remove leading directory name
	for i--; i >= 0; i-- {
		if name[i] == '/' {
			name = name[i+1:]
			break
		}
	}

	return name
}

// TempDir returns the default directory to use for temporary files.
func TempDir() string {
	dir := Getenv("TMPDIR")
	if dir == "" {
		if runtime.GOOS == "android" {
			dir = "/data/local/tmp"
		} else {
			dir = "/tmp"
		}
	}
	return dir
}

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}
<commit_msg>os: fix build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package os

import (
	"runtime"
	"sync/atomic"
	"syscall"
)

// File represents an open file descriptor.
type File struct {
	*file
}

// file is the real representation of *File.
// The extra level of indirection ensures that no clients of os
// can overwrite this data, which could cause the finalizer
// to close the wrong file descriptor.
type file struct {
	fd      int
	name    string
	dirinfo *dirInfo // nil unless directory being read
	nepipe  int32    // number of consecutive EPIPE in Write
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (f *File) Fd() uintptr {
	if f == nil {
		return ^(uintptr(0))
	}
	return uintptr(f.fd)
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(fd uintptr, name string) *File {
	fdi := int(fd)
	if fdi < 0 {
		return nil
	}
	f := &File{&file{fd: fdi, name: name}}
	runtime.SetFinalizer(f.file, (*file).close)
	return f
}

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf  []byte // buffer for directory I/O
	nbuf int    // length of buf; return value from Getdirentries
	bufp int    // location of next record in buf.
}

func epipecheck(file *File, e error) {
	if e == syscall.EPIPE {
		if atomic.AddInt32(&file.nepipe, 1) >= 10 {
			sigpipe()
		}
	} else {
		atomic.StoreInt32(&file.nepipe, 0)
	}
}

// DevNull is the name of the operating system's ``null device.''
// On Unix-like systems, it is "/dev/null"; on Windows, "NUL".
const DevNull = "/dev/null"

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func OpenFile(name string, flag int, perm FileMode) (file *File, err error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm))
	if e != nil {
		return nil, &PathError{"open", name, e}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return NewFile(uintptr(r), name), nil
}

// Close closes the File, rendering it unusable for I/O.
// It returns an error, if any.
func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	return f.file.close()
}

func (file *file) close() error {
	if file == nil || file.fd < 0 {
		return syscall.EINVAL
	}
	var err error
	if e := syscall.Close(file.fd); e != nil {
		err = &PathError{"close", file.name, e}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (f *File) Stat() (fi FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	var stat syscall.Stat_t
	err = syscall.Fstat(f.fd, &stat)
	if err != nil {
		return nil, &PathError{"stat", f.name, err}
	}
	return fileInfoFromStat(&stat, f.name), nil
}

// Stat returns a FileInfo describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Stat(name, &stat)
	if err != nil {
		return nil, &PathError{"stat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

// Lstat returns a FileInfo describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link.  Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(name, &stat)
	if err != nil {
		return nil, &PathError{"lstat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	return fi, err
}

// Darwin and FreeBSD can't read or write 2GB+ at a time,
// even on 64-bit systems. See golang.org/issue/7812.
// Use 1GB instead of, say, 2GB-1, to keep subsequent
// reads aligned.
const (
	needsMaxRW = runtime.GOOS == "darwin" || runtime.GOOS == "freebsd"
	maxRW      = 1 << 30
)

// read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an error, if any.
func (f *File) read(b []byte) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Read(f.fd, b))
}

// pread reads len(b) bytes from the File starting at byte offset off.
// It returns the number of bytes read and the error, if any.
// EOF is signaled by a zero count with err set to nil.
func (f *File) pread(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pread(f.fd, b, off))
}

// write writes len(b) bytes to the File.
// It returns the number of bytes written and an error, if any.
func (f *File) write(b []byte) (n int, err error) {
	for {
		bcap := b
		if needsMaxRW && len(bcap) > maxRW {
			bcap = bcap[:maxRW]
		}
		m, err := fixCount(syscall.Write(f.fd, bcap))
		n += m

		// If the syscall wrote some data but not all (short write)
		// or it returned EINTR, then assume it stopped early for
		// reasons that are uninteresting to the caller, and try again.
		if 0 < m && m < len(bcap) || err == syscall.EINTR {
			b = b[m:]
			continue
		}

		if needsMaxRW && len(bcap) != len(b) && err == nil {
			b = b[m:]
			continue
		}

		return n, err
	}
}

// pwrite writes len(b) bytes to the File starting at byte offset off.
// It returns the number of bytes written and an error, if any.
func (f *File) pwrite(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pwrite(f.fd, b, off))
}

// seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an error, if any.
func (f *File) seek(offset int64, whence int) (ret int64, err error) {
	return syscall.Seek(f.fd, offset, whence)
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
// If there is an error, it will be of type *PathError.
func Truncate(name string, size int64) error {
	if e := syscall.Truncate(name, size); e != nil {
		return &PathError{"truncate", name, e}
	}
	return nil
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func Remove(name string) error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	e := syscall.Unlink(name)
	if e == nil {
		return nil
	}
	e1 := syscall.Rmdir(name)
	if e1 == nil {
		return nil
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1
	}
	return &PathError{"remove", name, e}
}

// basename removes trailing slashes and the leading directory name from path name
func basename(name string) string {
	i := len(name) - 1
	// Remove trailing slashes
	for ; i > 0 && name[i] == '/'; i-- {
		name = name[:i]
	}
	// Remove leading directory name
	for i--; i >= 0; i-- {
		if name[i] == '/' {
			name = name[i+1:]
			break
		}
	}

	return name
}

// TempDir returns the default directory to use for temporary files.
func TempDir() string {
	dir := Getenv("TMPDIR")
	if dir == "" {
		if runtime.GOOS == "android" {
			dir = "/data/local/tmp"
		} else {
			dir = "/tmp"
		}
	}
	return dir
}

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>interface<commit_after><|endoftext|>


<commit_before>package test

import (
	"github.com/eaciit/toolkit"
	//. "github.com/frezadev/hdc/hive"
	. "github.com/eaciit/hdc/hive"
	//. "github.com/RyanCi/hdc/hive"
	//"log"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

type SportMatch struct {
	Point       string
	HomeTeam    string
	awayTeam    string
	MarkerImage string
	Information string
	Fixture     string
	Capacity    string
	Tv          string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	e := h.Conn.Open()
	e = h.Conn.TestConnection()
	h.Conn.Close()
	fatalCheck(t, "Populate", e)
}

// /* Populate will exec query and immidiately return the value into object
// Populate is suitable for short type query that return limited data,
// Exec is suitable for long type query that return massive amount of data and require time to produce it
// Ideally Populate should call Exec as well but already have predefined function on it receiving process
// */
func TestHivePopulate(t *testing.T) {
	q := "select * from sample_07 limit 5;"

	var result []toolkit.M

	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	e = h.Populate(q, &result)
	fatalCheck(t, "Populate", e)

	if len(result) != 5 {
		t.Logf("Error want %d got %d", 5, len(result))
	}

	t.Logf("Result: \n%s", toolkit.JsonString(result))

	h.Conn.Close()
}

func TestHiveExec(t *testing.T) {
	i := 0
	q := "select * from sample_07 limit 5;"

	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	e = h.Exec(q, func(x HiveResult) error {
		i++
		t.Logf("Receiving data: %s", toolkit.JsonString(x))
		return nil
	})

	if e != nil {
		t.Fatalf("Error exec query: %s", e.Error())
	}

	if i < 5 {
		t.Fatalf("Error receive result. Expect %d got %d", 5, i)
	}

	h.Conn.Close()
}

func TestHiveExecMulti(t *testing.T) {
	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var ms1, ms2 []HiveResult
	q := "select * from sample_07 limit 5"

	e = h.Exec(q, func(x HiveResult) error {
		ms1 = append(ms1, x)
		return nil
	})

	fatalCheck(t, "HS1 exec", e)

	e = h.Exec(q, func(x HiveResult) error {
		ms2 = append(ms2, x)
		return nil
	})

	fatalCheck(t, "HS2 Exec", e)

	t.Logf("Value of HS1\n%s\n\nValue of HS2\n%s", toolkit.JsonString(ms1), toolkit.JsonString(ms2))

	h.Conn.Close()
}

func TestLoad(t *testing.T) {
	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var Student Students

	retVal, err := h.Load("students", "|", "dd/MM/yyyy", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}

//this function works on simple csv and json file
func TestLoadFile(t *testing.T) {
	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var Student Students
	//test csv
	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &Student)

	var SportMatch SportMatch
	//test json
	retValSport, err := h.LoadFile("/home/developer/test json.txt", "SportMatch", "json", "dd/MM/yyyy", &SportMatch)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
	t.Log(retValSport)
}

func TestLoadFileWithWorker(t *testing.T) {
	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		t.Log(err)
	}

	h.Conn.Close()
	t.Log(retVal)
}
<commit_msg>test go profiling<commit_after>package test

import (
	// "github.com/eaciit/toolkit"
	. "github.com/frezadev/hdc/hive"
	// . "github.com/eaciit/hdc/hive"
	//. "github.com/RyanCi/hdc/hive"
	//"log"
	"github.com/pkg/profile"
	"os"
	"testing"
)

var h *Hive
var e error

type Sample7 struct {
	Code        string `tag_name:"code"`
	Description string `tag_name:"description"`
	Total_emp   string `tag_name:"total_emp"`
	Salary      string `tag_name:"salary"`
}

type Students struct {
	Name    string
	Age     int
	Phone   string
	Address string
}

type SportMatch struct {
	Point       string
	HomeTeam    string
	awayTeam    string
	MarkerImage string
	Information string
	Fixture     string
	Capacity    string
	Tv          string
}

func killApp(code int) {
	if h != nil {
		h.Conn.Close()
	}
	os.Exit(code)
}

func fatalCheck(t *testing.T, what string, e error) {
	if e != nil {
		t.Fatalf("%s: %s", what, e.Error())
	}
}

func TestHiveConnect(t *testing.T) {
	h = HiveConfig("192.168.0.223:10000", "default", "hdfs", "", "")
	e := h.Conn.Open()
	e = h.Conn.TestConnection()
	h.Conn.Close()
	fatalCheck(t, "Populate", e)
}

// /* Populate will exec query and immidiately return the value into object
// Populate is suitable for short type query that return limited data,
// Exec is suitable for long type query that return massive amount of data and require time to produce it
// Ideally Populate should call Exec as well but already have predefined function on it receiving process
// */
/*func TestHivePopulate(t *testing.T) {
	q := "select * from sample_07 limit 5;"

	var result []toolkit.M

	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	e = h.Populate(q, &result)
	fatalCheck(t, "Populate", e)

	if len(result) != 5 {
		t.Logf("Error want %d got %d", 5, len(result))
	}

	t.Logf("Result: \n%s", toolkit.JsonString(result))

	h.Conn.Close()
}

func TestHiveExec(t *testing.T) {
	i := 0
	q := "select * from sample_07 limit 5;"

	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	e = h.Exec(q, func(x HiveResult) error {
		i++
		t.Logf("Receiving data: %s", toolkit.JsonString(x))
		return nil
	})

	if e != nil {
		t.Fatalf("Error exec query: %s", e.Error())
	}

	if i < 5 {
		t.Fatalf("Error receive result. Expect %d got %d", 5, i)
	}

	h.Conn.Close()
}

func TestHiveExecMulti(t *testing.T) {
	e := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var ms1, ms2 []HiveResult
	q := "select * from sample_07 limit 5"

	e = h.Exec(q, func(x HiveResult) error {
		ms1 = append(ms1, x)
		return nil
	})

	fatalCheck(t, "HS1 exec", e)

	e = h.Exec(q, func(x HiveResult) error {
		ms2 = append(ms2, x)
		return nil
	})

	fatalCheck(t, "HS2 Exec", e)

	t.Logf("Value of HS1\n%s\n\nValue of HS2\n%s", toolkit.JsonString(ms1), toolkit.JsonString(ms2))

	h.Conn.Close()
}*/

func TestLoad(t *testing.T) {
	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var Student Students

	retVal, err := h.Load("students", "|", "dd/MM/yyyy", &Student)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
}

//this function works on simple csv and json file
func TestLoadFile(t *testing.T) {
	p := profile.Start(profile.MemProfile)

	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var Student Students
	//test csv
	retVal, err := h.LoadFile("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &Student)

	var SportMatch SportMatch
	//test json
	retValSport, err := h.LoadFile("/home/developer/test json.txt", "SportMatch", "json", "dd/MM/yyyy", &SportMatch)

	if err != nil {
		t.Log(err)
	}
	h.Conn.Close()
	t.Log(retVal)
	t.Log(retValSport)

	p.Stop()
}

/*func TestLoadFileWithWorker(t *testing.T) {
	err := h.Conn.Open()
	fatalCheck(t, "Populate", e)

	var student Students

	totalWorker := 10
	retVal, err := h.LoadFileWithWorker("/home/developer/contoh.txt", "students", "csv", "dd/MM/yyyy", &student, totalWorker)

	if err != nil {
		t.Log(err)
	}

	h.Conn.Close()
	t.Log(retVal)
}*/
<|endoftext|>


<commit_before>package machinestatus

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/rancher/lasso/pkg/dynamic"
	"github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1/plan"
	capicontrollers "github.com/rancher/rancher/pkg/generated/controllers/cluster.x-k8s.io/v1alpha4"
	mgmtcontrollers "github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3"
	provisioningcontrollers "github.com/rancher/rancher/pkg/generated/controllers/provisioning.cattle.io/v1"
	rkecontroller "github.com/rancher/rancher/pkg/generated/controllers/rke.cattle.io/v1"
	"github.com/rancher/rancher/pkg/provisioningv2/rke2/planner"
	"github.com/rancher/rancher/pkg/wrangler"
	"github.com/rancher/wrangler/pkg/condition"
	"github.com/rancher/wrangler/pkg/data"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/relatedresource"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	capi "sigs.k8s.io/cluster-api/api/v1alpha4"
)

const (
	Provisioned = condition.Cond("Provisioned")
)

type handler struct {
	secrets              corecontrollers.SecretCache
	machines             capicontrollers.MachineController
	bootstrapCache       rkecontroller.RKEBootstrapCache
	bootstrapController  rkecontroller.RKEBootstrapController
	capiClusterCache     capicontrollers.ClusterCache
	provClusterCache     provisioningcontrollers.ClusterCache
	mgmtClusterCache     mgmtcontrollers.ClusterCache
	rkeControlPlaneCache rkecontroller.RKEControlPlaneCache
	dynamic              *dynamic.Controller
}

func Register(ctx context.Context, clients *wrangler.Context) {
	h := handler{
		secrets:              clients.Core.Secret().Cache(),
		machines:             clients.CAPI.Machine(),
		bootstrapCache:       clients.RKE.RKEBootstrap().Cache(),
		bootstrapController:  clients.RKE.RKEBootstrap(),
		capiClusterCache:     clients.CAPI.Cluster().Cache(),
		mgmtClusterCache:     clients.Mgmt.Cluster().Cache(),
		provClusterCache:     clients.Provisioning.Cluster().Cache(),
		rkeControlPlaneCache: clients.RKE.RKEControlPlane().Cache(),
		dynamic:              clients.Dynamic,
	}
	clients.CAPI.Machine().OnChange(ctx, "machine-status", h.OnChange)

	relatedresource.Watch(ctx, "machine-trigger-from-secret", func(namespace, name string, obj runtime.Object) ([]relatedresource.Key, error) {
		if secret, ok := obj.(*corev1.Secret); ok {
			if secret.Type == planner.SecretTypeMachinePlan {
				return []relatedresource.Key{{
					Namespace: secret.Namespace,
					Name:      secret.Labels[planner.MachineNameLabel],
				}}, nil
			}
		}
		return nil, nil
	}, clients.CAPI.Machine(), clients.Core.Secret())

	h.dynamic.OnChange(ctx, "machine-trigger", func(gvk schema.GroupVersionKind) bool {
		return gvk.Group == "rke-machine.cattle.io"
	}, func(obj runtime.Object) (runtime.Object, error) {
		m, err := meta.Accessor(obj)
		if err != nil {
			return nil, err
		}
		for _, owner := range m.GetOwnerReferences() {
			if owner.Kind == "Machine" {
				h.machines.Enqueue(m.GetNamespace(), owner.Name)
			}
		}
		return obj, nil
	})
}

func (h *handler) setJoinURLFromOutput(machine *capi.Machine, nodePlan *plan.Node) error {
	if nodePlan == nil || !planner.IsEtcdOnlyInitNode(machine) || machine.Annotations[planner.JoinURLAnnotation] != "" {
		return nil
	}

	address, ok := nodePlan.Output["capture-address"]
	if !ok {
		return nil
	}

	var str string
	scanner := bufio.NewScanner(bytes.NewBuffer(address))
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "{") {
			str = line
			break
		}
	}

	if str == "" {
		return nil
	}

	dbInfo := &dbinfo{}
	if err := json.Unmarshal([]byte(str), dbInfo); err != nil {
		return err
	}

	if len(dbInfo.Members) == 0 {
		return nil
	}

	if len(dbInfo.Members[0].ClientURLs) == 0 {
		return nil
	}

	u, err := url.Parse(dbInfo.Members[0].ClientURLs[0])
	if err != nil {
		return err
	}

	cluster, err := h.capiClusterCache.Get(machine.Namespace, machine.Spec.ClusterName)
	if err != nil {
		return err
	}

	if cluster.Spec.ControlPlaneRef == nil {
		return nil
	}

	rkeControlPlane, err := h.rkeControlPlaneCache.Get(machine.Namespace, cluster.Spec.ControlPlaneRef.Name)
	if err != nil {
		return err
	}

	joinURL := fmt.Sprintf("https://%s:%d", u.Hostname(),
		planner.GetRuntimeSupervisorPort(rkeControlPlane.Spec.KubernetesVersion))
	machine = machine.DeepCopy()
	if machine.Annotations == nil {
		machine.Annotations = map[string]string{}
	}
	machine.Annotations[planner.JoinURLAnnotation] = joinURL
	_, err = h.machines.Update(machine)
	return err
}

func (h *handler) OnChange(key string, machine *capi.Machine) (*capi.Machine, error) {
	if machine == nil ||
		machine.Spec.Bootstrap.ConfigRef == nil ||
		machine.Spec.Bootstrap.ConfigRef.Kind != "RKEBootstrap" {
		return machine, nil
	}

	rkeBootstrap, err := h.bootstrapCache.Get(machine.Namespace, machine.Spec.Bootstrap.ConfigRef.Name)
	if err != nil {
		return machine, err
	}

	secret, err := h.secrets.Get(machine.Namespace, planner.PlanSecretFromBootstrapName(rkeBootstrap.Name))
	if apierror.IsNotFound(err) {
		// When the secret exists this handler will be triggered, so don't error
		return machine, nil
	} else if err != nil {
		return machine, err
	}

	plan, err := planner.SecretToNode(secret)
	if err != nil {
		return machine, err
	}

	if err := h.setJoinURLFromOutput(machine, plan); err != nil {
		return machine, err
	}

	status, reason, message, providerID, err := h.getInfraMachineState(machine)
	if err != nil {
		return machine, err
	}

	if status == "" {
		status, reason, message = planner.GetPlanStatusReasonMessage(machine, plan)
	}

	if status == corev1.ConditionTrue && providerID == "" {
		status = corev1.ConditionUnknown
		reason = "NoProviderID"
		message = "waiting for node to be registered in Kubernetes"
		provCluster, err := h.provClusterCache.Get(machine.Namespace, machine.Spec.ClusterName)
		if err == nil {
			mgmtCluster, err := h.mgmtClusterCache.Get(provCluster.Status.ClusterName)
			if err == nil {
				if condition.Cond("Ready").IsTrue(mgmtCluster) {
					h.bootstrapController.Enqueue(machine.Spec.Bootstrap.ConfigRef.Namespace, machine.Spec.Bootstrap.ConfigRef.Name)
				} else if planner.IsOnlyEtcd(machine) {
					message = "waiting for cluster agent to be available on a control plane node"
					h.machines.EnqueueAfter(machine.Namespace, machine.Name, 2*time.Second)
				} else {
					message = "waiting for cluster agent to be available"
					h.machines.EnqueueAfter(machine.Namespace, machine.Name, 2*time.Second)
				}
			}
		}
	}

	if corev1.ConditionStatus(Provisioned.GetStatus(machine)) != status ||
		Provisioned.GetReason(machine) != string(reason) ||
		Provisioned.GetMessage(machine) != message {
		machine := machine.DeepCopy()
		newCond := capi.Condition{
			Type:               capi.ConditionType(Provisioned),
			Status:             status,
			LastTransitionTime: metav1.Now(),
			Reason:             string(reason),
			Message:            message,
		}
		if status == corev1.ConditionFalse {
			newCond.Severity = capi.ConditionSeverityError
		} else {
			newCond.Severity = capi.ConditionSeverityInfo
		}

		set := false
		for i, cond := range machine.Status.Conditions {
			if string(cond.Type) == string(Provisioned) {
				set = true
				machine.Status.Conditions[i] = newCond
				break
			}
		}

		if !set {
			machine.Status.Conditions = append(machine.Status.Conditions, newCond)
		}

		_, err := h.machines.UpdateStatus(machine)
		if err != nil {
			return machine, err
		}
	}

	return machine, nil
}

func (h *handler) getInfraMachineState(capiMachine *capi.Machine) (status corev1.ConditionStatus, reason, message, providerID string, err error) {
	if capiMachine.Status.FailureReason != nil && capiMachine.Status.FailureMessage != nil {
		return corev1.ConditionFalse, "MachineCreateFailed",
			fmt.Sprintf("failed creating server (%s) in infrastructure provider: %s: %s",
				capiMachine.Spec.InfrastructureRef.Kind,
				*capiMachine.Status.FailureReason,
				*capiMachine.Status.FailureMessage), "", nil
	}
	gvk := schema.FromAPIVersionAndKind(capiMachine.Spec.InfrastructureRef.APIVersion, capiMachine.Spec.InfrastructureRef.Kind)
	machine, err := h.dynamic.Get(gvk, capiMachine.Namespace, capiMachine.Spec.InfrastructureRef.Name)
	if apierror.IsNotFound(err) {
		if capiMachine.DeletionTimestamp != nil {
			return corev1.ConditionUnknown, "MachineDeleted", "machine is being deleted", "", nil
		}
		return corev1.ConditionUnknown, "NoMachineDefined", "waiting for machine to be defined", "", nil
	} else if err != nil {
		return "", "", "", "", err
	}

	obj, err := data.Convert(machine)
	if err != nil {
		return "", "", "", "", err
	}

	if capiMachine.Spec.InfrastructureRef.APIVersion == "rke-machine.cattle.io/v1" {
		if obj.String("status", "jobName") == "" {
			return corev1.ConditionUnknown, "NoJob", "waiting to schedule machine create", "", nil
		}

		if !obj.Bool("status", "jobComplete") {
			return corev1.ConditionUnknown, "Creating",
				fmt.Sprintf("creating server (%s) in infrastructure provider", capiMachine.Spec.InfrastructureRef.Kind), "", nil
		}
	}

	return "", "", "", obj.String("spec", "providerID"), nil
}

type dbinfo struct {
	Members []member `json:"members,omitempty"`
}
type member struct {
	ClientURLs []string `json:"clientURLs,omitempty"`
}
<commit_msg>Set InfrastructureReady condition on machine deletion<commit_after>package machinestatus

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/rancher/lasso/pkg/dynamic"
	"github.com/rancher/rancher/pkg/apis/rke.cattle.io/v1/plan"
	capicontrollers "github.com/rancher/rancher/pkg/generated/controllers/cluster.x-k8s.io/v1alpha4"
	mgmtcontrollers "github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3"
	provisioningcontrollers "github.com/rancher/rancher/pkg/generated/controllers/provisioning.cattle.io/v1"
	rkecontroller "github.com/rancher/rancher/pkg/generated/controllers/rke.cattle.io/v1"
	"github.com/rancher/rancher/pkg/provisioningv2/rke2/planner"
	"github.com/rancher/rancher/pkg/wrangler"
	"github.com/rancher/wrangler/pkg/condition"
	"github.com/rancher/wrangler/pkg/data"
	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
	"github.com/rancher/wrangler/pkg/relatedresource"
	corev1 "k8s.io/api/core/v1"
	apierror "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	capi "sigs.k8s.io/cluster-api/api/v1alpha4"
)

const (
	Provisioned         = condition.Cond("Provisioned")
	InfrastructureReady = condition.Cond(capi.InfrastructureReadyCondition)
)

type handler struct {
	secrets              corecontrollers.SecretCache
	machines             capicontrollers.MachineController
	bootstrapCache       rkecontroller.RKEBootstrapCache
	bootstrapController  rkecontroller.RKEBootstrapController
	capiClusterCache     capicontrollers.ClusterCache
	provClusterCache     provisioningcontrollers.ClusterCache
	mgmtClusterCache     mgmtcontrollers.ClusterCache
	rkeControlPlaneCache rkecontroller.RKEControlPlaneCache
	dynamic              *dynamic.Controller
}

func Register(ctx context.Context, clients *wrangler.Context) {
	h := handler{
		secrets:              clients.Core.Secret().Cache(),
		machines:             clients.CAPI.Machine(),
		bootstrapCache:       clients.RKE.RKEBootstrap().Cache(),
		bootstrapController:  clients.RKE.RKEBootstrap(),
		capiClusterCache:     clients.CAPI.Cluster().Cache(),
		mgmtClusterCache:     clients.Mgmt.Cluster().Cache(),
		provClusterCache:     clients.Provisioning.Cluster().Cache(),
		rkeControlPlaneCache: clients.RKE.RKEControlPlane().Cache(),
		dynamic:              clients.Dynamic,
	}
	clients.CAPI.Machine().OnChange(ctx, "machine-status", h.OnChange)

	relatedresource.Watch(ctx, "machine-trigger-from-secret", func(namespace, name string, obj runtime.Object) ([]relatedresource.Key, error) {
		if secret, ok := obj.(*corev1.Secret); ok {
			if secret.Type == planner.SecretTypeMachinePlan {
				return []relatedresource.Key{{
					Namespace: secret.Namespace,
					Name:      secret.Labels[planner.MachineNameLabel],
				}}, nil
			}
		}
		return nil, nil
	}, clients.CAPI.Machine(), clients.Core.Secret())

	h.dynamic.OnChange(ctx, "machine-trigger", func(gvk schema.GroupVersionKind) bool {
		return gvk.Group == "rke-machine.cattle.io"
	}, func(obj runtime.Object) (runtime.Object, error) {
		m, err := meta.Accessor(obj)
		if err != nil {
			return nil, err
		}
		for _, owner := range m.GetOwnerReferences() {
			if owner.Kind == "Machine" {
				h.machines.Enqueue(m.GetNamespace(), owner.Name)
			}
		}
		return obj, nil
	})
}

func (h *handler) setJoinURLFromOutput(machine *capi.Machine, nodePlan *plan.Node) error {
	if nodePlan == nil || !planner.IsEtcdOnlyInitNode(machine) || machine.Annotations[planner.JoinURLAnnotation] != "" {
		return nil
	}

	address, ok := nodePlan.Output["capture-address"]
	if !ok {
		return nil
	}

	var str string
	scanner := bufio.NewScanner(bytes.NewBuffer(address))
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "{") {
			str = line
			break
		}
	}

	if str == "" {
		return nil
	}

	dbInfo := &dbinfo{}
	if err := json.Unmarshal([]byte(str), dbInfo); err != nil {
		return err
	}

	if len(dbInfo.Members) == 0 {
		return nil
	}

	if len(dbInfo.Members[0].ClientURLs) == 0 {
		return nil
	}

	u, err := url.Parse(dbInfo.Members[0].ClientURLs[0])
	if err != nil {
		return err
	}

	cluster, err := h.capiClusterCache.Get(machine.Namespace, machine.Spec.ClusterName)
	if err != nil {
		return err
	}

	if cluster.Spec.ControlPlaneRef == nil {
		return nil
	}

	rkeControlPlane, err := h.rkeControlPlaneCache.Get(machine.Namespace, cluster.Spec.ControlPlaneRef.Name)
	if err != nil {
		return err
	}

	joinURL := fmt.Sprintf("https://%s:%d", u.Hostname(),
		planner.GetRuntimeSupervisorPort(rkeControlPlane.Spec.KubernetesVersion))
	machine = machine.DeepCopy()
	if machine.Annotations == nil {
		machine.Annotations = map[string]string{}
	}
	machine.Annotations[planner.JoinURLAnnotation] = joinURL
	_, err = h.machines.Update(machine)
	return err
}

func (h *handler) OnChange(key string, machine *capi.Machine) (*capi.Machine, error) {
	if machine == nil ||
		machine.Spec.Bootstrap.ConfigRef == nil ||
		machine.Spec.Bootstrap.ConfigRef.Kind != "RKEBootstrap" {
		return machine, nil
	}

	status, reason, message, providerID, err := h.getInfraMachineState(machine)
	if machine.DeletionTimestamp != nil && apierror.IsNotFound(err) {
		// If the machine is being deleted and the infrastructure machine object is not found,
		// then update the status of the machine object so the CAPI controller picks it up.
		return h.setMachineCondition(machine, InfrastructureReady, status, reason, message)
	} else if err != nil {
		return machine, err
	}

	rkeBootstrap, err := h.bootstrapCache.Get(machine.Namespace, machine.Spec.Bootstrap.ConfigRef.Name)
	if err != nil {
		return machine, err
	}

	secret, err := h.secrets.Get(machine.Namespace, planner.PlanSecretFromBootstrapName(rkeBootstrap.Name))
	if apierror.IsNotFound(err) {
		// When the secret exists this handler will be triggered, so don't error
		return machine, nil
	} else if err != nil {
		return machine, err
	}

	plan, err := planner.SecretToNode(secret)
	if err != nil {
		return machine, err
	}

	if err := h.setJoinURLFromOutput(machine, plan); err != nil {
		return machine, err
	}

	if status == "" {
		status, reason, message = planner.GetPlanStatusReasonMessage(machine, plan)
	}

	if status == corev1.ConditionTrue && providerID == "" {
		status = corev1.ConditionUnknown
		reason = "NoProviderID"
		message = "waiting for node to be registered in Kubernetes"
		provCluster, err := h.provClusterCache.Get(machine.Namespace, machine.Spec.ClusterName)
		if err == nil {
			mgmtCluster, err := h.mgmtClusterCache.Get(provCluster.Status.ClusterName)
			if err == nil {
				if condition.Cond("Ready").IsTrue(mgmtCluster) {
					h.bootstrapController.Enqueue(machine.Spec.Bootstrap.ConfigRef.Namespace, machine.Spec.Bootstrap.ConfigRef.Name)
				} else if planner.IsOnlyEtcd(machine) {
					message = "waiting for cluster agent to be available on a control plane node"
					h.machines.EnqueueAfter(machine.Namespace, machine.Name, 2*time.Second)
				} else {
					message = "waiting for cluster agent to be available"
					h.machines.EnqueueAfter(machine.Namespace, machine.Name, 2*time.Second)
				}
			}
		}
	}

	return h.setMachineCondition(machine, Provisioned, status, reason, message)
}

func (h *handler) setMachineCondition(machine *capi.Machine, cond condition.Cond, status corev1.ConditionStatus, reason, message string) (*capi.Machine, error) {
	if corev1.ConditionStatus(cond.GetStatus(machine)) != status ||
		cond.GetReason(machine) != reason ||
		cond.GetMessage(machine) != message {
		machine := machine.DeepCopy()
		newCond := capi.Condition{
			Type:               capi.ConditionType(cond),
			Status:             status,
			LastTransitionTime: metav1.Now(),
			Reason:             reason,
			Message:            message,
		}
		if cond != InfrastructureReady && status == corev1.ConditionFalse {
			newCond.Severity = capi.ConditionSeverityError
		} else {
			newCond.Severity = capi.ConditionSeverityInfo
		}

		set := false
		for i, c := range machine.Status.Conditions {
			if string(c.Type) == string(cond) {
				set = true
				machine.Status.Conditions[i] = newCond
				break
			}
		}

		if !set {
			machine.Status.Conditions = append(machine.Status.Conditions, newCond)
		}

		_, err := h.machines.UpdateStatus(machine)
		if err != nil {
			return machine, err
		}
	}

	return machine, nil
}

func (h *handler) getInfraMachineState(capiMachine *capi.Machine) (status corev1.ConditionStatus, reason, message, providerID string, err error) {
	if capiMachine.Status.FailureReason != nil && capiMachine.Status.FailureMessage != nil {
		return corev1.ConditionFalse, "MachineCreateFailed",
			fmt.Sprintf("failed creating server (%s) in infrastructure provider: %s: %s",
				capiMachine.Spec.InfrastructureRef.Kind,
				*capiMachine.Status.FailureReason,
				*capiMachine.Status.FailureMessage), "", nil
	}
	gvk := schema.FromAPIVersionAndKind(capiMachine.Spec.InfrastructureRef.APIVersion, capiMachine.Spec.InfrastructureRef.Kind)
	machine, err := h.dynamic.Get(gvk, capiMachine.Namespace, capiMachine.Spec.InfrastructureRef.Name)
	if apierror.IsNotFound(err) {
		if capiMachine.DeletionTimestamp != nil {
			return corev1.ConditionFalse, capi.DeletedReason, "machine infrastructure is deleted", "", err
		}
		return corev1.ConditionUnknown, "NoMachineDefined", "waiting for machine to be defined", "", nil
	} else if err != nil {
		return "", "", "", "", err
	}

	obj, err := data.Convert(machine)
	if err != nil {
		return "", "", "", "", err
	}

	if capiMachine.Spec.InfrastructureRef.APIVersion == "rke-machine.cattle.io/v1" {
		if obj.String("status", "jobName") == "" {
			return corev1.ConditionUnknown, "NoJob", "waiting to schedule machine create", "", nil
		}

		if !obj.Bool("status", "jobComplete") {
			return corev1.ConditionUnknown, "Creating",
				fmt.Sprintf("creating server (%s) in infrastructure provider", capiMachine.Spec.InfrastructureRef.Kind), "", nil
		}
	}

	return "", "", "", obj.String("spec", "providerID"), nil
}

type dbinfo struct {
	Members []member `json:"members,omitempty"`
}
type member struct {
	ClientURLs []string `json:"clientURLs,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This LevelDB Go implementation is based on LevelDB C++ implementation.
// Which contains the following header:
//   Copyright (c) 2011 The LevelDB Authors. All rights reserved.
//   Use of this source code is governed by a BSD-style license that can be
//   found in the LEVELDBCPP_LICENSE file. See the LEVELDBCPP_AUTHORS file
//   for names of contributors.

package db

import (
	"fmt"
	"leveldb/descriptor"
	"leveldb/log"
)

// logging

func (s *session) print(v ...interface{}) {
	s.desc.Print(fmt.Sprint(v...))
}

func (s *session) printf(format string, v ...interface{}) {
	s.desc.Print(fmt.Sprintf(format, v...))
}

// file utils

func (s *session) getLogFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeLog)
}

func (s *session) getTableFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeTable)
}

func (s *session) getTempFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeTemp)
}

func (s *session) getFiles(t descriptor.FileType) []descriptor.File {
	return s.desc.GetFiles(t)
}

// session state

type stateNum struct {
	num        uint64
	isCommited bool
}

func (p *stateNum) set(num uint64) {
	if p.num != num {
		p.num = num
		p.isCommited = false
	}
}

func (p *stateNum) commited(num uint64) {
	if p.num == num {
		p.isCommited = true
	}
}

func (s *session) setVersion(v *version) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	st.version = v
	st.versions = append(st.versions, v)
}

func (s *session) setFileNum(num uint64) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	st.nextNum.set(num)
}

func (s *session) markFileNum(num uint64) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	if num > st.nextNum.num {
		st.nextNum.set(num)
	}
}

func (s *session) allocFileNumNL() (num uint64) {
	st := &s.st
	num = st.nextNum.num
	st.nextNum.num++
	st.nextNum.isCommited = false
	return
}

func (s *session) allocFileNum() (num uint64) {
	s.st.Lock()
	defer s.st.Unlock()
	return s.allocFileNumNL()
}

func (s *session) reuseFileNum(num uint64) {
	st := &s.st
	st.Lock()
	if st.nextNum.num == num+1 {
		st.nextNum.num = num
	}
	st.Unlock()
}

func (s *session) version() *version {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	return st.version
}

func (s *session) seq() uint64 {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	return st.seq
}

func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
	st := &s.st
	st.RLock()
	defer st.RUnlock()

	if snapshot || !st.nextNum.isCommited {
		r.setNextNum(st.nextNum.num)
	}

	if snapshot {
		if !r.hasLogNum {
			r.setLogNum(st.logNum)
		}

		if !r.hasLogNum {
			r.setSeq(st.seq)
		}

		for level, ik := range st.compactPointers {
			r.addCompactPointer(level, ik)
		}
	}

	if snapshot {
		r.setComparer(s.cmp.cmp.Name())
	}
}

func (s *session) recordCommited(r *sessionRecord) {
	st := &s.st
	st.Lock()
	defer st.Unlock()

	if r.hasNextNum {
		st.nextNum.commited(r.nextNum)
	}

	if r.hasLogNum {
		st.logNum = r.logNum
	}

	if r.hasSeq {
		st.seq = r.seq
	}

	for _, p := range r.compactPointers {
		st.compactPointers[p.level] = iKey(p.key)
	}
}

func (s *session) createManifest(num uint64, r *sessionRecord, v *version) (err error) {
	file := s.desc.GetFile(num, descriptor.TypeManifest)
	s.manifestWriter, err = file.Create()
	if err != nil {
		return
	}

	s.manifest = log.NewWriter(s.manifestWriter)

	if v == nil {
		v = s.st.version
	}

	if r == nil {
		r = new(sessionRecord)
	}
	s.fillRecord(r, true)
	v.fillRecord(r)
	defer func() {
		if err == nil {
			s.recordCommited(r)
			s.manifestFile = file
		} else {
			s.manifest = nil
			s.manifestWriter.Close()
			s.manifestWriter = nil
			file.Remove()
		}
	}()
	err = s.manifest.Append(r.encode())
	if err != nil {
		return
	}
	return s.desc.SetMainManifest(file)
}

func (s *session) flushManifest(r *sessionRecord) (err error) {
	s.fillRecord(r, false)
	err = s.manifest.Append(r.encode())
	if err != nil {
		return
	}
	s.recordCommited(r)
	return
}
<commit_msg>db: sync after write to manifest and remove old manifest after creating new one<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// This LevelDB Go implementation is based on LevelDB C++ implementation.
// Which contains the following header:
//   Copyright (c) 2011 The LevelDB Authors. All rights reserved.
//   Use of this source code is governed by a BSD-style license that can be
//   found in the LEVELDBCPP_LICENSE file. See the LEVELDBCPP_AUTHORS file
//   for names of contributors.

package db

import (
	"fmt"
	"leveldb/descriptor"
	"leveldb/log"
)

// logging

func (s *session) print(v ...interface{}) {
	s.desc.Print(fmt.Sprint(v...))
}

func (s *session) printf(format string, v ...interface{}) {
	s.desc.Print(fmt.Sprintf(format, v...))
}

// file utils

func (s *session) getLogFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeLog)
}

func (s *session) getTableFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeTable)
}

func (s *session) getTempFile(num uint64) descriptor.File {
	return s.desc.GetFile(num, descriptor.TypeTemp)
}

func (s *session) getFiles(t descriptor.FileType) []descriptor.File {
	return s.desc.GetFiles(t)
}

// session state

type stateNum struct {
	num        uint64
	isCommited bool
}

func (p *stateNum) set(num uint64) {
	if p.num != num {
		p.num = num
		p.isCommited = false
	}
}

func (p *stateNum) commited(num uint64) {
	if p.num == num {
		p.isCommited = true
	}
}

func (s *session) setVersion(v *version) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	st.version = v
	st.versions = append(st.versions, v)
}

func (s *session) setFileNum(num uint64) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	st.nextNum.set(num)
}

func (s *session) markFileNum(num uint64) {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	if num > st.nextNum.num {
		st.nextNum.set(num)
	}
}

func (s *session) allocFileNumNL() (num uint64) {
	st := &s.st
	num = st.nextNum.num
	st.nextNum.num++
	st.nextNum.isCommited = false
	return
}

func (s *session) allocFileNum() (num uint64) {
	s.st.Lock()
	defer s.st.Unlock()
	return s.allocFileNumNL()
}

func (s *session) reuseFileNum(num uint64) {
	st := &s.st
	st.Lock()
	if st.nextNum.num == num+1 {
		st.nextNum.num = num
	}
	st.Unlock()
}

func (s *session) version() *version {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	return st.version
}

func (s *session) seq() uint64 {
	st := &s.st
	st.Lock()
	defer st.Unlock()
	return st.seq
}

func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
	st := &s.st
	st.RLock()
	defer st.RUnlock()

	if snapshot || !st.nextNum.isCommited {
		r.setNextNum(st.nextNum.num)
	}

	if snapshot {
		if !r.hasLogNum {
			r.setLogNum(st.logNum)
		}

		if !r.hasLogNum {
			r.setSeq(st.seq)
		}

		for level, ik := range st.compactPointers {
			r.addCompactPointer(level, ik)
		}
	}

	if snapshot {
		r.setComparer(s.cmp.cmp.Name())
	}
}

func (s *session) recordCommited(r *sessionRecord) {
	st := &s.st
	st.Lock()
	defer st.Unlock()

	if r.hasNextNum {
		st.nextNum.commited(r.nextNum)
	}

	if r.hasLogNum {
		st.logNum = r.logNum
	}

	if r.hasSeq {
		st.seq = r.seq
	}

	for _, p := range r.compactPointers {
		st.compactPointers[p.level] = iKey(p.key)
	}
}

func (s *session) createManifest(num uint64, r *sessionRecord, v *version) (err error) {
	file := s.desc.GetFile(num, descriptor.TypeManifest)
	s.manifestWriter, err = file.Create()
	if err != nil {
		return
	}

	s.manifest = log.NewWriter(s.manifestWriter)

	if v == nil {
		v = s.st.version
	}

	if r == nil {
		r = new(sessionRecord)
	}
	s.fillRecord(r, true)
	v.fillRecord(r)
	defer func() {
		if err == nil {
			s.recordCommited(r)
			if s.manifestFile != nil {
				s.manifestFile.Remove()
			}
			s.manifestFile = file
		} else {
			s.manifest = nil
			s.manifestWriter.Close()
			s.manifestWriter = nil
			file.Remove()
		}
	}()
	err = s.manifest.Append(r.encode())
	if err != nil {
		return
	}
	err = s.manifestWriter.Sync()
	if err != nil {
		return
	}
	return s.desc.SetMainManifest(file)
}

func (s *session) flushManifest(r *sessionRecord) (err error) {
	s.fillRecord(r, false)
	err = s.manifest.Append(r.encode())
	if err != nil {
		return
	}
	err = s.manifestWriter.Sync()
	if err != nil {
		return
	}
	s.recordCommited(r)
	return
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"sort"

	"k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	kapi "k8s.io/kubernetes/pkg/apis/core"
	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/third_party/forked/golang/expansion"

	buildadmission "github.com/openshift/origin/pkg/build/admission"
	buildapi "github.com/openshift/origin/pkg/build/apis/build"
	buildclient "github.com/openshift/origin/pkg/build/client"
	buildlister "github.com/openshift/origin/pkg/build/generated/listers/build/internalversion"
	buildutil "github.com/openshift/origin/pkg/build/util"
	envresolve "github.com/openshift/origin/pkg/pod/envresolve"

	"github.com/golang/glog"
)

type ByCreationTimestamp []*buildapi.Build

func (b ByCreationTimestamp) Len() int {
	return len(b)
}

func (b ByCreationTimestamp) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b ByCreationTimestamp) Less(i, j int) bool {
	return !b[j].CreationTimestamp.Time.After(b[i].CreationTimestamp.Time)
}

// HandleBuildPruning handles the deletion of old successful and failed builds
// based on settings in the BuildConfig.
func HandleBuildPruning(buildConfigName string, namespace string, buildLister buildlister.BuildLister, buildConfigGetter buildlister.BuildConfigLister, buildDeleter buildclient.BuildDeleter) error {
	buildConfig, err := buildConfigGetter.BuildConfigs(namespace).Get(buildConfigName)
	if err != nil {
		return err
	}

	var buildsToDelete []*buildapi.Build
	var errList []error

	if buildConfig.Spec.SuccessfulBuildsHistoryLimit != nil {
		successfulBuilds, err := buildutil.BuildConfigBuilds(buildLister, namespace, buildConfigName, func(build *buildapi.Build) bool { return build.Status.Phase == buildapi.BuildPhaseComplete })
		if err != nil {
			return err
		}
		sort.Sort(ByCreationTimestamp(successfulBuilds))

		successfulBuildsHistoryLimit := int(*buildConfig.Spec.SuccessfulBuildsHistoryLimit)
		glog.V(4).Infof("Current builds: %v, SuccessfulBuildsHistoryLimit: %v", len(successfulBuilds), successfulBuildsHistoryLimit)
		if len(successfulBuilds) > successfulBuildsHistoryLimit {
			glog.V(4).Infof("Preparing to prune %v of %v old successful builds, successfulBuildsHistoryLimit set to %v", (len(successfulBuilds) - successfulBuildsHistoryLimit), len(successfulBuilds), successfulBuildsHistoryLimit)
			buildsToDelete = append(buildsToDelete, successfulBuilds[successfulBuildsHistoryLimit:]...)
		}
	}

	if buildConfig.Spec.FailedBuildsHistoryLimit != nil {
		failedBuilds, err := buildutil.BuildConfigBuilds(buildLister, namespace, buildConfigName, func(build *buildapi.Build) bool {
			return (build.Status.Phase == buildapi.BuildPhaseFailed || build.Status.Phase == buildapi.BuildPhaseCancelled || build.Status.Phase == buildapi.BuildPhaseError)
		})
		if err != nil {
			return err
		}
		sort.Sort(ByCreationTimestamp(failedBuilds))

		failedBuildsHistoryLimit := int(*buildConfig.Spec.FailedBuildsHistoryLimit)
		glog.V(4).Infof("Current builds: %v, FailedBuildsHistoryLimit: %v", len(failedBuilds), failedBuildsHistoryLimit)
		if len(failedBuilds) > failedBuildsHistoryLimit {
			glog.V(4).Infof("Preparing to prune %v of %v old failed builds, failedBuildsHistoryLimit set to %v", (len(failedBuilds) - failedBuildsHistoryLimit), len(failedBuilds), failedBuildsHistoryLimit)
			buildsToDelete = append(buildsToDelete, failedBuilds[failedBuildsHistoryLimit:]...)
		}
	}

	for i, b := range buildsToDelete {
		glog.V(4).Infof("Pruning old build: %s/%s", b.Namespace, b.Name)
		if err := buildDeleter.DeleteBuild(buildsToDelete[i]); err != nil {
			errList = append(errList, err)
		}
	}
	if errList != nil {
		return kerrors.NewAggregate(errList)
	}

	return nil
}

func SetBuildPodNameAnnotation(build *buildapi.Build, podName string) {
	if build.Annotations == nil {
		build.Annotations = map[string]string{}
	}
	build.Annotations[buildapi.BuildPodNameAnnotation] = podName
}

func HasBuildPodNameAnnotation(build *buildapi.Build) bool {
	if build.Annotations == nil {
		return false
	}
	_, hasAnnotation := build.Annotations[buildapi.BuildPodNameAnnotation]
	return hasAnnotation
}

// ErrEnvVarResolver is an error type for build environment resolution errors
type ErrEnvVarResolver struct {
	message kerrors.Aggregate
}

// Error returns a string representation of the error
func (e ErrEnvVarResolver) Error() string {
	return fmt.Sprintf("%v", e.message)
}

// ResolveValueFrom resolves valueFrom references in build environment variables
// including references to existing environment variables, jsonpath references to
// the build object, secrets, and configmaps.
// The build.Strategy.BuildStrategy.Env is replaced with the resolved references.
func ResolveValueFrom(pod *v1.Pod, client kclientset.Interface) error {
	var outputEnv []kapi.EnvVar
	var allErrs []error

	build, version, err := buildadmission.GetBuildFromPod(pod)
	if err != nil {
		return nil
	}

	mapEnvs := map[string]string{}
	mapping := expansion.MappingFuncFor(mapEnvs)
	inputEnv := buildutil.GetBuildEnv(build)
	store := envresolve.NewResourceStore()

	for _, e := range inputEnv {
		var value string
		var err error

		if e.Value != "" {
			value = expansion.Expand(e.Value, mapping)
		} else if e.ValueFrom != nil {
			value, err = envresolve.GetEnvVarRefValue(client, build.Namespace, store, e.ValueFrom, build, nil)
			if err != nil {
				allErrs = append(allErrs, err)
				continue
			}
		}

		outputEnv = append(outputEnv, kapi.EnvVar{Name: e.Name, Value: value})
		mapEnvs[e.Name] = value
	}

	if len(allErrs) > 0 {
		return ErrEnvVarResolver{utilerrors.NewAggregate(allErrs)}
	}

	buildutil.SetBuildEnv(build, outputEnv)
	return buildadmission.SetBuildInPod(pod, build, version)
}
<commit_msg>JenkinsPipelineStrategy builds should not be pruned on BuildConfig save<commit_after>package common

import (
	"fmt"
	"sort"

	"k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	kapi "k8s.io/kubernetes/pkg/apis/core"
	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/third_party/forked/golang/expansion"

	buildadmission "github.com/openshift/origin/pkg/build/admission"
	buildapi "github.com/openshift/origin/pkg/build/apis/build"
	buildclient "github.com/openshift/origin/pkg/build/client"
	buildlister "github.com/openshift/origin/pkg/build/generated/listers/build/internalversion"
	buildutil "github.com/openshift/origin/pkg/build/util"
	envresolve "github.com/openshift/origin/pkg/pod/envresolve"

	"github.com/golang/glog"
)

type ByCreationTimestamp []*buildapi.Build

func (b ByCreationTimestamp) Len() int {
	return len(b)
}

func (b ByCreationTimestamp) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b ByCreationTimestamp) Less(i, j int) bool {
	return !b[j].CreationTimestamp.Time.After(b[i].CreationTimestamp.Time)
}

// HandleBuildPruning handles the deletion of old successful and failed builds
// based on settings in the BuildConfig.
func HandleBuildPruning(buildConfigName string, namespace string, buildLister buildlister.BuildLister, buildConfigGetter buildlister.BuildConfigLister, buildDeleter buildclient.BuildDeleter) error {
	buildConfig, err := buildConfigGetter.BuildConfigs(namespace).Get(buildConfigName)
	if err != nil {
		return err
	}

	if buildConfig.Spec.Strategy.JenkinsPipelineStrategy != nil {
		glog.V(4).Infof("Build pruning for %s/%s is handled by Jenkins, skipping.", buildConfig.Namespace, buildConfig.Name)
		return nil
	}

	var buildsToDelete []*buildapi.Build
	var errList []error

	if buildConfig.Spec.SuccessfulBuildsHistoryLimit != nil {
		successfulBuilds, err := buildutil.BuildConfigBuilds(buildLister, namespace, buildConfigName, func(build *buildapi.Build) bool { return build.Status.Phase == buildapi.BuildPhaseComplete })
		if err != nil {
			return err
		}
		sort.Sort(ByCreationTimestamp(successfulBuilds))

		successfulBuildsHistoryLimit := int(*buildConfig.Spec.SuccessfulBuildsHistoryLimit)
		glog.V(4).Infof("Current builds: %v, SuccessfulBuildsHistoryLimit: %v", len(successfulBuilds), successfulBuildsHistoryLimit)
		if len(successfulBuilds) > successfulBuildsHistoryLimit {
			glog.V(4).Infof("Preparing to prune %v of %v old successful builds, successfulBuildsHistoryLimit set to %v", (len(successfulBuilds) - successfulBuildsHistoryLimit), len(successfulBuilds), successfulBuildsHistoryLimit)
			buildsToDelete = append(buildsToDelete, successfulBuilds[successfulBuildsHistoryLimit:]...)
		}
	}

	if buildConfig.Spec.FailedBuildsHistoryLimit != nil {
		failedBuilds, err := buildutil.BuildConfigBuilds(buildLister, namespace, buildConfigName, func(build *buildapi.Build) bool {
			return (build.Status.Phase == buildapi.BuildPhaseFailed || build.Status.Phase == buildapi.BuildPhaseCancelled || build.Status.Phase == buildapi.BuildPhaseError)
		})
		if err != nil {
			return err
		}
		sort.Sort(ByCreationTimestamp(failedBuilds))

		failedBuildsHistoryLimit := int(*buildConfig.Spec.FailedBuildsHistoryLimit)
		glog.V(4).Infof("Current builds: %v, FailedBuildsHistoryLimit: %v", len(failedBuilds), failedBuildsHistoryLimit)
		if len(failedBuilds) > failedBuildsHistoryLimit {
			glog.V(4).Infof("Preparing to prune %v of %v old failed builds, failedBuildsHistoryLimit set to %v", (len(failedBuilds) - failedBuildsHistoryLimit), len(failedBuilds), failedBuildsHistoryLimit)
			buildsToDelete = append(buildsToDelete, failedBuilds[failedBuildsHistoryLimit:]...)
		}
	}

	for i, b := range buildsToDelete {
		glog.V(4).Infof("Pruning old build: %s/%s", b.Namespace, b.Name)
		if err := buildDeleter.DeleteBuild(buildsToDelete[i]); err != nil {
			errList = append(errList, err)
		}
	}
	if errList != nil {
		return kerrors.NewAggregate(errList)
	}

	return nil
}

func SetBuildPodNameAnnotation(build *buildapi.Build, podName string) {
	if build.Annotations == nil {
		build.Annotations = map[string]string{}
	}
	build.Annotations[buildapi.BuildPodNameAnnotation] = podName
}

func HasBuildPodNameAnnotation(build *buildapi.Build) bool {
	if build.Annotations == nil {
		return false
	}
	_, hasAnnotation := build.Annotations[buildapi.BuildPodNameAnnotation]
	return hasAnnotation
}

// ErrEnvVarResolver is an error type for build environment resolution errors
type ErrEnvVarResolver struct {
	message kerrors.Aggregate
}

// Error returns a string representation of the error
func (e ErrEnvVarResolver) Error() string {
	return fmt.Sprintf("%v", e.message)
}

// ResolveValueFrom resolves valueFrom references in build environment variables
// including references to existing environment variables, jsonpath references to
// the build object, secrets, and configmaps.
// The build.Strategy.BuildStrategy.Env is replaced with the resolved references.
func ResolveValueFrom(pod *v1.Pod, client kclientset.Interface) error {
	var outputEnv []kapi.EnvVar
	var allErrs []error

	build, version, err := buildadmission.GetBuildFromPod(pod)
	if err != nil {
		return nil
	}

	mapEnvs := map[string]string{}
	mapping := expansion.MappingFuncFor(mapEnvs)
	inputEnv := buildutil.GetBuildEnv(build)
	store := envresolve.NewResourceStore()

	for _, e := range inputEnv {
		var value string
		var err error

		if e.Value != "" {
			value = expansion.Expand(e.Value, mapping)
		} else if e.ValueFrom != nil {
			value, err = envresolve.GetEnvVarRefValue(client, build.Namespace, store, e.ValueFrom, build, nil)
			if err != nil {
				allErrs = append(allErrs, err)
				continue
			}
		}

		outputEnv = append(outputEnv, kapi.EnvVar{Name: e.Name, Value: value})
		mapEnvs[e.Name] = value
	}

	if len(allErrs) > 0 {
		return ErrEnvVarResolver{utilerrors.NewAggregate(allErrs)}
	}

	buildutil.SetBuildEnv(build, outputEnv)
	return buildadmission.SetBuildInPod(pod, build, version)
}
<|endoftext|>


<commit_before>package physics

import (
	"io"
	"math"
	"os"

	"chunkymonkey/proto"
	. "chunkymonkey/types"
)

const (
	// Guestimated gravity value. Unknown how accurate this is.
	gravityBlocksPerSecond2 = 3.0

	gravityBlocksPerTick2 = gravityBlocksPerSecond2 / TicksPerSecond

	// Air resistance, as a denominator of a velocity component
	airResistance = 5

	// Min velocity component before we clamp to zero
	minVel = 0.01

	objBlockDistance = 4.25 / PixelsPerBlock
)

type blockAxisMove byte

const (
	blockAxisMoveX = blockAxisMove(iota)
	blockAxisMoveY = blockAxisMove(iota)
	blockAxisMoveZ = blockAxisMove(iota)
)

type BlockQueryFn func(*BlockXyz) (isSolid bool, isWithinChunk bool)

type PointObject struct {
	// Used in knowing what to send as client updates
	LastSentPosition AbsIntXyz
	LastSentVelocity Velocity

	// Used in physical modelling
	position  AbsXyz
	velocity  AbsVelocity
	onGround  bool
	remainder TickTime
}

func (obj *PointObject) Position() *AbsXyz {
	return &obj.position
}

func (obj *PointObject) Init(position *AbsXyz, velocity *AbsVelocity) {
	obj.LastSentPosition = *position.ToAbsIntXyz()
	obj.LastSentVelocity = *velocity.ToVelocity()
	obj.position = *position
	obj.velocity = *velocity
	obj.onGround = false
}

// Generates any packets needed to update clients as to the position and
// velocity of the object.
// It assumes that the clients have either been sent packets via this method
// before, or that the previous position/velocity sent was generated from the
// LastSentPosition and LastSentVelocity attributes.
func (obj *PointObject) SendUpdate(writer io.Writer, entityId EntityId, look *LookBytes) (err os.Error) {
	curPosition := obj.position.ToAbsIntXyz()

	dx := curPosition.X - obj.LastSentPosition.X
	dy := curPosition.Y - obj.LastSentPosition.Y
	dz := curPosition.Z - obj.LastSentPosition.Z

	if dx != 0 || dy != 0 || dz != 0 {
		if dx >= -128 && dx <= 127 && dy >= -128 && dy <= 127 && dz >= -128 && dz <= 127 {
			err = proto.WriteEntityRelMove(
				writer, entityId,
				&RelMove{
					RelMoveCoord(dx),
					RelMoveCoord(dy),
					RelMoveCoord(dz),
				},
			)
		} else {
			err = proto.WriteEntityTeleport(
				writer, entityId,
				curPosition, look)
		}
		if err != nil {
			return
		}
		obj.LastSentPosition = *curPosition
	}

	curVelocity := obj.velocity.ToVelocity()
	if curVelocity.X != obj.LastSentVelocity.X || curVelocity.Y != obj.LastSentVelocity.Y || curVelocity.Z != obj.LastSentVelocity.Z {
		if err = proto.WriteEntityVelocity(writer, entityId, curVelocity); err != nil {
			return
		}
		obj.LastSentVelocity = *curVelocity
	}

	return
}

func (obj *PointObject) Tick(blockQuery BlockQueryFn) (leftBlock bool) {
	// TODO this algorithm can probably be sped up a bit, but initially trying
	// to keep things simple and more or less correct
	// TODO flowing water movement of items

	p := &obj.position
	v := &obj.velocity

	// FIXME note that if the block under the item should become non-solid,
	// then we need to turn off onGround to re-enable physics
	// TODO if the object has stopped moving (i.e is at rest on top of a solid
	// block and not inside a flowing block), take the object out of a
	// "physically active" list. Note that the object will have to be re-added
	// if any blocks it is adjacent to change in solidity or "flow"
	stopped := obj.updateVelocity()

	if stopped {
		// The object isn't moving, we're done
		obj.remainder = 0.0
		return
	}

	// Enforce max absolute velocity per dimension
	v.X.Constrain()
	v.Y.Constrain()
	v.Z.Constrain()

	// t0 = time at start of tick,
	// t1 = time at end of tick,
	// t = current time in tick (t0 <= t <= t1)
	var t0, t1, t TickTime

	// `Dt` and `dt` means delta-time, that is, time relative to `t`
	var nextBlockXdt, nextBlockYdt, nextBlockZdt TickTime

	var dt TickTime

	t0 = 0.0
	t1 = 1.0 + obj.remainder
	dt = 0

	var move blockAxisMove

	// Project the velocity in block space to see if we hit anything solid, and
	// stop the object's velocity component if so

	for t = t0; t < t1; t += dt {
		// How long after t0 is it that the object hits a block boundary on
		// each axis?
		nextBlockXdt = calcNextBlockDt(p.X, v.X)
		nextBlockYdt = calcNextBlockDt(p.Y, v.Y)
		nextBlockZdt = calcNextBlockDt(p.Z, v.Z)

		// In the axis of which block are we moving? In X, Y or Z axis?
		move, dt = getBlockAxisMove(nextBlockXdt, nextBlockYdt, nextBlockZdt)

		// Don't calculate beyond 1 tick of time
		if t+dt >= t1 {
			// It will be after the end of this tick when the object crosses a
			// block boundary
			dt = t1 - t
			p.ApplyVelocity(dt, v)

			// We're all done
			break
		} else {

			// Examine the block being entered
			blockLoc := obj.nextBlockToEnter(move)
			// FIXME deal better with the case where the block goes over the
			// top (Y > 128) - BlockYCoord is an int8, so it'll overflow
			if blockLoc.Y < 0 {
				break
			}

			// Is it solid?
			isSolid, isWithinChunk := blockQuery(blockLoc)
			if isSolid {
				// Collision - cancel axis movement
				switch move {
				case blockAxisMoveX:
					applyCollision(&p.X, &v.X)
				case blockAxisMoveY:
					applyCollision(&p.Y, &v.Y)
					obj.onGround = true
				case blockAxisMoveZ:
					applyCollision(&p.Z, &v.Z)
				}

				// Move the object up to the block boundary
				p.ApplyVelocity(dt, v)
			} else {
				// No collision, continue as normal
				// HACK: We add 1e-4 to dt to "break past" the block boundary,
				// otherwise we end up at rest on it in an infinite loop. dt
				// would otherwise be *approximately* sufficient to reach the
				// block boundary.
				p.ApplyVelocity(dt+1e-4, v)
				if !isWithinChunk {
					// Object has left the chunk, finish early
					break
				}
			}
		}
	}

	if p.Y < 0 {
		leftBlock = true
	}
	obj.remainder = t1 - t
	return
}

func (obj *PointObject) updateVelocity() (stopped bool) {
	v := &obj.velocity

	if !obj.onGround {
		v.Y -= gravityBlocksPerTick2 * AbsVelocityCoord(1.0+float64(obj.remainder))
	}

	stopped = true

	if v.X > -minVel && v.X < minVel {
		v.X = 0
	} else {
		v.X -= v.X / airResistance
		stopped = false
	}
	if v.Y > -minVel && v.Y < minVel {
		v.Y = 0
	} else {
		v.Y -= v.Y / airResistance
		stopped = false
	}
	if v.Z > -minVel && v.Z < minVel {
		v.Z = 0
	} else {
		v.Z -= v.Z / airResistance
		stopped = false
	}

	return
}

func (obj *PointObject) nextBlockToEnter(move blockAxisMove) *BlockXyz {
	p := &obj.position
	v := &obj.velocity

	block := p.ToBlockXyz()

	switch move {
	case blockAxisMoveX:
		if v.X > 0 {
			block.X += 1
		} else {
			block.X -= 1
		}
	case blockAxisMoveY:
		if v.Y > 0 {
			block.Y += 1
		} else {
			block.Y -= 1
		}
	case blockAxisMoveZ:
		if v.Z > 0 {
			block.Z += 1
		} else {
			block.Z -= 1
		}
	}

	return block
}

// In one dimension, calculates time taken for movement from position `p` with
// velocity `v` until intersection with a block boundary. Note that if v is
// small enough or zero then math.MaxFloat64 is returned.
func calcNextBlockDt(p AbsCoord, v AbsVelocityCoord) TickTime {
	if v > -1e-20 && v < 1e-20 {
		return math.MaxFloat64
	}

	if p < 0 {
		p = -p
		v = -v
	}

	var p_prime AbsCoord
	if v > 0 {
		p_prime = AbsCoord(math.Floor(float64(p + 1.0)))
	} else {
		p_prime = AbsCoord(math.Floor(float64(p)))
	}

	return TickTime(float64(p_prime-p) / float64(v))
}

// Given 3 time deltas, it returns the axis that the smallest was on, and the
// value of the smallest time delta. This is used to know on which axis and how
// long until the next block transition is. Only +ve numbers should be passed
// in for it to be sensible.
func getBlockAxisMove(xDt, yDt, zDt TickTime) (move blockAxisMove, dt TickTime) {
	if xDt <= yDt {
		if xDt <= zDt {
			return blockAxisMoveX, xDt
		} else {
			return blockAxisMoveZ, zDt
		}
	} else {
		if yDt <= zDt {
			return blockAxisMoveY, yDt
		} else {
			return blockAxisMoveZ, zDt
		}
	}
	return
}

func applyCollision(p *AbsCoord, v *AbsVelocityCoord) {
	if *v > 0 {
		*p = AbsCoord(math.Ceil(float64(*p)) - objBlockDistance)
	} else {
		*p = AbsCoord(math.Floor(float64(*p)) + objBlockDistance)
	}
	*v = 0
}
<commit_msg>Adding TODO to resync physical positions every so often.<commit_after>package physics

import (
	"io"
	"math"
	"os"

	"chunkymonkey/proto"
	. "chunkymonkey/types"
)

const (
	// Guestimated gravity value. Unknown how accurate this is.
	gravityBlocksPerSecond2 = 3.0

	gravityBlocksPerTick2 = gravityBlocksPerSecond2 / TicksPerSecond

	// Air resistance, as a denominator of a velocity component
	airResistance = 5

	// Min velocity component before we clamp to zero
	minVel = 0.01

	objBlockDistance = 4.25 / PixelsPerBlock
)

type blockAxisMove byte

const (
	blockAxisMoveX = blockAxisMove(iota)
	blockAxisMoveY = blockAxisMove(iota)
	blockAxisMoveZ = blockAxisMove(iota)
)

type BlockQueryFn func(*BlockXyz) (isSolid bool, isWithinChunk bool)

type PointObject struct {
	// Used in knowing what to send as client updates
	LastSentPosition AbsIntXyz
	LastSentVelocity Velocity

	// TODO Add a counter so that a full teleport is sent every so often so that
	// positions don't drift out of sync.

	// Used in physical modelling
	position  AbsXyz
	velocity  AbsVelocity
	onGround  bool
	remainder TickTime
}

func (obj *PointObject) Position() *AbsXyz {
	return &obj.position
}

func (obj *PointObject) Init(position *AbsXyz, velocity *AbsVelocity) {
	obj.LastSentPosition = *position.ToAbsIntXyz()
	obj.LastSentVelocity = *velocity.ToVelocity()
	obj.position = *position
	obj.velocity = *velocity
	obj.onGround = false
}

// Generates any packets needed to update clients as to the position and
// velocity of the object.
// It assumes that the clients have either been sent packets via this method
// before, or that the previous position/velocity sent was generated from the
// LastSentPosition and LastSentVelocity attributes.
func (obj *PointObject) SendUpdate(writer io.Writer, entityId EntityId, look *LookBytes) (err os.Error) {
	curPosition := obj.position.ToAbsIntXyz()

	dx := curPosition.X - obj.LastSentPosition.X
	dy := curPosition.Y - obj.LastSentPosition.Y
	dz := curPosition.Z - obj.LastSentPosition.Z

	if dx != 0 || dy != 0 || dz != 0 {
		if dx >= -128 && dx <= 127 && dy >= -128 && dy <= 127 && dz >= -128 && dz <= 127 {
			err = proto.WriteEntityRelMove(
				writer, entityId,
				&RelMove{
					RelMoveCoord(dx),
					RelMoveCoord(dy),
					RelMoveCoord(dz),
				},
			)
		} else {
			err = proto.WriteEntityTeleport(
				writer, entityId,
				curPosition, look)
		}
		if err != nil {
			return
		}
		obj.LastSentPosition = *curPosition
	}

	curVelocity := obj.velocity.ToVelocity()
	if curVelocity.X != obj.LastSentVelocity.X || curVelocity.Y != obj.LastSentVelocity.Y || curVelocity.Z != obj.LastSentVelocity.Z {
		if err = proto.WriteEntityVelocity(writer, entityId, curVelocity); err != nil {
			return
		}
		obj.LastSentVelocity = *curVelocity
	}

	return
}

func (obj *PointObject) Tick(blockQuery BlockQueryFn) (leftBlock bool) {
	// TODO this algorithm can probably be sped up a bit, but initially trying
	// to keep things simple and more or less correct
	// TODO flowing water movement of items

	p := &obj.position
	v := &obj.velocity

	// FIXME note that if the block under the item should become non-solid,
	// then we need to turn off onGround to re-enable physics
	// TODO if the object has stopped moving (i.e is at rest on top of a solid
	// block and not inside a flowing block), take the object out of a
	// "physically active" list. Note that the object will have to be re-added
	// if any blocks it is adjacent to change in solidity or "flow"
	stopped := obj.updateVelocity()

	if stopped {
		// The object isn't moving, we're done
		obj.remainder = 0.0
		return
	}

	// Enforce max absolute velocity per dimension
	v.X.Constrain()
	v.Y.Constrain()
	v.Z.Constrain()

	// t0 = time at start of tick,
	// t1 = time at end of tick,
	// t = current time in tick (t0 <= t <= t1)
	var t0, t1, t TickTime

	// `Dt` and `dt` means delta-time, that is, time relative to `t`
	var nextBlockXdt, nextBlockYdt, nextBlockZdt TickTime

	var dt TickTime

	t0 = 0.0
	t1 = 1.0 + obj.remainder
	dt = 0

	var move blockAxisMove

	// Project the velocity in block space to see if we hit anything solid, and
	// stop the object's velocity component if so

	for t = t0; t < t1; t += dt {
		// How long after t0 is it that the object hits a block boundary on
		// each axis?
		nextBlockXdt = calcNextBlockDt(p.X, v.X)
		nextBlockYdt = calcNextBlockDt(p.Y, v.Y)
		nextBlockZdt = calcNextBlockDt(p.Z, v.Z)

		// In the axis of which block are we moving? In X, Y or Z axis?
		move, dt = getBlockAxisMove(nextBlockXdt, nextBlockYdt, nextBlockZdt)

		// Don't calculate beyond 1 tick of time
		if t+dt >= t1 {
			// It will be after the end of this tick when the object crosses a
			// block boundary
			dt = t1 - t
			p.ApplyVelocity(dt, v)

			// We're all done
			break
		} else {

			// Examine the block being entered
			blockLoc := obj.nextBlockToEnter(move)
			// FIXME deal better with the case where the block goes over the
			// top (Y > 128) - BlockYCoord is an int8, so it'll overflow
			if blockLoc.Y < 0 {
				break
			}

			// Is it solid?
			isSolid, isWithinChunk := blockQuery(blockLoc)
			if isSolid {
				// Collision - cancel axis movement
				switch move {
				case blockAxisMoveX:
					applyCollision(&p.X, &v.X)
				case blockAxisMoveY:
					applyCollision(&p.Y, &v.Y)
					obj.onGround = true
				case blockAxisMoveZ:
					applyCollision(&p.Z, &v.Z)
				}

				// Move the object up to the block boundary
				p.ApplyVelocity(dt, v)
			} else {
				// No collision, continue as normal
				// HACK: We add 1e-4 to dt to "break past" the block boundary,
				// otherwise we end up at rest on it in an infinite loop. dt
				// would otherwise be *approximately* sufficient to reach the
				// block boundary.
				p.ApplyVelocity(dt+1e-4, v)
				if !isWithinChunk {
					// Object has left the chunk, finish early
					break
				}
			}
		}
	}

	if p.Y < 0 {
		leftBlock = true
	}
	obj.remainder = t1 - t
	return
}

func (obj *PointObject) updateVelocity() (stopped bool) {
	v := &obj.velocity

	if !obj.onGround {
		v.Y -= gravityBlocksPerTick2 * AbsVelocityCoord(1.0+float64(obj.remainder))
	}

	stopped = true

	if v.X > -minVel && v.X < minVel {
		v.X = 0
	} else {
		v.X -= v.X / airResistance
		stopped = false
	}
	if v.Y > -minVel && v.Y < minVel {
		v.Y = 0
	} else {
		v.Y -= v.Y / airResistance
		stopped = false
	}
	if v.Z > -minVel && v.Z < minVel {
		v.Z = 0
	} else {
		v.Z -= v.Z / airResistance
		stopped = false
	}

	return
}

func (obj *PointObject) nextBlockToEnter(move blockAxisMove) *BlockXyz {
	p := &obj.position
	v := &obj.velocity

	block := p.ToBlockXyz()

	switch move {
	case blockAxisMoveX:
		if v.X > 0 {
			block.X += 1
		} else {
			block.X -= 1
		}
	case blockAxisMoveY:
		if v.Y > 0 {
			block.Y += 1
		} else {
			block.Y -= 1
		}
	case blockAxisMoveZ:
		if v.Z > 0 {
			block.Z += 1
		} else {
			block.Z -= 1
		}
	}

	return block
}

// In one dimension, calculates time taken for movement from position `p` with
// velocity `v` until intersection with a block boundary. Note that if v is
// small enough or zero then math.MaxFloat64 is returned.
func calcNextBlockDt(p AbsCoord, v AbsVelocityCoord) TickTime {
	if v > -1e-20 && v < 1e-20 {
		return math.MaxFloat64
	}

	if p < 0 {
		p = -p
		v = -v
	}

	var p_prime AbsCoord
	if v > 0 {
		p_prime = AbsCoord(math.Floor(float64(p + 1.0)))
	} else {
		p_prime = AbsCoord(math.Floor(float64(p)))
	}

	return TickTime(float64(p_prime-p) / float64(v))
}

// Given 3 time deltas, it returns the axis that the smallest was on, and the
// value of the smallest time delta. This is used to know on which axis and how
// long until the next block transition is. Only +ve numbers should be passed
// in for it to be sensible.
func getBlockAxisMove(xDt, yDt, zDt TickTime) (move blockAxisMove, dt TickTime) {
	if xDt <= yDt {
		if xDt <= zDt {
			return blockAxisMoveX, xDt
		} else {
			return blockAxisMoveZ, zDt
		}
	} else {
		if yDt <= zDt {
			return blockAxisMoveY, yDt
		} else {
			return blockAxisMoveZ, zDt
		}
	}
	return
}

func applyCollision(p *AbsCoord, v *AbsVelocityCoord) {
	if *v > 0 {
		*p = AbsCoord(math.Ceil(float64(*p)) - objBlockDistance)
	} else {
		*p = AbsCoord(math.Floor(float64(*p)) + objBlockDistance)
	}
	*v = 0
}
<|endoftext|>


<commit_before>package daemon

import (
	"context"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	"github.com/datawire/dlib/dexec"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/v2/pkg/client/daemon/dns"
)

var errResolveDNotConfigured = errors.New("resolved not configured")

func (o *outbound) dnsServerWorker(c context.Context, onReady func()) error {
	err := o.tryResolveD(dgroup.WithGoroutineName(c, "/resolved"), onReady)
	if err == errResolveDNotConfigured {
		dlog.Info(c, "Unable to use systemd-resolved, falling back to local server")
		err = o.runOverridingServer(dgroup.WithGoroutineName(c, "/legacy"), onReady)
	}
	return err
}

func (o *outbound) runOverridingServer(c context.Context, onReady func()) error {
	if o.dnsIP == "" {
		dat, err := ioutil.ReadFile("/etc/resolv.conf")
		if err != nil {
			return err
		}
		for _, line := range strings.Split(string(dat), "\n") {
			if strings.HasPrefix(strings.TrimSpace(line), "nameserver") {
				fields := strings.Fields(line)
				o.dnsIP = fields[1]
				dlog.Infof(c, "Automatically set -dns=%v", o.dnsIP)
				break
			}
		}
	}
	if o.dnsIP == "" {
		return errors.New("couldn't determine dns ip from /etc/resolv.conf")
	}

	if o.fallbackIP == "" {
		if o.dnsIP == "8.8.8.8" {
			o.fallbackIP = "8.8.4.4"
		} else {
			o.fallbackIP = "8.8.8.8"
		}
		dlog.Infof(c, "Automatically set -fallback=%v", o.fallbackIP)
	}
	if o.fallbackIP == o.dnsIP {
		return errors.New("if your fallbackIP and your dnsIP are the same, you will have a dns loop")
	}

	o.setSearchPathFunc = func(c context.Context, paths []string) {
		search := make([]string, 0)
		for _, path := range paths {
			if strings.ContainsRune(path, '.') {
				search = append(search, path)
			}
		}
		search = append(search, "svc.cluster.local.", "cluster.local.", "")
		o.domainsLock.Lock()
		o.search = search
		o.domainsLock.Unlock()
	}

	listeners, err := o.dnsListeners(c)
	if err != nil {
		return err
	}
	dnsAddr, err := splitToUDPAddr(listeners[0].LocalAddr())
	if err != nil {
		return err
	}
	o.dnsRedirPort = dnsAddr.Port

	o.overridePrimaryDNS = true
	onReady()

	srv := dns.NewServer(c, listeners, o.fallbackIP+":53", o.resolve)
	dlog.Debug(c, "Starting server")
	err = srv.Run(c)
	dlog.Debug(c, "Server done")
	return err
}

// resolve looks up the given query, trying all the suffixes in the search path, and returns the
// matching IPs
func (o *outbound) resolve(query string) []string {
	if !strings.HasSuffix(query, ".") {
		query += "."
	}

	var ips []string
	o.domainsLock.RLock()
	for _, suffix := range o.search {
		name := query + suffix
		if ips = o.domains[strings.ToLower(name)]; ips != nil {
			break
		}
	}
	o.domainsLock.RUnlock()
	return shuffleIPs(ips)
}

func (o *outbound) dnsListeners(c context.Context) ([]net.PacketConn, error) {
	listeners := []net.PacketConn{o.dnsListener}
	if _, err := os.Stat("/.dockerenv"); err == nil {
		// Inside docker. Don't add docker bridge
		return listeners, nil
	}

	// This is the default docker bridge. We need to listen here because the nat logic we use to intercept
	// dns packets will divert the packet to the interface it originates from, which in the case of
	// containers is the docker bridge. Without this dns won't work from inside containers.
	output, err := dexec.CommandContext(c, "docker", "inspect", "bridge",
		"-f", "{{(index .IPAM.Config 0).Gateway}}").Output()
	if err != nil {
		dlog.Info(c, "not listening on docker bridge")
		return listeners, nil
	}

	localAddr, err := splitToUDPAddr(o.dnsListener.LocalAddr())
	if err != nil {
		return nil, err
	}

	dockerGatewayIP := net.ParseIP(strings.TrimSpace(string(output)))
	if dockerGatewayIP == nil || dockerGatewayIP.Equal(localAddr.IP) {
		return listeners, nil
	}

	// Check that the dockerGatewayIP is registered as an interface on this machine. When running WSL2 on
	// a Windows box, the gateway is managed by Windows and never visible to the Linux host and hence
	// will not be affected by the nat logic. Also, any attempt to listen to it will fail.
	found := false
	ifAddrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, ifAddr := range ifAddrs {
		_, network, err := net.ParseCIDR(ifAddr.String())
		if err != nil {
			continue
		}
		if network.Contains(dockerGatewayIP) {
			found = true
			break
		}
	}

	if !found {
		dlog.Infof(c, "docker gateway %s is not visible as a network interface", dockerGatewayIP)
		return listeners, nil
	}

	for {
		extraAddr := &net.UDPAddr{IP: dockerGatewayIP, Port: localAddr.Port}
		ls, err := net.ListenPacket("udp", extraAddr.String())
		if err == nil {
			dlog.Infof(c, "listening to docker bridge at %s", dockerGatewayIP)
			return append(listeners, ls), nil
		}

		// the extraAddr was busy, try next available port
		for localAddr.Port++; localAddr.Port <= math.MaxUint16; localAddr.Port++ {
			if ls, err = net.ListenPacket("udp", localAddr.String()); err == nil {
				if localAddr, err = splitToUDPAddr(ls.LocalAddr()); err != nil {
					ls.Close()
					return nil, err
				}
				_ = listeners[0].Close()
				listeners = []net.PacketConn{ls}
				break
			}
		}
		if localAddr.Port > math.MaxUint16 {
			return nil, fmt.Errorf("unable to find a free port for both %s and %s", localAddr.IP, extraAddr.IP)
		}
	}
}
<commit_msg>Ensure that only non-qualified names traverse the DNS search-path<commit_after>package daemon

import (
	"context"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"os"
	"strings"

	"github.com/pkg/errors"

	"github.com/datawire/dlib/dexec"
	"github.com/datawire/dlib/dgroup"
	"github.com/datawire/dlib/dlog"
	"github.com/telepresenceio/telepresence/v2/pkg/client/daemon/dns"
)

var errResolveDNotConfigured = errors.New("resolved not configured")

func (o *outbound) dnsServerWorker(c context.Context, onReady func()) error {
	err := o.tryResolveD(dgroup.WithGoroutineName(c, "/resolved"), onReady)
	if err == errResolveDNotConfigured {
		dlog.Info(c, "Unable to use systemd-resolved, falling back to local server")
		err = o.runOverridingServer(dgroup.WithGoroutineName(c, "/legacy"), onReady)
	}
	return err
}

func (o *outbound) runOverridingServer(c context.Context, onReady func()) error {
	if o.dnsIP == "" {
		dat, err := ioutil.ReadFile("/etc/resolv.conf")
		if err != nil {
			return err
		}
		for _, line := range strings.Split(string(dat), "\n") {
			if strings.HasPrefix(strings.TrimSpace(line), "nameserver") {
				fields := strings.Fields(line)
				o.dnsIP = fields[1]
				dlog.Infof(c, "Automatically set -dns=%v", o.dnsIP)
				break
			}
		}
	}
	if o.dnsIP == "" {
		return errors.New("couldn't determine dns ip from /etc/resolv.conf")
	}

	if o.fallbackIP == "" {
		if o.dnsIP == "8.8.8.8" {
			o.fallbackIP = "8.8.4.4"
		} else {
			o.fallbackIP = "8.8.8.8"
		}
		dlog.Infof(c, "Automatically set -fallback=%v", o.fallbackIP)
	}
	if o.fallbackIP == o.dnsIP {
		return errors.New("if your fallbackIP and your dnsIP are the same, you will have a dns loop")
	}

	o.setSearchPathFunc = func(c context.Context, paths []string) {
		namespaces := make(map[string]struct{})
		search := make([]string, 0)
		for _, path := range paths {
			if strings.ContainsRune(path, '.') {
				search = append(search, path)
			} else if path != "" {
				namespaces[path] = struct{}{}
			}
		}
		o.domainsLock.Lock()
		o.namespaces = namespaces
		o.search = search
		o.domainsLock.Unlock()
	}

	listeners, err := o.dnsListeners(c)
	if err != nil {
		return err
	}
	dnsAddr, err := splitToUDPAddr(listeners[0].LocalAddr())
	if err != nil {
		return err
	}
	o.dnsRedirPort = dnsAddr.Port

	o.overridePrimaryDNS = true
	onReady()

	srv := dns.NewServer(c, listeners, o.fallbackIP+":53", o.resolve)
	dlog.Debug(c, "Starting server")
	err = srv.Run(c)
	dlog.Debug(c, "Server done")
	return err
}

// resolve looks up the given query. Queries using qualified names will be dispatched to the
// resolveNoNs() function. An unqualified name query will be tried with all the suffixes in
// the search path and the IPs of the first match will be returned.
func (o *outbound) resolve(query string) []string {
	if !strings.HasSuffix(query, ".") {
		// Query must end with dot.
		return nil
	}
	if strings.Count(query, ".") > 1 {
		// More than just the ending dot, so don't use search-path
		return o.resolveNoSearch(query)
	}

	query = strings.ToLower(query)
	var ips []string
	o.domainsLock.RLock()
	for _, suffix := range o.search {
		if ips = o.domains[query+suffix]; ips != nil {
			break
		}
	}
	o.domainsLock.RUnlock()
	return shuffleIPs(ips)
}

func (o *outbound) dnsListeners(c context.Context) ([]net.PacketConn, error) {
	listeners := []net.PacketConn{o.dnsListener}
	if _, err := os.Stat("/.dockerenv"); err == nil {
		// Inside docker. Don't add docker bridge
		return listeners, nil
	}

	// This is the default docker bridge. We need to listen here because the nat logic we use to intercept
	// dns packets will divert the packet to the interface it originates from, which in the case of
	// containers is the docker bridge. Without this dns won't work from inside containers.
	output, err := dexec.CommandContext(c, "docker", "inspect", "bridge",
		"-f", "{{(index .IPAM.Config 0).Gateway}}").Output()
	if err != nil {
		dlog.Info(c, "not listening on docker bridge")
		return listeners, nil
	}

	localAddr, err := splitToUDPAddr(o.dnsListener.LocalAddr())
	if err != nil {
		return nil, err
	}

	dockerGatewayIP := net.ParseIP(strings.TrimSpace(string(output)))
	if dockerGatewayIP == nil || dockerGatewayIP.Equal(localAddr.IP) {
		return listeners, nil
	}

	// Check that the dockerGatewayIP is registered as an interface on this machine. When running WSL2 on
	// a Windows box, the gateway is managed by Windows and never visible to the Linux host and hence
	// will not be affected by the nat logic. Also, any attempt to listen to it will fail.
	found := false
	ifAddrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, ifAddr := range ifAddrs {
		_, network, err := net.ParseCIDR(ifAddr.String())
		if err != nil {
			continue
		}
		if network.Contains(dockerGatewayIP) {
			found = true
			break
		}
	}

	if !found {
		dlog.Infof(c, "docker gateway %s is not visible as a network interface", dockerGatewayIP)
		return listeners, nil
	}

	for {
		extraAddr := &net.UDPAddr{IP: dockerGatewayIP, Port: localAddr.Port}
		ls, err := net.ListenPacket("udp", extraAddr.String())
		if err == nil {
			dlog.Infof(c, "listening to docker bridge at %s", dockerGatewayIP)
			return append(listeners, ls), nil
		}

		// the extraAddr was busy, try next available port
		for localAddr.Port++; localAddr.Port <= math.MaxUint16; localAddr.Port++ {
			if ls, err = net.ListenPacket("udp", localAddr.String()); err == nil {
				if localAddr, err = splitToUDPAddr(ls.LocalAddr()); err != nil {
					ls.Close()
					return nil, err
				}
				_ = listeners[0].Close()
				listeners = []net.PacketConn{ls}
				break
			}
		}
		if localAddr.Port > math.MaxUint16 {
			return nil, fmt.Errorf("unable to find a free port for both %s and %s", localAddr.IP, extraAddr.IP)
		}
	}
}
<|endoftext|>