<commit_before>package shell

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandFsMetaSave{})
}

type commandFsMetaSave struct {
}

func (c *commandFsMetaSave) Name() string {
	return "fs.meta.save"
}

func (c *commandFsMetaSave) Help() string {
	return `save all directory and file meta data to a local file for metadata backup.

	fs.meta.save /               # save from the root
	fs.meta.save -v -o t.meta /  # save from the root, output to t.meta file.
	fs.meta.save /path/to/save   # save from the directory /path/to/save
	fs.meta.save .               # save from current directory
	fs.meta.save                 # save from current directory

	The meta data will be saved into a local <filer_host>-<port>-<time>.meta file.
	These meta data can be later loaded by fs.meta.load command,

`
}

func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
	// chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
	if err = fsMetaSaveCommand.Parse(args); err != nil {
		return nil
	}

	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
	if parseErr != nil {
		return parseErr
	}

	if *outputFileName != "" {
		fileName := *outputFileName
		if fileName == "" {
			t := time.Now()
			fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
				commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
		}

		dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if openErr != nil {
			return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
		}
		defer dst.Close()

		return doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
			sizeBuf := make([]byte, 4)
			for b := range outputChan {
				util.Uint32toBytes(sizeBuf, uint32(len(b)))
				dst.Write(sizeBuf)
				dst.Write(b)
			}
		}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
			bytes, err := proto.Marshal(entry)
			if err != nil {
				fmt.Fprintf(writer, "marshall error: %v\n", err)
				return
			}

			outputChan <- bytes
			return nil
		})
	}

	var chunksFileName = ""
	if chunksFileName != "" {

		dst, openErr := os.OpenFile(chunksFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if openErr != nil {
			return fmt.Errorf("failed to create file %s: %v", chunksFileName, openErr)
		}
		defer dst.Close()

		return doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
			for b := range outputChan {
				dst.Write(b)
			}
		}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
			for _, chunk := range entry.Entry.Chunks {
				dir := entry.Dir
				if dir == "/" {
					dir = ""
				}
				outputLine := fmt.Sprintf("%d\t%s\t%s/%s\n", chunk.Fid.FileKey, chunk.FileId, dir, entry.Entry.Name)
				outputChan <- []byte(outputLine)
			}
			return nil
		})
	}

	return err

}

func doTraverseBfsAndSaving(commandEnv *CommandEnv, writer io.Writer, path string, verbose bool, saveFn func(dst io.Writer, outputChan chan []byte), genFn func(entry *filer_pb.FullEntry, outputChan chan []byte) error) error {

	var wg sync.WaitGroup
	wg.Add(1)
	outputChan := make(chan []byte, 1024)
	go func() {
		saveFn(dst, outputChan)
		wg.Done()
	}()

	var dirCount, fileCount uint64

	err := doTraverseBfs(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {

		protoMessage := &filer_pb.FullEntry{
			Dir:   string(parentPath),
			Entry: entry,
		}

		if err := genFn(protoMessage, outputChan); err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		if entry.IsDirectory {
			atomic.AddUint64(&dirCount, 1)
		} else {
			atomic.AddUint64(&fileCount, 1)
		}

		if verbose {
			println(parentPath.Child(entry.Name))
		}

	})

	close(outputChan)

	wg.Wait()

	if err == nil {
		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
	}
	return err
}

func doTraverseBfs(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {

	K := 5

	var jobQueueWg sync.WaitGroup
	queue := util.NewQueue()
	jobQueueWg.Add(1)
	queue.Enqueue(parentPath)
	var isTerminating bool

	for i := 0; i < K; i++ {
		go func() {
			for {
				if isTerminating {
					break
				}
				t := queue.Dequeue()
				if t == nil {
					time.Sleep(329 * time.Millisecond)
					continue
				}
				dir := t.(util.FullPath)
				processErr := processOneDirectory(writer, filerClient, dir, queue, &jobQueueWg, fn)
				if processErr != nil {
					err = processErr
				}
				jobQueueWg.Done()
			}
		}()
	}
	jobQueueWg.Wait()
	isTerminating = true
	return
}

func processOneDirectory(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {

	return filer_pb.ReadDirAllEntries(filerClient, parentPath, "", func(entry *filer_pb.Entry, isLast bool) {

		fn(parentPath, entry)

		if entry.IsDirectory {
			subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
			if parentPath == "/" {
				subDir = "/" + entry.Name
			}
			jobQueueWg.Add(1)
			queue.Enqueue(util.FullPath(subDir))
		}
	})

}
<commit_msg>fix logic<commit_after>package shell

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandFsMetaSave{})
}

type commandFsMetaSave struct {
}

func (c *commandFsMetaSave) Name() string {
	return "fs.meta.save"
}

func (c *commandFsMetaSave) Help() string {
	return `save all directory and file meta data to a local file for metadata backup.

	fs.meta.save /               # save from the root
	fs.meta.save -v -o t.meta /  # save from the root, output to t.meta file.
	fs.meta.save /path/to/save   # save from the directory /path/to/save
	fs.meta.save .               # save from current directory
	fs.meta.save                 # save from current directory

	The meta data will be saved into a local <filer_host>-<port>-<time>.meta file.
	These meta data can be later loaded by fs.meta.load command,

`
}

func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
	// chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
	if err = fsMetaSaveCommand.Parse(args); err != nil {
		return nil
	}

	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
	if parseErr != nil {
		return parseErr
	}

	fileName := *outputFileName
	if fileName == "" {
		t := time.Now()
		fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
			commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	}

	dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if openErr != nil {
		return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
	}
	defer dst.Close()

	return doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
		sizeBuf := make([]byte, 4)
		for b := range outputChan {
			util.Uint32toBytes(sizeBuf, uint32(len(b)))
			dst.Write(sizeBuf)
			dst.Write(b)
		}
	}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
		bytes, err := proto.Marshal(entry)
		if err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		outputChan <- bytes
		return nil
	})

	var chunksFileName = ""
	if chunksFileName != "" {

		dst, openErr := os.OpenFile(chunksFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if openErr != nil {
			return fmt.Errorf("failed to create file %s: %v", chunksFileName, openErr)
		}
		defer dst.Close()

		return doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(dst io.Writer, outputChan chan []byte) {
			for b := range outputChan {
				dst.Write(b)
			}
		}, func(entry *filer_pb.FullEntry, outputChan chan []byte) (err error) {
			for _, chunk := range entry.Entry.Chunks {
				dir := entry.Dir
				if dir == "/" {
					dir = ""
				}
				outputLine := fmt.Sprintf("%d\t%s\t%s/%s\n", chunk.Fid.FileKey, chunk.FileId, dir, entry.Entry.Name)
				outputChan <- []byte(outputLine)
			}
			return nil
		})
	}

	return err

}

func doTraverseBfsAndSaving(commandEnv *CommandEnv, writer io.Writer, path string, verbose bool, saveFn func(dst io.Writer, outputChan chan []byte), genFn func(entry *filer_pb.FullEntry, outputChan chan []byte) error) error {

	var wg sync.WaitGroup
	wg.Add(1)
	outputChan := make(chan []byte, 1024)
	go func() {
		saveFn(dst, outputChan)
		wg.Done()
	}()

	var dirCount, fileCount uint64

	err := doTraverseBfs(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {

		protoMessage := &filer_pb.FullEntry{
			Dir:   string(parentPath),
			Entry: entry,
		}

		if err := genFn(protoMessage, outputChan); err != nil {
			fmt.Fprintf(writer, "marshall error: %v\n", err)
			return
		}

		if entry.IsDirectory {
			atomic.AddUint64(&dirCount, 1)
		} else {
			atomic.AddUint64(&fileCount, 1)
		}

		if verbose {
			println(parentPath.Child(entry.Name))
		}

	})

	close(outputChan)

	wg.Wait()

	if err == nil {
		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
	}
	return err
}

func doTraverseBfs(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {

	K := 5

	var jobQueueWg sync.WaitGroup
	queue := util.NewQueue()
	jobQueueWg.Add(1)
	queue.Enqueue(parentPath)
	var isTerminating bool

	for i := 0; i < K; i++ {
		go func() {
			for {
				if isTerminating {
					break
				}
				t := queue.Dequeue()
				if t == nil {
					time.Sleep(329 * time.Millisecond)
					continue
				}
				dir := t.(util.FullPath)
				processErr := processOneDirectory(writer, filerClient, dir, queue, &jobQueueWg, fn)
				if processErr != nil {
					err = processErr
				}
				jobQueueWg.Done()
			}
		}()
	}
	jobQueueWg.Wait()
	isTerminating = true
	return
}

func processOneDirectory(writer io.Writer, filerClient filer_pb.FilerClient, parentPath util.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup, fn func(parentPath util.FullPath, entry *filer_pb.Entry)) (err error) {

	return filer_pb.ReadDirAllEntries(filerClient, parentPath, "", func(entry *filer_pb.Entry, isLast bool) {

		fn(parentPath, entry)

		if entry.IsDirectory {
			subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
			if parentPath == "/" {
				subDir = "/" + entry.Name
			}
			jobQueueWg.Add(1)
			queue.Enqueue(util.FullPath(subDir))
		}
	})

}
<|endoftext|>


<commit_before>// +build windows

package sys

import (
	"errors"
	"syscall"
	"unsafe"
)

const (
	INFINITE uint32 = 0xFFFFFFFF
)

const (
	MAXIMUM_WAIT_OBJECTS int = 64
)

const (
	WAIT_OBJECT_0    = 0
	WAIT_ABANDONED_0 = 0x00000080
	WAIT_TIMEOUT     = 0x00000102
	WAIT_FAILED      = 0xFFFFFFFF
)

var (
	kernel32               = syscall.MustLoadDLL("kernel32.dll")
	waitForMultipleObjects = kernel32.MustFindProc("WaitForMultipleObjects")
	errTimeout             = errors.New("WaitForMultipleObjects timeout")
)

// DWORD WINAPI WaitForMultipleObjects(
//   _In_       DWORD  nCount,
//   _In_ const HANDLE *lpHandles,
//   _In_       BOOL   bWaitAll,
//   _In_       DWORD  dwMilliseconds
// );
func WaitForMultipleObjects(handles *[]syscall.Handle, waitAll bool, milliseconds uint32) (syscall.Handle, error) {
	count := len(*handles)
	ret, _, err := waitForMultipleObjects.Call(uintptr(count),
		uintptr(unsafe.Pointer(handles)), boolToUintptr(waitAll), uintptr(milliseconds))
	if err != nil {
		return syscall.InvalidHandle, err
	}
	if WAIT_OBJECT_0 <= ret && ret < WAIT_OBJECT_0+uintptr(count) {
		return (*handles)[ret-WAIT_OBJECT_0], nil
	}
	return syscall.InvalidHandle, errTimeout
}

func boolToUintptr(b bool) uintptr {
	if b {
		return 1
	} else {
		return 0
	}
}
<commit_msg>sys: Revise implementation of WaitForMultipleObjects.<commit_after>package sys

import (
	"errors"
	"unsafe"

	"golang.org/x/sys/windows"
)

const (
	INFINITE = 0xFFFFFFFF
)

const (
	WAIT_OBJECT_0    = 0
	WAIT_ABANDONED_0 = 0x00000080
	WAIT_TIMEOUT     = 0x00000102
	WAIT_FAILED      = 0xFFFFFFFF
)

var (
	kernel32               = windows.NewLazySystemDLL("kernel32.dll")
	waitForMultipleObjects = kernel32.NewProc("WaitForMultipleObjects")
	errTimeout             = errors.New("WaitForMultipleObjects timeout")
)

// WaitForMultipleObjects blocks until any of the objects is triggerd or
// timeout.
//
// DWORD WINAPI WaitForMultipleObjects(
//   _In_       DWORD  nCount,
//   _In_ const HANDLE *lpHandles,
//   _In_       BOOL   bWaitAll,
//   _In_       DWORD  dwMilliseconds
// );
func WaitForMultipleObjects(handles []windows.Handle, waitAll bool,
	timeout uint32) (trigger int, abandoned bool, err error) {

	count := uintptr(len(handles))
	ret, _, err := waitForMultipleObjects.Call(count,
		uintptr(unsafe.Pointer(&handles[0])), boolToUintptr(waitAll), uintptr(timeout))
	switch {
	case WAIT_OBJECT_0 <= ret && ret < WAIT_OBJECT_0+count:
		return int(ret - WAIT_OBJECT_0), false, nil
	case WAIT_ABANDONED_0 <= ret && ret < WAIT_ABANDONED_0+count:
		return int(ret - WAIT_ABANDONED_0), true, nil
	case ret == WAIT_TIMEOUT:
		return -1, false, errTimeout
	default:
		return -1, false, err
	}
}

func boolToUintptr(b bool) uintptr {
	if b {
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package task

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/jack-zh/ztodo/utils"
	"io"
	"os"
	"strconv"
	"time"
)

type CloudTask struct {
	Task       string `json:"task"`
	Token      string `json:"token"`
	Createtime string `json:"createtime"`
	Doingtime  string `json:"doingtime"`
	Updatetime string `json:"updatetime"`
	Donetime   string `json:"donetime"`
	Status     string `json:"status"`
}

type UserConfig struct {
	Usertoken string `json:"usertoken"`
	Pushtime  string `json:"pushtime"`
	Pushtoken string `json:"pushtoken"`
	Username  string `json:"username"`
	Password  string `json:"password"`
}

type CloudTasks struct {
	WorkFilename       string
	WorkTasks          []CloudTask
	BackupFilename     string
	BackupTasks        []CloudTask
	UserConfigFilename string
	UserConfig         UserConfig
}

func CloudNewList(workfilename string, backfilename string, userconfig_filename string) *CloudTasks {
	return &CloudTasks{workfilename, nil, backfilename, nil, userconfig_filename, UserConfig{}}
}

func (l *CloudTasks) CloudGetUserConfigByFile() error {
	fd, err := os.Open(l.UserConfigFilename)
	if err != nil {
		return nil
	}
	str := ""
	br := bufio.NewReader(fd)
	for {
		linestr, _, readerr := br.ReadLine()
		if readerr == io.EOF {
			break
		}
		if readerr != nil {
			return readerr
		}
		str += string(linestr)
	}

	userconfig := &l.UserConfig
	jsonerr := json.Unmarshal([]byte(str), userconfig)
	if jsonerr != nil {
		return jsonerr
	}
	return nil
}

func (l *CloudTasks) CloudSaveUserConfigToFile() error {
	fd, err := os.Create(l.UserConfigFilename)
	if err != nil {
		return err
	}
	defer fd.Close()

	jsonstr, err := json.Marshal(l.UserConfig)
	if err != nil {
		fmt.Println("error:", err)
	}
	fd.WriteString(string(jsonstr))
	return nil
}

func (l *CloudTasks) CloudUpdateTaskStatus(n int, upstr string) error {
	l.CloudGetAllWorkTaskByFile()
	if n > 0 && n <= len(l.WorkTasks) {
		l.WorkTasks[n-1].Status = upstr
		time_str := time.Now().Format("2006-01-02 15:04:05")
		l.WorkTasks[n-1].Updatetime = time_str
		if upstr == "Future" {
			l.WorkTasks[n-1].Doingtime = "2006-01-02 15:04:05"
			l.WorkTasks[n-1].Donetime = "2006-01-02 15:04:05"
		} else if upstr == "Done" {
			l.WorkTasks[n-1].Donetime = time_str
		} else {
			l.WorkTasks[n-1].Doingtime = time_str
		}
	} else {
		return errors.New("index out of range")
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudGetAllBackupTaskByFile() error {
	// backup task get by json str
	backupfd, backuperropenfile := os.Open(l.BackupFilename)
	backup_tasks_string := ""
	if backuperropenfile == nil {
		backupbr := bufio.NewReader(backupfd)
		for {
			backupstr, _, backuperr := backupbr.ReadLine()
			if backuperr == io.EOF {
				break
			}
			if backuperr != nil {
				return backuperr
			}
			backup_tasks_string += string(backupstr)
		}

		backuptasks := &l.BackupTasks
		backupjsonunmarshalerr := json.Unmarshal([]byte(backup_tasks_string), backuptasks)
		if backupjsonunmarshalerr != nil {
			return backupjsonunmarshalerr
		}
	}
	return nil
}

func (l *CloudTasks) CloudGetAllWorkTaskByFile() error {

	// work task get by json str
	workfd, workerropenfile := os.Open(l.WorkFilename)
	work_tasks_string := ""
	if workerropenfile == nil {

		work_br := bufio.NewReader(workfd)
		for {
			workstr, _, workerr := work_br.ReadLine()
			if workerr == io.EOF {
				break
			}
			if workerr != nil {
				return workerr
			}
			work_tasks_string += string(workstr)
		}
		worktasks := &l.WorkTasks
		workjsonunmarshalerr := json.Unmarshal([]byte(work_tasks_string), worktasks)
		if workjsonunmarshalerr != nil {

			return workjsonunmarshalerr
		}
	}
	return nil
}

func (l *CloudTasks) CloudTaskToFile() error {
	// work tasks json to file
	workfd, err := os.Create(l.WorkFilename)
	if err != nil {
		return err
	}
	defer workfd.Close()

	workjsonstr, err := json.Marshal(l.WorkTasks)
	if err != nil {
		fmt.Println("error:", err)
	}
	workfd.WriteString(string(workjsonstr))

	// backup task json to file
	backfd, err := os.Create(l.BackupFilename)
	if err != nil {
		return err
	}
	defer backfd.Close()

	backupjsonstr, err := json.Marshal(l.BackupTasks)
	if err != nil {
		fmt.Println("error:", err)
	}
	backfd.WriteString(string(backupjsonstr))

	return nil
}

func (l *CloudTasks) CloudAddTask(s string) error {
	create_time_str := time.Now().Format("2006-01-02 15:04:05")
	doing_time_str := "2006-01-02 15:04:05"
	done_time_str := "2006-01-02 15:04:05"
	status := "Future"
	task_str := s
	token, _ := utils.GenUUID()
	task := CloudTask{
		Task:       task_str,
		Token:      token,
		Createtime: create_time_str,
		Doingtime:  doing_time_str,
		Donetime:   done_time_str,
		Status:     status,
		Updatetime: create_time_str,
	}
	l.CloudGetAllWorkTaskByFile()
	l.WorkTasks = append(l.WorkTasks, task)
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudRmOneTask(n int) {
	task := l.WorkTasks[n]
	task.Updatetime = time.Now().Format("2006-01-02 15:04:05")
	l.BackupTasks = append(l.BackupTasks, task)
	l.WorkTasks = append(l.WorkTasks[:n], l.WorkTasks[n+1:]...)
}

func (l *CloudTasks) CloudRemoveTask(n int) error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	if n <= len(l.WorkTasks) && n > 0 {
		l.CloudRmOneTask(n - 1)
		return l.CloudTaskToFile()
	} else {
		return errors.New("index out of range")
	}
}

func (l *CloudTasks) CloudDoneTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Done")
}

func (l *CloudTasks) CloudDoingTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Doing")
}

func (l *CloudTasks) CloudUndoTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Future")
}

func (l *CloudTasks) CloudCleanTask() error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	for n := len(l.WorkTasks) - 1; n >= 0; n-- {
		if l.WorkTasks[n].Status == "Done" {
			l.CloudRmOneTask(n)
		}
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudClearTask() error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	for n := len(l.WorkTasks) - 1; n >= 0; n-- {
		l.CloudRmOneTask(n)
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudTasksPrint(i int) {
	if i == -1 {
		for ti := 0; ti < len(l.WorkTasks); ti++ {
			task := l.WorkTasks[ti]
			printTask(task, ti+1)
		}
	} else {
		if i <= len(l.WorkTasks) && i > 0 {
			task := l.WorkTasks[i-1]
			printTask(task, i)
		}
	}
}

func (l *CloudTasks) CloudTasksPrintVerbose(i int) {
	if i == -1 {
		for ti := 0; ti < len(l.WorkTasks); ti++ {
			task := l.WorkTasks[ti]
			printTaskVerbose(task, ti+1)
		}

	} else {
		if i <= len(l.WorkTasks) && i > 0 {
			task := l.WorkTasks[i-1]
			printTaskVerbose(task, i)
		}
	}
}

func printTask(task CloudTask, i int) {
	fmt.Printf("%-3s: [%-6s] [%s] %s\n", strconv.Itoa(i), task.Status, task.Updatetime, task.Task)
}

func printTaskVerbose(task CloudTask, i int) {
	fmt.Printf("%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n----------------------------------------\n",
		"token", task.Token,
		"num", strconv.Itoa(i),
		"task", task.Task,
		"status", task.Status,
		"create time", task.Createtime,
		"doing time", task.Doingtime,
		"done time", task.Donetime,
		"update time", task.Updatetime)
}
<commit_msg>fix print bug<commit_after>package task

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/jack-zh/ztodo/utils"
	"io"
	"os"
	"strconv"
	"time"
)

type CloudTask struct {
	Task       string `json:"task"`
	Token      string `json:"token"`
	Createtime string `json:"createtime"`
	Doingtime  string `json:"doingtime"`
	Updatetime string `json:"updatetime"`
	Donetime   string `json:"donetime"`
	Status     string `json:"status"`
}

type UserConfig struct {
	Usertoken string `json:"usertoken"`
	Pushtime  string `json:"pushtime"`
	Pushtoken string `json:"pushtoken"`
	Username  string `json:"username"`
	Password  string `json:"password"`
}

type CloudTasks struct {
	WorkFilename       string
	WorkTasks          []CloudTask
	BackupFilename     string
	BackupTasks        []CloudTask
	UserConfigFilename string
	UserConfig         UserConfig
}

func CloudNewList(workfilename string, backfilename string, userconfig_filename string) *CloudTasks {
	return &CloudTasks{workfilename, nil, backfilename, nil, userconfig_filename, UserConfig{}}
}

func (l *CloudTasks) CloudGetUserConfigByFile() error {
	fd, err := os.Open(l.UserConfigFilename)
	if err != nil {
		return nil
	}
	str := ""
	br := bufio.NewReader(fd)
	for {
		linestr, _, readerr := br.ReadLine()
		if readerr == io.EOF {
			break
		}
		if readerr != nil {
			return readerr
		}
		str += string(linestr)
	}

	userconfig := &l.UserConfig
	jsonerr := json.Unmarshal([]byte(str), userconfig)
	if jsonerr != nil {
		return jsonerr
	}
	return nil
}

func (l *CloudTasks) CloudSaveUserConfigToFile() error {
	fd, err := os.Create(l.UserConfigFilename)
	if err != nil {
		return err
	}
	defer fd.Close()

	jsonstr, err := json.Marshal(l.UserConfig)
	if err != nil {
		fmt.Println("error:", err)
	}
	fd.WriteString(string(jsonstr))
	return nil
}

func (l *CloudTasks) CloudUpdateTaskStatus(n int, upstr string) error {
	l.CloudGetAllWorkTaskByFile()
	if n > 0 && n <= len(l.WorkTasks) {
		l.WorkTasks[n-1].Status = upstr
		time_str := time.Now().Format("2006-01-02 15:04:05")
		l.WorkTasks[n-1].Updatetime = time_str
		if upstr == "Future" {
			l.WorkTasks[n-1].Doingtime = "2006-01-02 15:04:05"
			l.WorkTasks[n-1].Donetime = "2006-01-02 15:04:05"
		} else if upstr == "Done" {
			l.WorkTasks[n-1].Donetime = time_str
		} else {
			l.WorkTasks[n-1].Doingtime = time_str
		}
	} else {
		return errors.New("index out of range")
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudGetAllBackupTaskByFile() error {
	// backup task get by json str
	backupfd, backuperropenfile := os.Open(l.BackupFilename)
	backup_tasks_string := ""
	if backuperropenfile == nil {
		backupbr := bufio.NewReader(backupfd)
		for {
			backupstr, _, backuperr := backupbr.ReadLine()
			if backuperr == io.EOF {
				break
			}
			if backuperr != nil {
				return backuperr
			}
			backup_tasks_string += string(backupstr)
		}

		backuptasks := &l.BackupTasks
		backupjsonunmarshalerr := json.Unmarshal([]byte(backup_tasks_string), backuptasks)
		if backupjsonunmarshalerr != nil {
			return backupjsonunmarshalerr
		}
	}
	return nil
}

func (l *CloudTasks) CloudGetAllWorkTaskByFile() error {

	// work task get by json str
	workfd, workerropenfile := os.Open(l.WorkFilename)
	work_tasks_string := ""
	if workerropenfile == nil {

		work_br := bufio.NewReader(workfd)
		for {
			workstr, _, workerr := work_br.ReadLine()
			if workerr == io.EOF {
				break
			}
			if workerr != nil {
				return workerr
			}
			work_tasks_string += string(workstr)
		}
		worktasks := &l.WorkTasks
		workjsonunmarshalerr := json.Unmarshal([]byte(work_tasks_string), worktasks)
		if workjsonunmarshalerr != nil {

			return workjsonunmarshalerr
		}
	}
	return nil
}

func (l *CloudTasks) CloudTaskToFile() error {
	// work tasks json to file
	workfd, err := os.Create(l.WorkFilename)
	if err != nil {
		return err
	}
	defer workfd.Close()

	workjsonstr, err := json.Marshal(l.WorkTasks)
	if err != nil {
		fmt.Println("error:", err)
	}
	workfd.WriteString(string(workjsonstr))

	// backup task json to file
	backfd, err := os.Create(l.BackupFilename)
	if err != nil {
		return err
	}
	defer backfd.Close()

	backupjsonstr, err := json.Marshal(l.BackupTasks)
	if err != nil {
		fmt.Println("error:", err)
	}
	backfd.WriteString(string(backupjsonstr))

	return nil
}

func (l *CloudTasks) CloudAddTask(s string) error {
	create_time_str := time.Now().Format("2006-01-02 15:04:05")
	doing_time_str := "2006-01-02 15:04:05"
	done_time_str := "2006-01-02 15:04:05"
	status := "Future"
	task_str := s
	token, _ := utils.GenUUID()
	task := CloudTask{
		Task:       task_str,
		Token:      token,
		Createtime: create_time_str,
		Doingtime:  doing_time_str,
		Donetime:   done_time_str,
		Status:     status,
		Updatetime: create_time_str,
	}
	l.CloudGetAllWorkTaskByFile()
	l.WorkTasks = append(l.WorkTasks, task)
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudRmOneTask(n int) {
	task := l.WorkTasks[n]
	task.Updatetime = time.Now().Format("2006-01-02 15:04:05")
	l.BackupTasks = append(l.BackupTasks, task)
	l.WorkTasks = append(l.WorkTasks[:n], l.WorkTasks[n+1:]...)
}

func (l *CloudTasks) CloudRemoveTask(n int) error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	if n <= len(l.WorkTasks) && n > 0 {
		l.CloudRmOneTask(n - 1)
		return l.CloudTaskToFile()
	} else {
		return errors.New("index out of range")
	}
}

func (l *CloudTasks) CloudDoneTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Done")
}

func (l *CloudTasks) CloudDoingTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Doing")
}

func (l *CloudTasks) CloudUndoTask(n int) error {
	return l.CloudUpdateTaskStatus(n, "Future")
}

func (l *CloudTasks) CloudCleanTask() error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	for n := len(l.WorkTasks) - 1; n >= 0; n-- {
		if l.WorkTasks[n].Status == "Done" {
			l.CloudRmOneTask(n)
		}
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudClearTask() error {
	l.CloudGetAllWorkTaskByFile()
	l.CloudGetAllBackupTaskByFile()
	for n := len(l.WorkTasks) - 1; n >= 0; n-- {
		l.CloudRmOneTask(n)
	}
	return l.CloudTaskToFile()
}

func (l *CloudTasks) CloudTasksPrint(i int) {
	if i == -1 {
		for ti := 0; ti < len(l.WorkTasks); ti++ {
			task := l.WorkTasks[ti]
			printTask(task, ti+1)
		}
	} else {
		if i <= len(l.WorkTasks) && i > 0 {
			task := l.WorkTasks[i-1]
			printTask(task, i)
		}
	}
}

func (l *CloudTasks) CloudTasksPrintVerbose(i int) {
	if i == -1 {
		for ti := 0; ti < len(l.WorkTasks); ti++ {
			task := l.WorkTasks[ti]
			printTaskVerbose(task, ti+1)
		}

	} else {
		if i <= len(l.WorkTasks) && i > 0 {
			task := l.WorkTasks[i-1]
			printTaskVerbose(task, i)
		}
	}
}

func printTask(task CloudTask, i int) {
	fmt.Printf("%-3s: [%-6s] [%s] %s\n", strconv.Itoa(i), task.Status, task.Updatetime, task.Task)
}

func printTaskVerbose(task CloudTask, i int) {
	if task.Doingtime != "2006-01-02 15:04:05" && task.Donetime != "2006-01-02 15:04:05" {
		fmt.Printf("%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n----------------------------------------\n",
			"token", task.Token,
			"num", strconv.Itoa(i),
			"task", task.Task,
			"status", task.Status,
			"create time", task.Createtime,
			"doing time", task.Doingtime,
			"done time", task.Donetime,
			"update time", task.Updatetime)
	} else if task.Doingtime == "2006-01-02 15:04:05" && task.Donetime != "2006-01-02 15:04:05" {
		fmt.Printf("%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n----------------------------------------\n",
			"token", task.Token,
			"num", strconv.Itoa(i),
			"task", task.Task,
			"status", task.Status,
			"create time", task.Createtime,
			"done time", task.Donetime,
			"update time", task.Updatetime)
	} else if task.Doingtime != "2006-01-02 15:04:05" && task.Donetime == "2006-01-02 15:04:05" {
		fmt.Printf("%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n----------------------------------------\n",
			"token", task.Token,
			"num", strconv.Itoa(i),
			"task", task.Task,
			"status", task.Status,
			"create time", task.Createtime,
			"doing time", task.Doingtime,
			"update time", task.Updatetime)
	} else {
		fmt.Printf("%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n%13s:  %s\n----------------------------------------\n",
			"token", task.Token,
			"num", strconv.Itoa(i),
			"task", task.Task,
			"status", task.Status,
			"create time", task.Createtime,
			"update time", task.Updatetime)
	}
}
<|endoftext|>


<commit_before>// Based on ssh/terminal:
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package logrus

import "syscall"

const ioctlReadTermios = syscall.TCGETS

type Termios syscall.Termios
<commit_msg>switch terminal_linux to x/sys/unix from syscall<commit_after>// Based on ssh/terminal:
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

package logrus

import "golang.org/x/sys/unix"

const ioctlReadTermios = unix.TCGETS

type Termios syscall.Termios
<|endoftext|>


<commit_before>package terraform

import (
	"bufio"
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strings"
)

// DiffChangeType is an enum with the kind of changes a diff has planned.
type DiffChangeType byte

const (
	DiffInvalid DiffChangeType = iota
	DiffNone
	DiffCreate
	DiffUpdate
	DiffDestroy
	DiffDestroyCreate
)

// Diff trackes the changes that are necessary to apply a configuration
// to an existing infrastructure.
type Diff struct {
	// Modules contains all the modules that have a diff
	Modules []*ModuleDiff
}

// AddModule adds the module with the given path to the diff.
//
// This should be the preferred method to add module diffs since it
// allows us to optimize lookups later as well as control sorting.
func (d *Diff) AddModule(path []string) *ModuleDiff {
	m := &ModuleDiff{Path: path}
	m.init()
	d.Modules = append(d.Modules, m)
	return m
}

// ModuleByPath is used to lookup the module diff for the given path.
// This should be the preferred lookup mechanism as it allows for future
// lookup optimizations.
func (d *Diff) ModuleByPath(path []string) *ModuleDiff {
	if d == nil {
		return nil
	}
	for _, mod := range d.Modules {
		if mod.Path == nil {
			panic("missing module path")
		}
		if reflect.DeepEqual(mod.Path, path) {
			return mod
		}
	}
	return nil
}

// RootModule returns the ModuleState for the root module
func (d *Diff) RootModule() *ModuleDiff {
	root := d.ModuleByPath(rootModulePath)
	if root == nil {
		panic("missing root module")
	}
	return root
}

// Empty returns true if the diff has no changes.
func (d *Diff) Empty() bool {
	for _, m := range d.Modules {
		if !m.Empty() {
			return false
		}
	}

	return true
}

func (d *Diff) String() string {
	var buf bytes.Buffer

	keys := make([]string, 0, len(d.Modules))
	lookup := make(map[string]*ModuleDiff)
	for _, m := range d.Modules {
		key := fmt.Sprintf("module.%s", strings.Join(m.Path[1:], "."))
		keys = append(keys, key)
		lookup[key] = m
	}
	sort.Strings(keys)

	for _, key := range keys {
		m := lookup[key]
		mStr := m.String()

		// If we're the root module, we just write the output directly.
		if reflect.DeepEqual(m.Path, rootModulePath) {
			buf.WriteString(mStr + "\n")
			continue
		}

		buf.WriteString(fmt.Sprintf("%s:\n", key))

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			buf.WriteString(fmt.Sprintf("  %s\n", s.Text()))
		}
	}

	return strings.TrimSpace(buf.String())
}

func (d *Diff) init() {
	if d.Modules == nil {
		rootDiff := &ModuleDiff{Path: rootModulePath}
		d.Modules = []*ModuleDiff{rootDiff}
	}
	for _, m := range d.Modules {
		m.init()
	}
}

// ModuleDiff tracks the differences between resources to apply within
// a single module.
type ModuleDiff struct {
	Path      []string
	Resources map[string]*InstanceDiff
	Destroy   bool // Set only by the destroy plan
}

func (d *ModuleDiff) init() {
	if d.Resources == nil {
		d.Resources = make(map[string]*InstanceDiff)
	}
	for _, r := range d.Resources {
		r.init()
	}
}

// ChangeType returns the type of changes that the diff for this
// module includes.
//
// At a module level, this will only be DiffNone, DiffUpdate, DiffDestroy, or
// DiffCreate. If an instance within the module has a DiffDestroyCreate
// then this will register as a DiffCreate for a module.
func (d *ModuleDiff) ChangeType() DiffChangeType {
	result := DiffNone
	for _, r := range d.Resources {
		change := r.ChangeType()
		switch change {
		case DiffCreate, DiffDestroy:
			if result == DiffNone {
				result = change
			}
		case DiffDestroyCreate, DiffUpdate:
			result = DiffUpdate
		}
	}

	return result
}

// Empty returns true if the diff has no changes within this module.
func (d *ModuleDiff) Empty() bool {
	if len(d.Resources) == 0 {
		return true
	}

	for _, rd := range d.Resources {
		if !rd.Empty() {
			return false
		}
	}

	return true
}

// Instances returns the instance diffs for the id given. This can return
// multiple instance diffs if there are counts within the resource.
func (d *ModuleDiff) Instances(id string) []*InstanceDiff {
	var result []*InstanceDiff
	for k, diff := range d.Resources {
		if k == id || strings.HasPrefix(k, id+".") {
			if !diff.Empty() {
				result = append(result, diff)
			}
		}
	}

	return result
}

// IsRoot says whether or not this module diff is for the root module.
func (d *ModuleDiff) IsRoot() bool {
	return reflect.DeepEqual(d.Path, rootModulePath)
}

// String outputs the diff in a long but command-line friendly output
// format that users can read to quickly inspect a diff.
func (d *ModuleDiff) String() string {
	var buf bytes.Buffer

	if d.Destroy {
		buf.WriteString("DESTROY MODULE\n")
	}

	names := make([]string, 0, len(d.Resources))
	for name, _ := range d.Resources {
		names = append(names, name)
	}
	sort.Strings(names)

	for _, name := range names {
		rdiff := d.Resources[name]

		crud := "UPDATE"
		switch {
		case rdiff.RequiresNew() && (rdiff.Destroy || rdiff.DestroyTainted):
			crud = "DESTROY/CREATE"
		case rdiff.Destroy:
			crud = "DESTROY"
		case rdiff.RequiresNew():
			crud = "CREATE"
		}

		buf.WriteString(fmt.Sprintf(
			"%s: %s\n",
			crud,
			name))

		keyLen := 0
		keys := make([]string, 0, len(rdiff.Attributes))
		for key, _ := range rdiff.Attributes {
			if key == "id" {
				continue
			}

			keys = append(keys, key)
			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
		sort.Strings(keys)

		for _, attrK := range keys {
			attrDiff := rdiff.Attributes[attrK]

			v := attrDiff.New
			u := attrDiff.Old
			if attrDiff.NewComputed {
				v = "<computed>"
			}

			if attrDiff.Sensitive {
				u = "<sensitive>"
				v = "<sensitive>"
			}

			updateMsg := ""
			if attrDiff.RequiresNew {
				updateMsg = " (forces new resource)"
			} else if attrDiff.Sensitive {
				updateMsg = " (attribute changed)"
			}

			buf.WriteString(fmt.Sprintf(
				"  %s:%s %#v => %#v%s\n",
				attrK,
				strings.Repeat(" ", keyLen-len(attrK)),
				u,
				v,
				updateMsg))
		}
	}

	return buf.String()
}

// InstanceDiff is the diff of a resource from some state to another.
type InstanceDiff struct {
	Attributes     map[string]*ResourceAttrDiff
	Destroy        bool
	DestroyTainted bool
}

// ResourceAttrDiff is the diff of a single attribute of a resource.
type ResourceAttrDiff struct {
	Old         string      // Old Value
	New         string      // New Value
	NewComputed bool        // True if new value is computed (unknown currently)
	NewRemoved  bool        // True if this attribute is being removed
	NewExtra    interface{} // Extra information for the provider
	RequiresNew bool        // True if change requires new resource
	Sensitive   bool        // True if the data should not be displayed in UI output
	Type        DiffAttrType
}

// Empty returns true if the diff for this attr is neutral
func (d *ResourceAttrDiff) Empty() bool {
	return d.Old == d.New && !d.NewComputed && !d.NewRemoved
}

func (d *ResourceAttrDiff) GoString() string {
	return fmt.Sprintf("*%#v", *d)
}

// DiffAttrType is an enum type that says whether a resource attribute
// diff is an input attribute (comes from the configuration) or an
// output attribute (comes as a result of applying the configuration). An
// example input would be "ami" for AWS and an example output would be
// "private_ip".
type DiffAttrType byte

const (
	DiffAttrUnknown DiffAttrType = iota
	DiffAttrInput
	DiffAttrOutput
)

func (d *InstanceDiff) init() {
	if d.Attributes == nil {
		d.Attributes = make(map[string]*ResourceAttrDiff)
	}
}

// ChangeType returns the DiffChangeType represented by the diff
// for this single instance.
func (d *InstanceDiff) ChangeType() DiffChangeType {
	if d.Empty() {
		return DiffNone
	}

	if d.RequiresNew() && (d.Destroy || d.DestroyTainted) {
		return DiffDestroyCreate
	}

	if d.Destroy {
		return DiffDestroy
	}

	if d.RequiresNew() {
		return DiffCreate
	}

	return DiffUpdate
}

// Empty returns true if this diff encapsulates no changes.
func (d *InstanceDiff) Empty() bool {
	if d == nil {
		return true
	}

	return !d.Destroy && len(d.Attributes) == 0
}

func (d *InstanceDiff) GoString() string {
	return fmt.Sprintf("*%#v", *d)
}

// RequiresNew returns true if the diff requires the creation of a new
// resource (implying the destruction of the old).
func (d *InstanceDiff) RequiresNew() bool {
	if d == nil {
		return false
	}

	if d.DestroyTainted {
		return true
	}

	for _, rd := range d.Attributes {
		if rd != nil && rd.RequiresNew {
			return true
		}
	}

	return false
}

// Same checks whether or not two InstanceDiff's are the "same". When
// we say "same", it is not necessarily exactly equal. Instead, it is
// just checking that the same attributes are changing, a destroy
// isn't suddenly happening, etc.
func (d *InstanceDiff) Same(d2 *InstanceDiff) (bool, string) {
	if d == nil && d2 == nil {
		return true, ""
	} else if d == nil || d2 == nil {
		return false, "both nil"
	}

	if d.Destroy != d2.Destroy {
		return false, fmt.Sprintf(
			"diff: Destroy; old: %t, new: %t", d.Destroy, d2.Destroy)
	}
	if d.RequiresNew() != d2.RequiresNew() {
		return false, fmt.Sprintf(
			"diff RequiresNew; old: %t, new: %t", d.RequiresNew(), d2.RequiresNew())
	}

	// Go through the old diff and make sure the new diff has all the
	// same attributes. To start, build up the check map to be all the keys.
	checkOld := make(map[string]struct{})
	checkNew := make(map[string]struct{})
	for k, _ := range d.Attributes {
		checkOld[k] = struct{}{}
	}
	for k, _ := range d2.Attributes {
		checkNew[k] = struct{}{}
	}

	// Make an ordered list so we are sure the approximated hashes are left
	// to process at the end of the loop
	keys := make([]string, 0, len(d.Attributes))
	for k, _ := range d.Attributes {
		keys = append(keys, k)
	}
	sort.StringSlice(keys).Sort()

	for _, k := range keys {
		diffOld := d.Attributes[k]

		if _, ok := checkOld[k]; !ok {
			// We're not checking this key for whatever reason (see where
			// check is modified).
			continue
		}

		// Remove this key since we'll never hit it again
		delete(checkOld, k)
		delete(checkNew, k)

		_, ok := d2.Attributes[k]
		if !ok {
			// If there's no new attribute, and the old diff expected the attribute
			// to be removed, that's just fine.
			if diffOld.NewRemoved {
				continue
			}

			// No exact match, but maybe this is a set containing computed
			// values. So check if there is an approximate hash in the key
			// and if so, try to match the key.
			if strings.Contains(k, "~") {
				// TODO (SvH): There should be a better way to do this...
				parts := strings.Split(k, ".")
				parts2 := strings.Split(k, ".")
				re := regexp.MustCompile(`^~\d+$`)
				for i, part := range parts {
					if re.MatchString(part) {
						parts2[i] = `\d+`
					}
				}
				re, err := regexp.Compile("^" + strings.Join(parts2, `\.`) + "$")
				if err != nil {
					return false, fmt.Sprintf("regexp failed to compile; err: %#v", err)
				}
				for k2, _ := range checkNew {
					if re.MatchString(k2) {
						delete(checkNew, k2)

						if diffOld.NewComputed && strings.HasSuffix(k, ".#") {
							// This is a computed list or set, so remove any keys with this
							// prefix from the check list.
							prefix := k2[:len(k2)-1]
							for k2, _ := range checkNew {
								if strings.HasPrefix(k2, prefix) {
									delete(checkNew, k2)
								}
							}
						}
						ok = true
						break
					}
				}
			}

			// This is a little tricky, but when a diff contains a computed list
			// or set that can only be interpolated after the apply command has
			// created the dependent resources, it could turn out that the result
			// is actually the same as the existing state which would remove the
			// key from the diff.
			if diffOld.NewComputed && strings.HasSuffix(k, ".#") {
				ok = true
			}

			// Similarly, in a RequiresNew scenario, a list that shows up in the plan
			// diff can disappear from the apply diff, which is calculated from an
			// empty state.
			if d.RequiresNew() && strings.HasSuffix(k, ".#") {
				ok = true
			}

			if !ok {
				return false, fmt.Sprintf("attribute mismatch: %s", k)
			}
		}

		if diffOld.NewComputed && strings.HasSuffix(k, ".#") {
			// This is a computed list or set, so remove any keys with this
			// prefix from the check list.
			kprefix := k[:len(k)-1]
			for k2, _ := range checkOld {
				if strings.HasPrefix(k2, kprefix) {
					delete(checkOld, k2)
				}
			}
			for k2, _ := range checkNew {
				if strings.HasPrefix(k2, kprefix) {
					delete(checkNew, k2)
				}
			}
		}

		// TODO: check for the same value if not computed
	}

	// Check for leftover attributes
	if len(checkNew) > 0 {
		extras := make([]string, 0, len(checkNew))
		for attr, _ := range checkNew {
			extras = append(extras, attr)
		}
		return false,
			fmt.Sprintf("extra attributes: %s", strings.Join(extras, ", "))
	}

	return true, ""
}
<commit_msg>Don't check any parts of a computed hash in Same<commit_after>package terraform

import (
	"bufio"
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strings"
)

// DiffChangeType is an enum with the kind of changes a diff has planned.
type DiffChangeType byte

const (
	DiffInvalid DiffChangeType = iota
	DiffNone
	DiffCreate
	DiffUpdate
	DiffDestroy
	DiffDestroyCreate
)

// Diff trackes the changes that are necessary to apply a configuration
// to an existing infrastructure.
type Diff struct {
	// Modules contains all the modules that have a diff
	Modules []*ModuleDiff
}

// AddModule adds the module with the given path to the diff.
//
// This should be the preferred method to add module diffs since it
// allows us to optimize lookups later as well as control sorting.
func (d *Diff) AddModule(path []string) *ModuleDiff {
	m := &ModuleDiff{Path: path}
	m.init()
	d.Modules = append(d.Modules, m)
	return m
}

// ModuleByPath is used to lookup the module diff for the given path.
// This should be the preferred lookup mechanism as it allows for future
// lookup optimizations.
func (d *Diff) ModuleByPath(path []string) *ModuleDiff {
	if d == nil {
		return nil
	}
	for _, mod := range d.Modules {
		if mod.Path == nil {
			panic("missing module path")
		}
		if reflect.DeepEqual(mod.Path, path) {
			return mod
		}
	}
	return nil
}

// RootModule returns the ModuleState for the root module
func (d *Diff) RootModule() *ModuleDiff {
	root := d.ModuleByPath(rootModulePath)
	if root == nil {
		panic("missing root module")
	}
	return root
}

// Empty returns true if the diff has no changes.
func (d *Diff) Empty() bool {
	for _, m := range d.Modules {
		if !m.Empty() {
			return false
		}
	}

	return true
}

func (d *Diff) String() string {
	var buf bytes.Buffer

	keys := make([]string, 0, len(d.Modules))
	lookup := make(map[string]*ModuleDiff)
	for _, m := range d.Modules {
		key := fmt.Sprintf("module.%s", strings.Join(m.Path[1:], "."))
		keys = append(keys, key)
		lookup[key] = m
	}
	sort.Strings(keys)

	for _, key := range keys {
		m := lookup[key]
		mStr := m.String()

		// If we're the root module, we just write the output directly.
		if reflect.DeepEqual(m.Path, rootModulePath) {
			buf.WriteString(mStr + "\n")
			continue
		}

		buf.WriteString(fmt.Sprintf("%s:\n", key))

		s := bufio.NewScanner(strings.NewReader(mStr))
		for s.Scan() {
			buf.WriteString(fmt.Sprintf("  %s\n", s.Text()))
		}
	}

	return strings.TrimSpace(buf.String())
}

func (d *Diff) init() {
	if d.Modules == nil {
		rootDiff := &ModuleDiff{Path: rootModulePath}
		d.Modules = []*ModuleDiff{rootDiff}
	}
	for _, m := range d.Modules {
		m.init()
	}
}

// ModuleDiff tracks the differences between resources to apply within
// a single module.
type ModuleDiff struct {
	Path      []string
	Resources map[string]*InstanceDiff
	Destroy   bool // Set only by the destroy plan
}

func (d *ModuleDiff) init() {
	if d.Resources == nil {
		d.Resources = make(map[string]*InstanceDiff)
	}
	for _, r := range d.Resources {
		r.init()
	}
}

// ChangeType returns the type of changes that the diff for this
// module includes.
//
// At a module level, this will only be DiffNone, DiffUpdate, DiffDestroy, or
// DiffCreate. If an instance within the module has a DiffDestroyCreate
// then this will register as a DiffCreate for a module.
func (d *ModuleDiff) ChangeType() DiffChangeType {
	result := DiffNone
	for _, r := range d.Resources {
		change := r.ChangeType()
		switch change {
		case DiffCreate, DiffDestroy:
			if result == DiffNone {
				result = change
			}
		case DiffDestroyCreate, DiffUpdate:
			result = DiffUpdate
		}
	}

	return result
}

// Empty returns true if the diff has no changes within this module.
func (d *ModuleDiff) Empty() bool {
	if len(d.Resources) == 0 {
		return true
	}

	for _, rd := range d.Resources {
		if !rd.Empty() {
			return false
		}
	}

	return true
}

// Instances returns the instance diffs for the id given. This can return
// multiple instance diffs if there are counts within the resource.
func (d *ModuleDiff) Instances(id string) []*InstanceDiff {
	var result []*InstanceDiff
	for k, diff := range d.Resources {
		if k == id || strings.HasPrefix(k, id+".") {
			if !diff.Empty() {
				result = append(result, diff)
			}
		}
	}

	return result
}

// IsRoot says whether or not this module diff is for the root module.
func (d *ModuleDiff) IsRoot() bool {
	return reflect.DeepEqual(d.Path, rootModulePath)
}

// String outputs the diff in a long but command-line friendly output
// format that users can read to quickly inspect a diff.
func (d *ModuleDiff) String() string {
	var buf bytes.Buffer

	if d.Destroy {
		buf.WriteString("DESTROY MODULE\n")
	}

	names := make([]string, 0, len(d.Resources))
	for name, _ := range d.Resources {
		names = append(names, name)
	}
	sort.Strings(names)

	for _, name := range names {
		rdiff := d.Resources[name]

		crud := "UPDATE"
		switch {
		case rdiff.RequiresNew() && (rdiff.Destroy || rdiff.DestroyTainted):
			crud = "DESTROY/CREATE"
		case rdiff.Destroy:
			crud = "DESTROY"
		case rdiff.RequiresNew():
			crud = "CREATE"
		}

		buf.WriteString(fmt.Sprintf(
			"%s: %s\n",
			crud,
			name))

		keyLen := 0
		keys := make([]string, 0, len(rdiff.Attributes))
		for key, _ := range rdiff.Attributes {
			if key == "id" {
				continue
			}

			keys = append(keys, key)
			if len(key) > keyLen {
				keyLen = len(key)
			}
		}
		sort.Strings(keys)

		for _, attrK := range keys {
			attrDiff := rdiff.Attributes[attrK]

			v := attrDiff.New
			u := attrDiff.Old
			if attrDiff.NewComputed {
				v = "<computed>"
			}

			if attrDiff.Sensitive {
				u = "<sensitive>"
				v = "<sensitive>"
			}

			updateMsg := ""
			if attrDiff.RequiresNew {
				updateMsg = " (forces new resource)"
			} else if attrDiff.Sensitive {
				updateMsg = " (attribute changed)"
			}

			buf.WriteString(fmt.Sprintf(
				"  %s:%s %#v => %#v%s\n",
				attrK,
				strings.Repeat(" ", keyLen-len(attrK)),
				u,
				v,
				updateMsg))
		}
	}

	return buf.String()
}

// InstanceDiff is the diff of a resource from some state to another.
type InstanceDiff struct {
	Attributes     map[string]*ResourceAttrDiff
	Destroy        bool
	DestroyTainted bool
}

// ResourceAttrDiff is the diff of a single attribute of a resource.
type ResourceAttrDiff struct {
	Old         string      // Old Value
	New         string      // New Value
	NewComputed bool        // True if new value is computed (unknown currently)
	NewRemoved  bool        // True if this attribute is being removed
	NewExtra    interface{} // Extra information for the provider
	RequiresNew bool        // True if change requires new resource
	Sensitive   bool        // True if the data should not be displayed in UI output
	Type        DiffAttrType
}

// Empty returns true if the diff for this attr is neutral
func (d *ResourceAttrDiff) Empty() bool {
	return d.Old == d.New && !d.NewComputed && !d.NewRemoved
}

func (d *ResourceAttrDiff) GoString() string {
	return fmt.Sprintf("*%#v", *d)
}

// DiffAttrType is an enum type that says whether a resource attribute
// diff is an input attribute (comes from the configuration) or an
// output attribute (comes as a result of applying the configuration). An
// example input would be "ami" for AWS and an example output would be
// "private_ip".
type DiffAttrType byte

const (
	DiffAttrUnknown DiffAttrType = iota
	DiffAttrInput
	DiffAttrOutput
)

func (d *InstanceDiff) init() {
	if d.Attributes == nil {
		d.Attributes = make(map[string]*ResourceAttrDiff)
	}
}

// ChangeType returns the DiffChangeType represented by the diff
// for this single instance.
func (d *InstanceDiff) ChangeType() DiffChangeType {
	if d.Empty() {
		return DiffNone
	}

	if d.RequiresNew() && (d.Destroy || d.DestroyTainted) {
		return DiffDestroyCreate
	}

	if d.Destroy {
		return DiffDestroy
	}

	if d.RequiresNew() {
		return DiffCreate
	}

	return DiffUpdate
}

// Empty returns true if this diff encapsulates no changes.
func (d *InstanceDiff) Empty() bool {
	if d == nil {
		return true
	}

	return !d.Destroy && len(d.Attributes) == 0
}

func (d *InstanceDiff) GoString() string {
	return fmt.Sprintf("*%#v", *d)
}

// RequiresNew returns true if the diff requires the creation of a new
// resource (implying the destruction of the old).
func (d *InstanceDiff) RequiresNew() bool {
	if d == nil {
		return false
	}

	if d.DestroyTainted {
		return true
	}

	for _, rd := range d.Attributes {
		if rd != nil && rd.RequiresNew {
			return true
		}
	}

	return false
}

// Same checks whether or not two InstanceDiff's are the "same". When
// we say "same", it is not necessarily exactly equal. Instead, it is
// just checking that the same attributes are changing, a destroy
// isn't suddenly happening, etc.
func (d *InstanceDiff) Same(d2 *InstanceDiff) (bool, string) {
	if d == nil && d2 == nil {
		return true, ""
	} else if d == nil || d2 == nil {
		return false, "both nil"
	}

	if d.Destroy != d2.Destroy {
		return false, fmt.Sprintf(
			"diff: Destroy; old: %t, new: %t", d.Destroy, d2.Destroy)
	}
	if d.RequiresNew() != d2.RequiresNew() {
		return false, fmt.Sprintf(
			"diff RequiresNew; old: %t, new: %t", d.RequiresNew(), d2.RequiresNew())
	}

	// Go through the old diff and make sure the new diff has all the
	// same attributes. To start, build up the check map to be all the keys.
	checkOld := make(map[string]struct{})
	checkNew := make(map[string]struct{})
	for k, _ := range d.Attributes {
		checkOld[k] = struct{}{}
	}
	for k, _ := range d2.Attributes {
		checkNew[k] = struct{}{}
	}

	// Make an ordered list so we are sure the approximated hashes are left
	// to process at the end of the loop
	keys := make([]string, 0, len(d.Attributes))
	for k, _ := range d.Attributes {
		keys = append(keys, k)
	}
	sort.StringSlice(keys).Sort()

	for _, k := range keys {
		diffOld := d.Attributes[k]

		if _, ok := checkOld[k]; !ok {
			// We're not checking this key for whatever reason (see where
			// check is modified).
			continue
		}

		// Remove this key since we'll never hit it again
		delete(checkOld, k)
		delete(checkNew, k)

		_, ok := d2.Attributes[k]
		if !ok {
			// If there's no new attribute, and the old diff expected the attribute
			// to be removed, that's just fine.
			if diffOld.NewRemoved {
				continue
			}

			// No exact match, but maybe this is a set containing computed
			// values. So check if there is an approximate hash in the key
			// and if so, try to match the key.
			if strings.Contains(k, "~") {
				parts := strings.Split(k, ".")
				parts2 := append([]string(nil), parts...)

				re := regexp.MustCompile(`^~\d+$`)
				for i, part := range parts {
					if re.MatchString(part) {
						// we're going to consider this the base of a
						// computed hash, and remove all longer matching fields
						ok = true

						parts2[i] = `\d+`
						parts2 = parts2[:i+1]
						break
					}
				}

				re, err := regexp.Compile("^" + strings.Join(parts2, `\.`))
				if err != nil {
					return false, fmt.Sprintf("regexp failed to compile; err: %#v", err)
				}

				for k2, _ := range checkNew {
					if re.MatchString(k2) {
						delete(checkNew, k2)
					}
				}
			}

			// This is a little tricky, but when a diff contains a computed list
			// or set that can only be interpolated after the apply command has
			// created the dependent resources, it could turn out that the result
			// is actually the same as the existing state which would remove the
			// key from the diff.
			if diffOld.NewComputed && strings.HasSuffix(k, ".#") {
				ok = true
			}

			// Similarly, in a RequiresNew scenario, a list that shows up in the plan
			// diff can disappear from the apply diff, which is calculated from an
			// empty state.
			if d.RequiresNew() && strings.HasSuffix(k, ".#") {
				ok = true
			}

			if !ok {
				return false, fmt.Sprintf("attribute mismatch: %s", k)
			}
		}

		if diffOld.NewComputed && strings.HasSuffix(k, ".#") {
			// This is a computed list or set, so remove any keys with this
			// prefix from the check list.
			kprefix := k[:len(k)-1]
			for k2, _ := range checkOld {
				if strings.HasPrefix(k2, kprefix) {
					delete(checkOld, k2)
				}
			}
			for k2, _ := range checkNew {
				if strings.HasPrefix(k2, kprefix) {
					delete(checkNew, k2)
				}
			}
		}

		// TODO: check for the same value if not computed
	}

	// Check for leftover attributes
	if len(checkNew) > 0 {
		extras := make([]string, 0, len(checkNew))
		for attr, _ := range checkNew {
			extras = append(extras, attr)
		}
		return false,
			fmt.Sprintf("extra attributes: %s", strings.Join(extras, ", "))
	}

	return true, ""
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package test

import (
	"testing"

	"github.com/couchbaselabs/tuqtng"
	"github.com/couchbaselabs/tuqtng/network"
)

func TestMain(t *testing.T) {
	qc := make(network.QueryChannel)
	go main.Main("dir:.", "json", qc)
	close(qc)
}
<commit_msg>TestSyntaxErr() using MockResponse<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package test

import (
	"testing"

	"github.com/couchbaselabs/tuqtng"
	"github.com/couchbaselabs/tuqtng/network"
	"github.com/couchbaselabs/tuqtng/query"
)

type MockResponse struct {
	err     error
	results []query.Value
	done    chan bool
}

func (this *MockResponse) SendError(err error) {
	this.err = err
	close(this.done)
}

func (this *MockResponse) SendResult(val query.Value) {
	this.results = append(this.results, val)
}

func (this *MockResponse) NoMoreResults() {
	close(this.done)
}

func run(qc network.QueryChannel, q string) ([]query.Value, error) {
	mr := &MockResponse{results: []query.Value{}, done: make(chan bool)}
	query := network.Query{
		Request:  network.UNQLStringQueryRequest{QueryString: q},
		Response: mr,
	}
	qc <- query
	<-mr.done
	return mr.results, mr.err
}

func start() network.QueryChannel {
	qc := make(network.QueryChannel)
	go main.Main("dir:.", "json", qc)
	return qc
}

func TestMainClose(t *testing.T) {
	qc := start()
	close(qc)
}

func TestSyntaxErr(t *testing.T) {
	qc := start()
	defer close(qc)

	r, err := run(qc, "this is a bad query")
	if err == nil || len(r) != 0 {
		t.Errorf("expected err")
	}
	r, err = run(qc, "") // empty string query
	if err == nil || len(r) != 0 {
		t.Errorf("expected err")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/tools/clientcmd"
)

type Config struct {
	KubeConfig  string
	KubeContext string
	Kubectl     string

	// If Cleanup is true, addons will be cleaned up both before and after provisioning
	Cleanup bool

	// RepoRoot is used as the base path for any parts of the framework that
	// require access to repo files, such as Helm charts and test fixtures.
	RepoRoot string

	Ginkgo    Ginkgo
	Framework Framework
	Addons    Addons
	Suite     Suite
}

func (c *Config) Validate() error {
	var errs []error
	if c.KubeConfig == "" {
		errs = append(errs, fmt.Errorf("--kubernetes-config must be specified"))
	}
	if c.RepoRoot == "" {
		errs = append(errs, fmt.Errorf("--repo-root must be specified"))
	}

	errs = append(errs, c.Ginkgo.Validate()...)
	errs = append(errs, c.Framework.Validate()...)
	errs = append(errs, c.Addons.Validate()...)
	errs = append(errs, c.Suite.Validate()...)

	return utilerrors.NewAggregate(errs)
}

// Register flags common to all e2e test suites.
func (c *Config) AddFlags(fs *flag.FlagSet) {
	kubeConfigFile := os.Getenv(clientcmd.RecommendedConfigPathEnvVar)
	if kubeConfigFile == "" {
		homeDir, err := os.UserHomeDir()
		if err != nil {
			panic("Failed to get user home directory: " + err.Error())
		}
		kubeConfigFile = filepath.Join(homeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName)
	}
	// Kubernetes API server config
	fs.StringVar(&c.KubeConfig, "kubernetes-config", kubeConfigFile, "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar)
	fs.StringVar(&c.KubeContext, "kubernetes-context", "", "config context to use for kuberentes. If unset, will use value from 'current-context'")
	fs.StringVar(&c.Kubectl, "kubectl-path", "kubectl", "path to the kubectl binary to use during e2e tests.")
	fs.BoolVar(&c.Cleanup, "cleanup", true, "If true, addons will be cleaned up both before and after provisioning")

	// TODO: get rid of this variable by bundling required files as part of test suite
	fs.StringVar(&c.RepoRoot, "repo-root", "", "Path to the root of the repository, used for access to repo-homed test fixtures.")

	c.Ginkgo.AddFlags(fs)
	c.Framework.AddFlags(fs)
	c.Addons.AddFlags(fs)
	c.Suite.AddFlags(fs)
}
<commit_msg>spelling: kubernetes<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/tools/clientcmd"
)

type Config struct {
	KubeConfig  string
	KubeContext string
	Kubectl     string

	// If Cleanup is true, addons will be cleaned up both before and after provisioning
	Cleanup bool

	// RepoRoot is used as the base path for any parts of the framework that
	// require access to repo files, such as Helm charts and test fixtures.
	RepoRoot string

	Ginkgo    Ginkgo
	Framework Framework
	Addons    Addons
	Suite     Suite
}

func (c *Config) Validate() error {
	var errs []error
	if c.KubeConfig == "" {
		errs = append(errs, fmt.Errorf("--kubernetes-config must be specified"))
	}
	if c.RepoRoot == "" {
		errs = append(errs, fmt.Errorf("--repo-root must be specified"))
	}

	errs = append(errs, c.Ginkgo.Validate()...)
	errs = append(errs, c.Framework.Validate()...)
	errs = append(errs, c.Addons.Validate()...)
	errs = append(errs, c.Suite.Validate()...)

	return utilerrors.NewAggregate(errs)
}

// Register flags common to all e2e test suites.
func (c *Config) AddFlags(fs *flag.FlagSet) {
	kubeConfigFile := os.Getenv(clientcmd.RecommendedConfigPathEnvVar)
	if kubeConfigFile == "" {
		homeDir, err := os.UserHomeDir()
		if err != nil {
			panic("Failed to get user home directory: " + err.Error())
		}
		kubeConfigFile = filepath.Join(homeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName)
	}
	// Kubernetes API server config
	fs.StringVar(&c.KubeConfig, "kubernetes-config", kubeConfigFile, "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar)
	fs.StringVar(&c.KubeContext, "kubernetes-context", "", "config context to use for kubernetes. If unset, will use value from 'current-context'")
	fs.StringVar(&c.Kubectl, "kubectl-path", "kubectl", "path to the kubectl binary to use during e2e tests.")
	fs.BoolVar(&c.Cleanup, "cleanup", true, "If true, addons will be cleaned up both before and after provisioning")

	// TODO: get rid of this variable by bundling required files as part of test suite
	fs.StringVar(&c.RepoRoot, "repo-root", "", "Path to the root of the repository, used for access to repo-homed test fixtures.")

	c.Ginkgo.AddFlags(fs)
	c.Framework.AddFlags(fs)
	c.Addons.AddFlags(fs)
	c.Suite.AddFlags(fs)
}
<|endoftext|>


<commit_before><commit_msg>codegen (#1747)<commit_after><|endoftext|>


<commit_before>package main

// Race Conditions: note that there are numerous points in the test
// below where either runtime.Gosched() is invoked, and/or
// <-time.After(time.Millisecond) is invoked.
//
// In these cases, the intent is to force the scheduler to context
// switch to perform actions that should occur in other goroutines.
// In NONE of these cases where the results tested _after_ the forced
// context switch is there a guarantee the event happens at any
// deadline time in the future.  The forced context switches are
// merely to speed up the test run.  Correctness is defined by the
// action after the forced context switch _eventually_ happening.

import (
	"context"
	"runtime"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type AppWatcher struct {
	id     int
	ch     WatchCh // <-chan string
	ctx    context.Context
	cancel context.CancelFunc
}

func TestMain(t *testing.T) {
	// create the Middleware => create backend
	m, b := NewMiddleware()

	w := make(map[int]AppWatcher) // watchers
	for ii := 0; ii < 5; ii++ {
		ctx, cancel := context.WithCancel(context.Background())
		wch := m.Watch(ctx, ii)
		w[ii] = AppWatcher{
			id:     ii,
			ch:     wch,
			ctx:    ctx,
			cancel: cancel,
		}
	}

	assert.Equal(t, m.NumWatchers(), 5)

	// Backend closes with nothing to be read in the queues
	//
	b.Fail(0)         // immediately fail watcher 0 from the backend
	runtime.Gosched() // yield so the canceled context can react
	select {
	case _, ok := <-w[0].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[0].cancel()       // cancel just because
		delete(w, 0)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[0].ch should have been closed by b.Fail(0)")
	}

	<-time.After(time.Millisecond) // yield hard to give other goroutines a chance to run

	assert.Equal(t, m.NumWatchers(), 4)

	//  Verify all (existing) watchers receive messages
	//
	b.Message("test-message")
	ReadAllWatchers(t, w)
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 4)

	// Backend closes with messages in the channels
	//
	b.Message("test-message")
	b.Fail(1)             // from the backend: cancel/close after write
	runtime.Gosched()     // yield so the canceled context can react
	ReadAllWatchers(t, w) // all watchers should return the message, they all got it
	select {
	case _, ok := <-w[1].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[1].cancel()       // clean-up
		delete(w, 1)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[1].ch should have been closed by b.Fail(1)")
	}
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 3)

	// Frontend closes with messages in the channels
	//
	b.Message("test-message")
	w[2].cancel()         // from the frontend: cancel/close after write
	runtime.Gosched()     // yield so the canceled context can react
	ReadAllWatchers(t, w) // all watchers should return the message, they all got it
	select {
	case _, ok := <-w[2].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[2].cancel()       // double-cancel, which should be ok
		delete(w, 2)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[2].ch should have been closed by w[2].Cancel()")
	}
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 2)

	// Frontend closes with the channels empty
	w[3].cancel() // from the frontend: cancel/close before write
	// no goroutine yield, cancelation is strictly ordered before write
	b.Message("test-message")
	select {
	case _, ok := <-w[3].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[3].cancel()       // double-cancel, which should be ok
		delete(w, 3)
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[3].ch should have been closed by w[3].cancel()")
	}
	ReadAllWatchers(t, w) // all (remaining) watchers should return a message

	<-time.After(time.Millisecond) // yield hard to give other goroutines a chance to run

	assert.Equal(t, m.NumWatchers(), 1)
}

func ReadAllWatchers(t *testing.T, watchers map[int]AppWatcher) {
	for id, w := range watchers {
		select {
		// do not test for w.ch.Done() as the important part is the channel has data for consumption
		case <-w.ch:
			// good - this is what's supposed to happen
		case <-time.After(time.Millisecond):
			assert.Failf(t, "nothing to read, unexpected", "watcher[%d]", id)
		}
	}
}

func AllWatchersBlockOnRead(t *testing.T, watchers map[int]AppWatcher) {
	for id, w := range watchers {
		select {
		case <-w.ch:
			assert.Fail(t, "watcher had pending read", "watcher[%d]", id)
		case <-w.ctx.Done():
			assert.Fail(t, "watcher ctx canceled", "watcher[%d]", id)
		case <-time.After(time.Millisecond):
			// good - this is what's supposed to happen
		}
	}
}
<commit_msg>go: enhance testing of select on a (dynamic) set of canceled contexts<commit_after>package main

// Race Conditions: note that there are numerous points in the test
// below where either runtime.Gosched() is invoked, and/or
// <-time.After(time.Millisecond) is invoked.
//
// In these cases, the intent is to force the scheduler to context
// switch to perform actions that should occur in other goroutines.
// In NONE of these cases where the results tested _after_ the forced
// context switch is there a guarantee the event happens at any
// deadline time in the future.  The forced context switches are
// merely to speed up the test run.  Correctness is defined by the
// action after the forced context switch _eventually_ happening.

import (
	"context"
	"runtime"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type AppWatcher struct {
	id     int
	ch     WatchCh // <-chan string
	ctx    context.Context
	cancel context.CancelFunc
}

func TestMain(t *testing.T) {
	// create the Middleware => create backend
	m, b := NewMiddleware()

	w := make(map[int]AppWatcher) // watchers
	for ii := 0; ii < 6; ii++ {
		ctx, cancel := context.WithCancel(context.Background())
		wch := m.Watch(ctx, ii)
		w[ii] = AppWatcher{
			id:     ii,
			ch:     wch,
			ctx:    ctx,
			cancel: cancel,
		}
	}

	assert.Equal(t, m.NumWatchers(), 6)

	// Backend closes with nothing to be read in the queues
	//
	b.Fail(0)         // immediately fail watcher 0 from the backend
	runtime.Gosched() // yield so the canceled context can react
	select {
	case _, ok := <-w[0].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[0].cancel()       // cancel just because
		delete(w, 0)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[0].ch should have been closed by b.Fail(0)")
	}

	<-time.After(time.Millisecond) // yield hard to give other goroutines a chance to run

	assert.Equal(t, m.NumWatchers(), 5)

	//  Verify all (existing) watchers receive messages
	//
	b.Message("test-message")
	ReadAllWatchers(t, w)
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 5)

	// Backend closes with messages in the channels
	//
	b.Message("test-message")
	b.Fail(1)             // from the backend: cancel/close after write
	runtime.Gosched()     // yield so the canceled context can react
	ReadAllWatchers(t, w) // all watchers should return the message, they all got it
	select {
	case _, ok := <-w[1].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[1].cancel()       // clean-up
		delete(w, 1)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[1].ch should have been closed by b.Fail(1)")
	}
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 4)

	// Frontend closes with messages in the channels
	//
	b.Message("test-message")
	w[2].cancel()         // from the frontend: cancel/close after write
	runtime.Gosched()     // yield so the canceled context can react
	ReadAllWatchers(t, w) // all watchers should return the message, they all got it
	select {
	case _, ok := <-w[2].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[2].cancel()       // double-cancel, which should be ok
		delete(w, 2)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[2].ch should have been closed by w[2].Cancel()")
	}
	AllWatchersBlockOnRead(t, w)

	assert.Equal(t, m.NumWatchers(), 3)

	// Frontend closes with the channels empty
	w[3].cancel() // from the frontend: cancel/close before write
	// no goroutine yield, cancelation is strictly ordered before write
	b.Message("test-message")
	select {
	case _, ok := <-w[3].ch:
		assert.False(t, ok) // indicating the channel was closed
		w[3].cancel()       // double-cancel, which should be ok
		delete(w, 3)
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[3].ch should have been closed by w[3].cancel()")
	}
	ReadAllWatchers(t, w) // all (remaining) watchers should return a message

	<-time.After(time.Millisecond) // yield hard to give other goroutines a chance to run

	assert.Equal(t, m.NumWatchers(), 2)

	// Middleware closes with message in the channels

	b.Message("test-message")
	m.Close()             // all watchers should now shutdown, just like canceling the context of each.
	ReadAllWatchers(t, w) // all watchers should return the test-message they received
	// remaining watchers [4, 5] should now return closed
	select {
	case _, ok := <-w[4].ch:
		assert.False(t, ok) // indicating the channel was closed
		delete(w, 4)        // remove the watcher, it's extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[4].ch should have been closed by m.Close()")
	}
	select {
	case _, ok := <-w[5].ch:
		assert.False(t, ok) // indicating the channel was closed
		delete(w, 5)        // remove the watcher, its been extinguished
	case <-time.After(time.Millisecond):
		assert.Fail(t, "w[5].ch should have been closed by m.Close()")
	}
	AllWatchersBlockOnRead(t, w) // all watchers should be closed and reaped, .. actually

	<-time.After(time.Millisecond)

	assert.Equal(t, m.NumWatchers(), 0) // all watchers should be closed and reaped
}

func ReadAllWatchers(t *testing.T, watchers map[int]AppWatcher) {
	for id, w := range watchers {
		select {
		// do not test for w.ch.Done() as the important part is the channel has data for consumption
		case <-w.ch:
			// good - this is what's supposed to happen
		case <-time.After(time.Millisecond):
			assert.Failf(t, "nothing to read, unexpected", "watcher[%d]", id)
		}
	}
}

func AllWatchersBlockOnRead(t *testing.T, watchers map[int]AppWatcher) {
	for id, w := range watchers {
		select {
		case <-w.ch:
			assert.Fail(t, "watcher had pending read", "watcher[%d]", id)
		case <-w.ctx.Done():
			assert.Fail(t, "watcher ctx canceled", "watcher[%d]", id)
		case <-time.After(time.Millisecond):
			// good - this is what's supposed to happen
		}
	}
}
<|endoftext|>


<commit_before>package bongo

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
)

// Fetch fetches the data from db by given parameters(fields of the struct)
func (b *Bongo) Fetch(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	if err := b.DB.Table(i.TableName()).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

// ById Fetches data from db by it's id
func (b *Bongo) ById(i Modellable, id int64) error {
	if err := b.DB.
		Table(i.TableName()).
		Where("id = ?", id).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

// Creates a new record with the given struct and its fields
func (b *Bongo) Create(i Modellable) error {
	if err := b.DB.Save(i).Error; err != nil {
		return err
	}

	return nil
}

// Update updates all fields of a struct with assigned data
func (b *Bongo) Update(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	// Update and Create is using the Save method, so they are
	// same functions but GORM handles, AfterCreate and AfterUpdate
	// in correct manner
	return b.Create(i)
}

// Delete deletes the data by it's id, it doesnt take any other fields
// into consideration
func (b *Bongo) Delete(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	if err := b.DB.Delete(i).Error; err != nil {
		return err
	}

	return nil
}

// FetchByIds fetches records by their ids and returns results in the same order
// as the ids; if no records in db we don't return error
func (b *Bongo) FetchByIds(i Modellable, data interface{}, ids []int64) error {
	if len(ids) == 0 {
		return nil
	}

	orderByQuery := ""
	comma := ""
	for _, id := range ids {
		orderByQuery = orderByQuery + comma + " id = " + strconv.FormatInt(id, 10) + " desc"
		comma = ","
	}

	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	query = query.Order(orderByQuery)

	query = query.Where(ids)

	query = query.Find(data)

	// supress not found errors
	return CheckErr(query)

}

func (b *Bongo) UpdatePartial(i Modellable, set map[string]interface{}) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	// init query
	query := b.DB

	query = query.Table(i.TableName())

	query = query.Where(i.GetId())

	if err := query.Update(set).Error; err != nil {
		return err
	}

	if err := b.Fetch(i); err != nil {
		return err
	}

	b.AfterUpdate(i)
	return nil
}

// selector, set
func (b *Bongo) UpdateMulti(i Modellable, rest ...map[string]interface{}) error {
	var set, selector map[string]interface{}

	switch len(rest) {
	case 1:
		set = rest[0]
		selector = nil
	case 2:
		selector = rest[0]
		set = rest[1]
	default:
		return WrongParameter
	}

	query := b.DB.Table(i.TableName())

	//add selector
	query = addWhere(query, selector)

	if err := query.Updates(set).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Count(i Modellable, where ...interface{}) (int, error) {
	var count int

	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	// add query
	query = query.Where(where[0], where[1:len(where)]...)

	return count, query.Count(&count).Error
}

func (b *Bongo) CountWithQuery(i Modellable, q *Query) (int, error) {
	query := b.BuildQuery(i, q)
	var count int
	return count, query.Count(&count).Error
}

type Scope func(d *gorm.DB) *gorm.DB

type Query struct {
	Selector   map[string]interface{}
	Sort       map[string]string
	Pluck      string
	Pagination Pagination
	Scopes     []Scope
}

func (q *Query) AddScope(scope Scope) {
	if q.Scopes == nil {
		q.Scopes = make([]Scope, 0)
	}

	q.Scopes = append(q.Scopes, scope)
}

type Pagination struct {
	Limit int
	Skip  int
}

func NewPagination(limit int, skip int) *Pagination {
	return &Pagination{
		Limit: limit,
		Skip:  skip,
	}
}

func NewQS(selector map[string]interface{}) *Query {
	return &Query{
		Selector: selector,
	}
}

// selector, sort, limit, pluck,
func (b *Bongo) Some(i Modellable, data interface{}, q *Query) error {
	err := b.executeQuery(i, data, q)
	if err == gorm.RecordNotFound {
		return nil
	}
	return err
}

func (b *Bongo) One(i Modellable, data interface{}, q *Query) error {
	q.Pagination.Limit = 1
	return b.executeQuery(i, data, q)
}

func (b *Bongo) BuildQuery(i Modellable, q *Query) *gorm.DB {
	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	// add sort options
	query = addSort(query, q.Sort)

	query = addSkip(query, q.Pagination.Skip)

	query = addLimit(query, q.Pagination.Limit)

	// add selector
	query = addWhere(query, q.Selector)

	// put scopes
	if q.Scopes != nil && len(q.Scopes) > 0 {
		for _, scope := range q.Scopes {
			query = query.Scopes(scope)
		}
	}

	return query
}

func (b *Bongo) executeQuery(i Modellable, data interface{}, q *Query) error {
	// init query
	query := b.BuildQuery(i, q)

	var err error
	// TODO refactor this part
	if q.Pluck != "" {
		if strings.Contains(q.Pluck, ",") {
			// add pluck data
			query = addPluck(query, q.Pluck)

			err = query.Find(data).Error
		} else {
			err = query.Pluck(q.Pluck, data).Error
		}
	} else {
		err = query.Find(data).Error
	}

	return err
}

func (b *Bongo) PublishEvent(eventName string, i Modellable) error {
	data, err := json.Marshal(i)
	if err != nil {
		b.log.Error("Error while marshalling for publish %s", err)
		return err
	}

	err = b.Broker.Publish(i.TableName()+"_"+eventName, data)
	if err != nil {
		b.log.Error("Error while publishing %s", err)
		return err
	}

	return nil
}

func (b *Bongo) AfterCreate(i Modellable) {
	b.PublishEvent("created", i)
}

func (b *Bongo) AfterUpdate(i Modellable) {
	b.PublishEvent("updated", i)
}

func (b *Bongo) AfterDelete(i Modellable) {
	b.PublishEvent("deleted", i)
}

// addSort injects sort parameters into query
func addSort(query *gorm.DB, options map[string]string) *gorm.DB {

	if options == nil {
		return query
	}

	if len(options) == 0 {
		return query
	}

	var opts []string
	for key, val := range options {
		opts = append(opts, fmt.Sprintf("%s %v", key, val))
	}
	return query.Order(strings.Join(opts, ","))
}

// addPluck basically adds select statement for
// only required fields
func addPluck(query *gorm.DB, plucked string) *gorm.DB {
	if plucked == "" {
		return query
	}

	return query.Select(plucked)
}

// addWhere adds where query
func addWhere(query *gorm.DB, selector map[string]interface{}) *gorm.DB {
	if selector == nil {
		return query
	}

	// instead sending one selector, do chaining here
	return query.Where(selector)
}

// addSkip adds skip parameter into sql query
func addSkip(query *gorm.DB, skip int) *gorm.DB {
	if skip > 0 {
		return query.Offset(skip)
	}

	return query
}

// addLimit adds limit into query if set
func addLimit(query *gorm.DB, limit int) *gorm.DB {
	// if limit is minus or 0 ignore
	if limit > 0 {
		return query.Limit(limit)
	}

	return query
}

// CheckErr checks error exitence and returns
// if found, but this function suppress RecordNotFound errors
func CheckErr(res *gorm.DB) error {
	if res == nil {
		return nil
	}

	if res.Error == nil {
		return nil
	}

	if res.Error == gorm.RecordNotFound {
		return nil
	}

	return res.Error
}
<commit_msg>bongo/model: checkErr function is commented<commit_after>package bongo

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
)

// Fetch fetches the data from db by given parameters(fields of the struct)
func (b *Bongo) Fetch(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	if err := b.DB.Table(i.TableName()).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

// ById Fetches data from db by it's id
func (b *Bongo) ById(i Modellable, id int64) error {
	if err := b.DB.
		Table(i.TableName()).
		Where("id = ?", id).
		Find(i).
		Error; err != nil {
		return err
	}

	return nil
}

// Creates a new record with the given struct and its fields
func (b *Bongo) Create(i Modellable) error {
	if err := b.DB.Save(i).Error; err != nil {
		return err
	}

	return nil
}

// Update updates all fields of a struct with assigned data
func (b *Bongo) Update(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	// Update and Create is using the Save method, so they are
	// same functions but GORM handles, AfterCreate and AfterUpdate
	// in correct manner
	return b.Create(i)
}

// Delete deletes the data by it's id, it doesnt take any other fields
// into consideration
func (b *Bongo) Delete(i Modellable) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	if err := b.DB.Delete(i).Error; err != nil {
		return err
	}

	return nil
}

// FetchByIds fetches records by their ids and returns results in the same order
// as the ids; if no records in db we don't return error
func (b *Bongo) FetchByIds(i Modellable, data interface{}, ids []int64) error {
	if len(ids) == 0 {
		return nil
	}

	orderByQuery := ""
	comma := ""
	for _, id := range ids {
		orderByQuery = orderByQuery + comma + " id = " + strconv.FormatInt(id, 10) + " desc"
		comma = ","
	}

	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	query = query.Order(orderByQuery)

	query = query.Where(ids)

	query = query.Find(data)

	// supress not found errors
	return CheckErr(query)

}

func (b *Bongo) UpdatePartial(i Modellable, set map[string]interface{}) error {
	if i.GetId() == 0 {
		return IdIsNotSet
	}

	// init query
	query := b.DB

	query = query.Table(i.TableName())

	query = query.Where(i.GetId())

	if err := query.Update(set).Error; err != nil {
		return err
	}

	if err := b.Fetch(i); err != nil {
		return err
	}

	b.AfterUpdate(i)
	return nil
}

// selector, set
func (b *Bongo) UpdateMulti(i Modellable, rest ...map[string]interface{}) error {
	var set, selector map[string]interface{}

	switch len(rest) {
	case 1:
		set = rest[0]
		selector = nil
	case 2:
		selector = rest[0]
		set = rest[1]
	default:
		return WrongParameter
	}

	query := b.DB.Table(i.TableName())

	//add selector
	query = addWhere(query, selector)

	if err := query.Updates(set).Error; err != nil {
		return err
	}

	return nil
}

func (b *Bongo) Count(i Modellable, where ...interface{}) (int, error) {
	var count int

	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	// add query
	query = query.Where(where[0], where[1:len(where)]...)

	return count, query.Count(&count).Error
}

func (b *Bongo) CountWithQuery(i Modellable, q *Query) (int, error) {
	query := b.BuildQuery(i, q)
	var count int
	return count, query.Count(&count).Error
}

type Scope func(d *gorm.DB) *gorm.DB

type Query struct {
	Selector   map[string]interface{}
	Sort       map[string]string
	Pluck      string
	Pagination Pagination
	Scopes     []Scope
}

func (q *Query) AddScope(scope Scope) {
	if q.Scopes == nil {
		q.Scopes = make([]Scope, 0)
	}

	q.Scopes = append(q.Scopes, scope)
}

type Pagination struct {
	Limit int
	Skip  int
}

func NewPagination(limit int, skip int) *Pagination {
	return &Pagination{
		Limit: limit,
		Skip:  skip,
	}
}

func NewQS(selector map[string]interface{}) *Query {
	return &Query{
		Selector: selector,
	}
}

// selector, sort, limit, pluck,
func (b *Bongo) Some(i Modellable, data interface{}, q *Query) error {
	err := b.executeQuery(i, data, q)
	if err == gorm.RecordNotFound {
		return nil
	}
	return err
}

func (b *Bongo) One(i Modellable, data interface{}, q *Query) error {
	q.Pagination.Limit = 1
	return b.executeQuery(i, data, q)
}

func (b *Bongo) BuildQuery(i Modellable, q *Query) *gorm.DB {
	// init query
	query := b.DB.Model(i)

	// add table name
	query = query.Table(i.TableName())

	// add sort options
	query = addSort(query, q.Sort)

	query = addSkip(query, q.Pagination.Skip)

	query = addLimit(query, q.Pagination.Limit)

	// add selector
	query = addWhere(query, q.Selector)

	// put scopes
	if q.Scopes != nil && len(q.Scopes) > 0 {
		for _, scope := range q.Scopes {
			query = query.Scopes(scope)
		}
	}

	return query
}

func (b *Bongo) executeQuery(i Modellable, data interface{}, q *Query) error {
	// init query
	query := b.BuildQuery(i, q)

	var err error
	// TODO refactor this part
	if q.Pluck != "" {
		if strings.Contains(q.Pluck, ",") {
			// add pluck data
			query = addPluck(query, q.Pluck)

			err = query.Find(data).Error
		} else {
			err = query.Pluck(q.Pluck, data).Error
		}
	} else {
		err = query.Find(data).Error
	}

	return err
}

func (b *Bongo) PublishEvent(eventName string, i Modellable) error {
	data, err := json.Marshal(i)
	if err != nil {
		b.log.Error("Error while marshalling for publish %s", err)
		return err
	}

	err = b.Broker.Publish(i.TableName()+"_"+eventName, data)
	if err != nil {
		b.log.Error("Error while publishing %s", err)
		return err
	}

	return nil
}

func (b *Bongo) AfterCreate(i Modellable) {
	b.PublishEvent("created", i)
}

func (b *Bongo) AfterUpdate(i Modellable) {
	b.PublishEvent("updated", i)
}

func (b *Bongo) AfterDelete(i Modellable) {
	b.PublishEvent("deleted", i)
}

// addSort injects sort parameters into query
func addSort(query *gorm.DB, options map[string]string) *gorm.DB {

	if options == nil {
		return query
	}

	if len(options) == 0 {
		return query
	}

	var opts []string
	for key, val := range options {
		opts = append(opts, fmt.Sprintf("%s %v", key, val))
	}
	return query.Order(strings.Join(opts, ","))
}

// addPluck basically adds select statement for
// only required fields
func addPluck(query *gorm.DB, plucked string) *gorm.DB {
	if plucked == "" {
		return query
	}

	return query.Select(plucked)
}

// addWhere adds where query
func addWhere(query *gorm.DB, selector map[string]interface{}) *gorm.DB {
	if selector == nil {
		return query
	}

	// instead sending one selector, do chaining here
	return query.Where(selector)
}

// addSkip adds skip parameter into sql query
func addSkip(query *gorm.DB, skip int) *gorm.DB {
	if skip > 0 {
		return query.Offset(skip)
	}

	return query
}

// addLimit adds limit into query if set
func addLimit(query *gorm.DB, limit int) *gorm.DB {
	// if limit is minus or 0 ignore
	if limit > 0 {
		return query.Limit(limit)
	}

	return query
}

// CheckErr checks error exitence and returns if found, but this function
// suppress RecordNotFound errors
func CheckErr(res *gorm.DB) error {
	if res == nil {
		return nil
	}

	if res.Error == nil {
		return nil
	}

	if res.Error == gorm.RecordNotFound {
		return nil
	}

	return res.Error
}
<|endoftext|>


<commit_before>package stack

import (
	"koding/db/models"
	"koding/db/mongodb/modelhelper"

	"github.com/koding/kite"
)

// LoginRequest represents a request model for "auth.login"
// kloud's kite method.
type LoginRequest struct {
	// GroupName is a team name, which we're going to log in to.
	//
	// If empty, default team is going to be used
	// instead and its name can be read from response value.
	GroupName string `json:"groupName"`
}

// LoginResponse represents a response model for "auth.login"
// kloud's kite method.
type LoginResponse struct {
	// ClientID represents a session ID used for
	// authentication with remote.api and Social API.
	ClientID string `json:"clientID"`

	// GroupName is a team name, which we have just logged in to.
	GroupName string `json:"groupName"`
}

// AuthLogin creates a jSession for the given username and team.
//
// If a session already exists, the method is a nop and returns
// already existing one.
//
// TODO(rjeczalik): Add AuthLogout to force creation of a new
// session.
func (k *Kloud) AuthLogin(r *kite.Request) (interface{}, error) {
	k.Log.Debug("auth login called by %q with %q", r.Username, r.Args.Raw)

	req, err := getLoginReq(r)
	if err != nil {
		return nil, err
	}

	ses, err := modelhelper.UserLogin(r.Username, req.GroupName)
	if err != nil {
		return nil, NewError(ErrInternalServer)
	}

	return &LoginResponse{
		ClientID:  ses.ClientId,
		GroupName: req.GroupName,
	}, nil
}

func getLoginReq(r *kite.Request) (*LoginRequest, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var req LoginRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, NewError(ErrBadRequest)
	}

	if req.GroupName == "" {
		req.GroupName = models.KDIOGroupName
	}

	return &req, nil
}
<commit_msg>kloud/stack: accept presenceEndpoint<commit_after>package stack

import (
	"bytes"
	"encoding/json"
	"errors"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net/http"

	"github.com/koding/kite"
)

// LoginRequest represents a request model for "auth.login"
// kloud's kite method.
type LoginRequest struct {
	// GroupName is a team name, which we're going to log in to.
	//
	// If empty, default team is going to be used
	// instead and its name can be read from response value.
	GroupName string `json:"groupName"`
}

// LoginResponse represents a response model for "auth.login"
// kloud's kite method.
type LoginResponse struct {
	// ClientID represents a session ID used for
	// authentication with remote.api and Social API.
	ClientID string `json:"clientID"`

	// GroupName is a team name, which we have just logged in to.
	GroupName string `json:"groupName"`
}

// AuthLogin creates a jSession for the given username and team.
//
// If a session already exists, the method is a nop and returns
// already existing one.
//
// TODO(rjeczalik): Add AuthLogout to force creation of a new
// session.
func (k *Kloud) AuthLogin(presenceEndpoint string) func(r *kite.Request) (interface{}, error) {
	return func(r *kite.Request) (interface{}, error) {
		k.Log.Debug("auth login called by %q with %q", r.Username, r.Args.Raw)

		req, err := getLoginReq(r)
		if err != nil {
			return nil, err
		}

		ses, err := modelhelper.UserLogin(r.Username, req.GroupName)
		if err != nil {
			return nil, NewError(ErrInternalServer)
		}

		if err := sendPresenceRequest(presenceEndpoint, r.Username, req.GroupName); err != nil {
			// we dont need to block user login if there is something wrong with socialapi.
			k.Log.Error("sendPresenceRequest failed with & for", err.Error(), r.Username)
		}

		return &LoginResponse{
			ClientID:  ses.ClientId,
			GroupName: req.GroupName,
		}, nil
	}
}

func getLoginReq(r *kite.Request) (*LoginRequest, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var req LoginRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, NewError(ErrBadRequest)
	}

	if req.GroupName == "" {
		req.GroupName = models.KDIOGroupName
	}

	return &req, nil
}

func sendPresenceRequest(url, username, groupName string) error {
	jsonValue, err := json.Marshal(struct {
		Username  string
		GroupName string
	}{
		Username:  username,
		GroupName: groupName,
	})
	if err != nil {
		return err
	}

	resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonValue))
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()
	if err != nil {
		return err
	}

	if resp.StatusCode < 400 {
		return errors.New("bad request")
	}

	return nil
}
<|endoftext|>


<commit_before>package stack

import (
	"koding/db/models"
	"koding/db/mongodb/modelhelper"

	"github.com/koding/kite"
)

// LoginRequest represents a request model for "auth.login"
// kloud's kite method.
type LoginRequest struct {
	// GroupName is a team name, which we're going to log in to.
	//
	// If empty, default team is going to be used
	// instead and its name can be read from response value.
	GroupName string `json:"groupName"`
}

// LoginResponse represents a response model for "auth.login"
// kloud's kite method.
type LoginResponse struct {
	// ClientID represents a session ID used for
	// authentication with remote.api and Social API.
	ClientID string `json:"clientID"`

	// GroupName is a team name, which we have just logged in to.
	GroupName string `json:"groupName"`
}

// Pinger send ping requests for presence.
type Pinger interface {
	Ping(string, string) error
}

// AuthLogin creates a jSession for the given username and team.
//
// If a session already exists, the method is a nop and returns
// already existing one.
//
// TODO(rjeczalik): Add AuthLogout to force creation of a new
// session.
func (k *Kloud) AuthLogin(p Pinger) func(r *kite.Request) (interface{}, error) {
	return func(r *kite.Request) (interface{}, error) {
		k.Log.Debug("auth login called by %q with %q", r.Username, r.Args.Raw)

		req, err := getLoginReq(r)
		if err != nil {
			return nil, err
		}

		ses, err := modelhelper.UserLogin(r.Username, req.GroupName)
		if err != nil {
			return nil, NewError(ErrInternalServer)
		}

		if err := p.Ping(r.Username, req.GroupName); err != nil {
			// we dont need to block user login if there is something wrong with socialapi.
			k.Log.Error("sendPresenceRequest failed with & for", err.Error(), r.Username)
		}

		return &LoginResponse{
			ClientID:  ses.ClientId,
			GroupName: req.GroupName,
		}, nil
	}
}

func getLoginReq(r *kite.Request) (*LoginRequest, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var req LoginRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, NewError(ErrBadRequest)
	}

	if req.GroupName == "" {
		req.GroupName = models.KDIOGroupName
	}

	return &req, nil
}
<commit_msg>kloud/stack: add better error handling & logging<commit_after>package stack

import (
	"koding/db/models"
	"koding/db/mongodb/modelhelper"

	mgo "gopkg.in/mgo.v2"

	"github.com/koding/kite"
)

// LoginRequest represents a request model for "auth.login"
// kloud's kite method.
type LoginRequest struct {
	// GroupName is a team name, which we're going to log in to.
	//
	// If empty, default team is going to be used
	// instead and its name can be read from response value.
	GroupName string `json:"groupName"`
}

// LoginResponse represents a response model for "auth.login"
// kloud's kite method.
type LoginResponse struct {
	// ClientID represents a session ID used for
	// authentication with remote.api and Social API.
	ClientID string `json:"clientID"`

	// GroupName is a team name, which we have just logged in to.
	GroupName string `json:"groupName"`
}

// Pinger send ping requests for presence.
type Pinger interface {
	Ping(string, string) error
}

// AuthLogin creates a jSession for the given username and team.
//
// If a session already exists, the method is a nop and returns
// already existing one.
//
// TODO(rjeczalik): Add AuthLogout to force creation of a new
// session.
func (k *Kloud) AuthLogin(p Pinger) func(r *kite.Request) (interface{}, error) {
	return func(r *kite.Request) (interface{}, error) {
		k.Log.Debug("AuthLogin called by %q with %q", r.Username, r.Args.Raw)

		req, err := getLoginReq(r)
		if err != nil {
			return nil, err
		}

		ses, err := modelhelper.UserLogin(r.Username, req.GroupName)
		switch err {
		case nil:
		case mgo.ErrNotFound:
			return nil, NewError(ErrBadRequest)
		case modelhelper.ErrNotParticipant:
			return nil, NewError(ErrNotAuthorized)
		default:
			k.Log.Debug("Got generic error for UserLogin, username: %q, err: %q, args: %q", r.Username, err.Error(), r.Args.Raw)
			return nil, NewError(ErrInternalServer)
		}

		if err := p.Ping(r.Username, req.GroupName); err != nil {
			// we dont need to block user login if there is something wrong with socialapi.
			k.Log.Error("Ping failed with %q for user %q", err.Error(), r.Username)
		}

		return &LoginResponse{
			ClientID:  ses.ClientId,
			GroupName: req.GroupName,
		}, nil
	}
}

func getLoginReq(r *kite.Request) (*LoginRequest, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var req LoginRequest
	if err := r.Args.One().Unmarshal(&req); err != nil {
		return nil, NewError(ErrBadRequest)
	}

	if req.GroupName == "" {
		req.GroupName = models.KDIOGroupName
	}

	return &req, nil
}
<|endoftext|>