<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/kkserver/kk-job/job"
	"github.com/kkserver/kk-lib/kk"
	"log"
	"os"
	"os/exec"
	"time"
)

func help() {
	fmt.Println("kk-job-slave <name> <127.0.0.1:8700> <kk.job.> <token> <workdir>")
}

func request(sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message, to string, timeout time.Duration, data interface{}, result interface{}) error {

	log.Printf("[REQUEST] %s ...\n", to)

	var b, _ = json.Marshal(data)
	var v = kk.Message{"REQUEST", "", to, "text/json", b}
	var r = sendRequest(&v, timeout)

	log.Printf("[REQUEST] %s %s\n", to, r.String())

	if r == nil {
		return errors.New(fmt.Sprintf("TO: %s fail", to))
	}

	if r.Method != "REQUEST" {
		return errors.New(fmt.Sprintf("TO: %s %s", to, r.String()))
	}

	if r.Type == "text/json" || r.Type == "application/json" {
		return json.Unmarshal(r.Content, result)
	}

	return nil
}

func createJSONFile(data interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	b, err := json.Marshal(data)

	if err != nil {

		var n = 0

		for n != len(b) {

			n, err := fd.Write(b)

			if err != nil {
				return err
			}

			if n != len(b) {
				b = b[n:]
			} else {
				break
			}
		}
	}

	return nil
}

func createTextFile(text string, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	b := []byte(text)

	var n = 0

	for n != len(b) {

		n, err := fd.Write(b)

		if err != nil {
			return err
		}

		if n != len(b) {
			b = b[n:]
		} else {
			break
		}
	}

	return nil
}

func createENVFile(options map[string]interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	fd.WriteString("#/bin/sh\n\n")

	for key, value := range options {
		var option, ok = value.(map[string]interface{})
		if ok {
			v, ok := option["value"]
			if ok {
				b, _ := json.Marshal(v)
				fmt.Fprintf(fd, "export %s=%s\n", key, string(b))
			}

		}
	}

	fd.WriteString("\n\n")

	return nil
}

func main() {

	log.SetFlags(log.Llongfile | log.LstdFlags)

	var args = os.Args
	var name string = ""
	var address string = ""
	var baseURL string = ""
	var token string = ""
	var workdir string = ""

	if len(args) > 4 {
		name = args[1]
		address = args[2]
		baseURL = args[3]
		token = args[4]
		workdir = args[5]
	} else {
		help()
		return
	}

	var sendRequest, _ = kk.TCPClientRequestConnect(name, address, map[string]interface{}{"exclusive": true})

	var process map[string]*kk.Dispatch = map[string]*kk.Dispatch{}
	var online func() = nil

	online = func() {

		var result = job.JobSlaveOnlineTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/online", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {
			log.Println(err)
		} else if result.Errno == 0 {
			createJSONFile(result.Slave, workdir+"/job/slave.json")
		} else {
			log.Println(result.Errmsg)
		}

		kk.GetDispatchMain().AsyncDelay(func() {

			go online()

		}, time.Second*6)

	}

	kk.GetDispatchMain().AsyncDelay(func() {

		go online()

	}, time.Second)

	var jobProcess func() = nil

	jobProcess = func() {

		var result = job.JobSlaveProcessTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/process", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {

			log.Println(err)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)

		} else if result.Errno == 0 {

			if result.Version != nil {

				var name = fmt.Sprintf("%s/job/%d_%d/", workdir, result.Version.JobId, result.Version.Version)

				_, err := os.Stat(name)

				if err != nil {

					var p = kk.NewDispatch()

					process[name] = p

					var exit = func() {

						kk.GetDispatchMain().Async(func() {

							p.Break()

							delete(process, name)

							go jobProcess()

						})

					}

					var fail = func(err error) {

						log.Println(err)

						createTextFile(err.Error(), name+"fail")

						var fail = job.JobVersionFailTaskResult{}

						request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
							"jobId":      fmt.Sprintf("%d", result.Version.JobId),
							"version":    fmt.Sprintf("%d", result.Version.Version),
							"statusText": err.Error()}, &fail)

						exit()

					}

					p.Async(func() {

						err := os.Mkdir(name, 0777)

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Job, name+"job.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Slave, name+"slave.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Version, name+"version.json")

						if err != nil {
							fail(err)
							return
						}

						var options = map[string]interface{}{}

						if result.Job.Options != "" {
							json.Unmarshal([]byte(result.Job.Options), &options)
						}

						if result.Version.Options != "" {
							json.Unmarshal([]byte(result.Job.Options), &options)
						}

						err = createENVFile(options, name+"options.sh")

						if err != nil {
							fail(err)
							return
						}

						cmd := exec.Command("/bin/sh", "-c", workdir+"/run.sh")

						cmd.Dir = name

						stdout, err := cmd.StdoutPipe()

						if err != nil {
							fail(err)
							return
						}

						defer stdout.Close()

						stderr, err := cmd.StderrPipe()

						if err != nil {
							fail(err)
							return
						}

						err = cmd.Start()

						if err != nil {
							fail(err)
							return
						}

						defer stderr.Close()

						go func() {

							rd := bufio.NewReader(stdout)

							for true {

								v, err := rd.ReadString('\n')

								if err != nil {
									break
								}

								var r = job.JobVersionLogTaskResult{}

								request(sendRequest, baseURL+"job/slave/log", time.Second, map[string]interface{}{
									"jobId":   fmt.Sprintf("%d", result.Version.JobId),
									"version": fmt.Sprintf("%d", result.Version.Version),
									"log":     fmt.Sprintf("[INFO] %s", v)}, &r)

							}

						}()

						go func() {

							rd := bufio.NewReader(stderr)

							for true {

								v, err := rd.ReadString('\n')

								if err != nil {
									break
								}

								var r = job.JobVersionLogTaskResult{}

								request(sendRequest, baseURL+"job/slave/log", time.Second, map[string]interface{}{
									"jobId":   fmt.Sprintf("%d", result.Version.JobId),
									"version": fmt.Sprintf("%d", result.Version.Version),
									"log":     fmt.Sprintf("[FAIL] %s", v)}, &r)

							}

						}()

						err = cmd.Wait()

						if err != nil {
							fail(err)
							return
						}

						var r = job.JobVersionOKTaskResult{}

						request(sendRequest, baseURL+"job/slave/ok", time.Second, map[string]interface{}{
							"jobId":   fmt.Sprintf("%d", result.Version.JobId),
							"version": fmt.Sprintf("%d", result.Version.Version)}, &r)

						exit()

					})

					return
				}

				var fail = job.JobVersionFailTaskResult{}

				request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
					"jobId":   fmt.Sprintf("%d", result.Version.JobId),
					"version": fmt.Sprintf("%d", result.Version.Version)}, &fail)

				log.Printf("[FAIL] jobId:%d version:%d %s\n", result.Version.JobId, result.Version.Version)

				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)

			} else {
				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)
			}

		} else {

			log.Println(result.Errmsg)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)
		}
	}

	kk.GetDispatchMain().AsyncDelay(func() {
		go jobProcess()
	}, time.Second)

	kk.DispatchMain()

	for _, v := range process {
		v.Break()
	}

}
<commit_msg>update<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/kkserver/kk-job/job"
	"github.com/kkserver/kk-lib/kk"
	"log"
	"os"
	"os/exec"
	"time"
)

func help() {
	fmt.Println("kk-job-slave <name> <127.0.0.1:8700> <kk.job.> <token> <workdir>")
}

func request(sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message, to string, timeout time.Duration, data interface{}, result interface{}) error {

	log.Printf("[REQUEST] %s ...\n", to)

	var b, _ = json.Marshal(data)
	var v = kk.Message{"REQUEST", "", to, "text/json", b}
	var r = sendRequest(&v, timeout)

	log.Printf("[REQUEST] %s %s\n", to, r.String())

	if r == nil {
		return errors.New(fmt.Sprintf("TO: %s fail", to))
	}

	if r.Method != "REQUEST" {
		return errors.New(fmt.Sprintf("TO: %s %s", to, r.String()))
	}

	if r.Type == "text/json" || r.Type == "application/json" {
		return json.Unmarshal(r.Content, result)
	}

	return nil
}

func createJSONFile(data interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	b, err := json.Marshal(data)

	if err != nil {

		var n = 0

		for n != len(b) {

			n, err := fd.Write(b)

			if err != nil {
				return err
			}

			if n != len(b) {
				b = b[n:]
			} else {
				break
			}
		}
	}

	return nil
}

func createTextFile(text string, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	b := []byte(text)

	var n = 0

	for n != len(b) {

		n, err := fd.Write(b)

		if err != nil {
			return err
		}

		if n != len(b) {
			b = b[n:]
		} else {
			break
		}
	}

	return nil
}

func createENVFile(options map[string]interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	fd.WriteString("#/bin/sh\n\n")

	for key, value := range options {
		var option, ok = value.(map[string]interface{})
		if ok {
			v, ok := option["value"]
			if ok {
				b, _ := json.Marshal(v)
				fmt.Fprintf(fd, "export %s=%s\n", key, string(b))
			}

		}
	}

	fd.WriteString("\n\n")

	return nil
}

func main() {

	log.SetFlags(log.Llongfile | log.LstdFlags)

	var args = os.Args
	var name string = ""
	var address string = ""
	var baseURL string = ""
	var token string = ""
	var workdir string = ""

	if len(args) > 4 {
		name = args[1]
		address = args[2]
		baseURL = args[3]
		token = args[4]
		workdir = args[5]
	} else {
		help()
		return
	}

	var sendRequest, _ = kk.TCPClientRequestConnect(name, address, map[string]interface{}{"exclusive": true})

	var process map[string]*kk.Dispatch = map[string]*kk.Dispatch{}
	var online func() = nil

	online = func() {

		var result = job.JobSlaveOnlineTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/online", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {
			log.Println(err)
		} else if result.Errno == 0 {
			createJSONFile(result.Slave, workdir+"/job/slave.json")
		} else {
			log.Println(result.Errmsg)
		}

		kk.GetDispatchMain().AsyncDelay(func() {

			go online()

		}, time.Second*6)

	}

	kk.GetDispatchMain().AsyncDelay(func() {

		var err = request(sendRequest, baseURL+"job/slave/login", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {

			log.Println(err)

			kk.GetDispatchMain().Break()

			return
		}

		go online()

	}, time.Second)

	var jobProcess func() = nil

	jobProcess = func() {

		var result = job.JobSlaveProcessTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/process", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {

			log.Println(err)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)

		} else if result.Errno == 0 {

			if result.Version != nil {

				var name = fmt.Sprintf("%s/job/%d_%d/", workdir, result.Version.JobId, result.Version.Version)

				_, err := os.Stat(name)

				if err != nil {

					var p = kk.NewDispatch()

					process[name] = p

					var exit = func() {

						kk.GetDispatchMain().Async(func() {

							p.Break()

							delete(process, name)

							go jobProcess()

						})

					}

					var fail = func(err error) {

						log.Println(err)

						createTextFile(err.Error(), name+"fail")

						var fail = job.JobVersionFailTaskResult{}

						request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
							"jobId":      fmt.Sprintf("%d", result.Version.JobId),
							"version":    fmt.Sprintf("%d", result.Version.Version),
							"statusText": err.Error()}, &fail)

						exit()

					}

					p.Async(func() {

						err := os.Mkdir(name, 0777)

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Job, name+"job.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Slave, name+"slave.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Version, name+"version.json")

						if err != nil {
							fail(err)
							return
						}

						var options = map[string]interface{}{}

						if result.Job.Options != "" {
							json.Unmarshal([]byte(result.Job.Options), &options)
						}

						if result.Version.Options != "" {
							json.Unmarshal([]byte(result.Job.Options), &options)
						}

						err = createENVFile(options, name+"options.sh")

						if err != nil {
							fail(err)
							return
						}

						cmd := exec.Command("/bin/sh", "-c", workdir+"/run.sh")

						cmd.Dir = name

						stdout, err := cmd.StdoutPipe()

						if err != nil {
							fail(err)
							return
						}

						defer stdout.Close()

						stderr, err := cmd.StderrPipe()

						if err != nil {
							fail(err)
							return
						}

						err = cmd.Start()

						if err != nil {
							fail(err)
							return
						}

						defer stderr.Close()

						go func() {

							rd := bufio.NewReader(stdout)

							for true {

								v, err := rd.ReadString('\n')

								if err != nil {
									break
								}

								var r = job.JobVersionLogTaskResult{}

								request(sendRequest, baseURL+"job/slave/log", time.Second, map[string]interface{}{
									"jobId":   fmt.Sprintf("%d", result.Version.JobId),
									"version": fmt.Sprintf("%d", result.Version.Version),
									"log":     fmt.Sprintf("[INFO] %s", v)}, &r)

							}

						}()

						go func() {

							rd := bufio.NewReader(stderr)

							for true {

								v, err := rd.ReadString('\n')

								if err != nil {
									break
								}

								var r = job.JobVersionLogTaskResult{}

								request(sendRequest, baseURL+"job/slave/log", time.Second, map[string]interface{}{
									"jobId":   fmt.Sprintf("%d", result.Version.JobId),
									"version": fmt.Sprintf("%d", result.Version.Version),
									"log":     fmt.Sprintf("[FAIL] %s", v)}, &r)

							}

						}()

						err = cmd.Wait()

						if err != nil {
							fail(err)
							return
						}

						var r = job.JobVersionOKTaskResult{}

						request(sendRequest, baseURL+"job/slave/ok", time.Second, map[string]interface{}{
							"jobId":   fmt.Sprintf("%d", result.Version.JobId),
							"version": fmt.Sprintf("%d", result.Version.Version)}, &r)

						exit()

					})

					return
				}

				var fail = job.JobVersionFailTaskResult{}

				request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
					"jobId":   fmt.Sprintf("%d", result.Version.JobId),
					"version": fmt.Sprintf("%d", result.Version.Version)}, &fail)

				log.Printf("[FAIL] jobId:%d version:%d %s\n", result.Version.JobId, result.Version.Version)

				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)

			} else {
				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)
			}

		} else {

			log.Println(result.Errmsg)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)
		}
	}

	kk.GetDispatchMain().AsyncDelay(func() {
		go jobProcess()
	}, time.Second)

	kk.DispatchMain()

	for _, v := range process {
		v.Break()
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package common

// seccomp default whitelists/blacklists.
// rkt tries not to diverge from docker here, for the moment.

var (
	// DockerDefaultSeccompWhitelist contains a default whitelist of syscalls,
	// used by docker for seccomp filtering.
	// See https://github.com/docker/docker/blob/master/profiles/seccomp/default.json
	DockerDefaultSeccompWhitelist = []string{
		"accept",
		"accept4",
		"access",
		"alarm",
		"bind",
		"brk",
		"capget",
		"capset",
		"chdir",
		"chmod",
		"chown",
		"chown32",
		"clock_getres",
		"clock_gettime",
		"clock_nanosleep",
		"close",
		"connect",
		"copy_file_range",
		"creat",
		"dup",
		"dup2",
		"dup3",
		"epoll_create",
		"epoll_create1",
		"epoll_ctl",
		"epoll_ctl_old",
		"epoll_pwait",
		"epoll_wait",
		"epoll_wait_old",
		"eventfd",
		"eventfd2",
		"execve",
		"execveat",
		"exit",
		"exit_group",
		"faccessat",
		"fadvise64",
		"fadvise64_64",
		"fallocate",
		"fanotify_mark",
		"fchdir",
		"fchmod",
		"fchmodat",
		"fchown",
		"fchown32",
		"fchownat",
		"fcntl",
		"fcntl64",
		"fdatasync",
		"fgetxattr",
		"flistxattr",
		"flock",
		"fork",
		"fremovexattr",
		"fsetxattr",
		"fstat",
		"fstat64",
		"fstatat64",
		"fstatfs",
		"fstatfs64",
		"fsync",
		"ftruncate",
		"ftruncate64",
		"futex",
		"futimesat",
		"getcpu",
		"getcwd",
		"getdents",
		"getdents64",
		"getegid",
		"getegid32",
		"geteuid",
		"geteuid32",
		"getgid",
		"getgid32",
		"getgroups",
		"getgroups32",
		"getitimer",
		"getpeername",
		"getpgid",
		"getpgrp",
		"getpid",
		"getppid",
		"getpriority",
		"getrandom",
		"getresgid",
		"getresgid32",
		"getresuid",
		"getresuid32",
		"getrlimit",
		"get_robust_list",
		"getrusage",
		"getsid",
		"getsockname",
		"getsockopt",
		"get_thread_area",
		"gettid",
		"gettimeofday",
		"getuid",
		"getuid32",
		"getxattr",
		"inotify_add_watch",
		"inotify_init",
		"inotify_init1",
		"inotify_rm_watch",
		"io_cancel",
		"ioctl",
		"io_destroy",
		"io_getevents",
		"ioprio_get",
		"ioprio_set",
		"io_setup",
		"io_submit",
		"ipc",
		"kill",
		"lchown",
		"lchown32",
		"lgetxattr",
		"link",
		"linkat",
		"listen",
		"listxattr",
		"llistxattr",
		"_llseek",
		"lremovexattr",
		"lseek",
		"lsetxattr",
		"lstat",
		"lstat64",
		"madvise",
		"memfd_create",
		"mincore",
		"mkdir",
		"mkdirat",
		"mknod",
		"mknodat",
		"mmap",
		"mmap2",
		"mprotect",
		"mq_getsetattr",
		"mq_notify",
		"mq_open",
		"mq_timedreceive",
		"mq_timedsend",
		"mq_unlink",
		"mremap",
		"msgctl",
		"msgget",
		"msgrcv",
		"msgsnd",
		"msync",
		"munlock",
		"munlockall",
		"munmap",
		"nanosleep",
		"newfstatat",
		"_newselect",
		"open",
		"openat",
		"pause",
		"personality", // this is args-filtered by docker
		"pipe",
		"pipe2",
		"poll",
		"ppoll",
		"prctl",
		"pread64",
		"preadv",
		"prlimit64",
		"pselect6",
		"pwrite64",
		"pwritev",
		"read",
		"readahead",
		"readlink",
		"readlinkat",
		"readv",
		"recv",
		"recvfrom",
		"recvmmsg",
		"recvmsg",
		"remap_file_pages",
		"removexattr",
		"rename",
		"renameat",
		"renameat2",
		"restart_syscall",
		"rmdir",
		"rt_sigaction",
		"rt_sigpending",
		"rt_sigprocmask",
		"rt_sigqueueinfo",
		"rt_sigreturn",
		"rt_sigsuspend",
		"rt_sigtimedwait",
		"rt_tgsigqueueinfo",
		"sched_getaffinity",
		"sched_getattr",
		"sched_getparam",
		"sched_get_priority_max",
		"sched_get_priority_min",
		"sched_getscheduler",
		"sched_rr_get_interval",
		"sched_setaffinity",
		"sched_setattr",
		"sched_setparam",
		"sched_setscheduler",
		"sched_yield",
		"seccomp",
		"select",
		"semctl",
		"semget",
		"semop",
		"semtimedop",
		"send",
		"sendfile",
		"sendfile64",
		"sendmmsg",
		"sendmsg",
		"sendto",
		"setfsgid",
		"setfsgid32",
		"setfsuid",
		"setfsuid32",
		"setgid",
		"setgid32",
		"setgroups",
		"setgroups32",
		"setitimer",
		"setpgid",
		"setpriority",
		"setregid",
		"setregid32",
		"setresgid",
		"setresgid32",
		"setresuid",
		"setresuid32",
		"setreuid",
		"setreuid32",
		"setrlimit",
		"set_robust_list",
		"setsid",
		"setsockopt",
		"set_thread_area",
		"set_tid_address",
		"setuid",
		"setuid32",
		"setxattr",
		"shmat",
		"shmctl",
		"shmdt",
		"shmget",
		"shutdown",
		"sigaltstack",
		"signalfd",
		"signalfd4",
		"sigreturn",
		"socket",
		"socketcall",
		"socketpair",
		"splice",
		"stat",
		"stat64",
		"statfs",
		"statfs64",
		"symlink",
		"symlinkat",
		"sync",
		"sync_file_range",
		"syncfs",
		"sysinfo",
		"syslog",
		"tee",
		"tgkill",
		"time",
		"timer_create",
		"timer_delete",
		"timerfd_create",
		"timerfd_gettime",
		"timerfd_settime",
		"timer_getoverrun",
		"timer_gettime",
		"timer_settime",
		"times",
		"tkill",
		"truncate",
		"truncate64",
		"ugetrlimit",
		"umask",
		"uname",
		"unlink",
		"unlinkat",
		"utime",
		"utimensat",
		"utimes",
		"vfork",
		"vmsplice",
		"wait4",
		"waitid",
		"waitpid",
		"write",
		"writev",
		"arch_prctl",
		"modify_ldt",
		"chroot",
		"clone", // this is args-filtered by docker
	}
	// DockerDefaultSeccompBlacklist contains a default blacklist of syscalls,
	// used by docker for seccomp filtering.
	// See https://github.com/docker/docker/blob/master/docs/security/seccomp.md
	DockerDefaultSeccompBlacklist = []string{
		"acct",
		"add_key",
		"adjtimex",
		"bpf",
		"clock_adjtime",
		"clock_settime",
		// "clone", // this is args-filtered by docker
		"create_module",
		"delete_module",
		"finit_module",
		"get_kernel_syms",
		"get_mempolicy",
		"init_module",
		"ioperm",
		"iopl",
		"kcmp",
		"kexec_file_load",
		"kexec_load",
		"keyctl",
		"lookup_dcookie",
		"mbind",
		"mount",
		"move_pages",
		"name_to_handle_at",
		"nfsservctl",
		"open_by_handle_at",
		"perf_event_open",
		// "personality", // this is args-filtered by docker
		"pivot_root",
		"process_vm_readv",
		"process_vm_writev",
		"ptrace",
		"query_module",
		"quotactl",
		"reboot",
		"request_key",
		"set_mempolicy",
		"setns",
		"settimeofday",
		"stime",
		"swapon",
		"swapoff",
		"sysfs",
		"_sysctl",
		"umount",
		"umount2",
		"unshare",
		"uselib",
		"userfaultfd",
		"ustat",
		"vm86",
		"vm86old",
	}

	// RktDefaultSeccompBlacklist contains a default blacklist of syscalls,
	// used by rkt for seccomp filtering.
	RktDefaultSeccompBlacklist = DockerDefaultSeccompBlacklist
	// RktDefaultSeccompWhitelist contains a default whitelist of syscalls,
	// used by rkt for seccomp filtering.
	RktDefaultSeccompWhitelist = DockerDefaultSeccompWhitelist
)
<commit_msg>seccomp/docker: update docker whitelist to include mlock<commit_after>// Copyright 2016 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package common

// seccomp default whitelists/blacklists.
// rkt tries not to diverge from docker here, for the moment.

var (
	// DockerDefaultSeccompWhitelist contains a default whitelist of syscalls,
	// used by docker for seccomp filtering.
	// See https://github.com/docker/docker/blob/master/profiles/seccomp/default.json
	DockerDefaultSeccompWhitelist = []string{
		"accept",
		"accept4",
		"access",
		"alarm",
		"bind",
		"brk",
		"capget",
		"capset",
		"chdir",
		"chmod",
		"chown",
		"chown32",
		"clock_getres",
		"clock_gettime",
		"clock_nanosleep",
		"close",
		"connect",
		"copy_file_range",
		"creat",
		"dup",
		"dup2",
		"dup3",
		"epoll_create",
		"epoll_create1",
		"epoll_ctl",
		"epoll_ctl_old",
		"epoll_pwait",
		"epoll_wait",
		"epoll_wait_old",
		"eventfd",
		"eventfd2",
		"execve",
		"execveat",
		"exit",
		"exit_group",
		"faccessat",
		"fadvise64",
		"fadvise64_64",
		"fallocate",
		"fanotify_mark",
		"fchdir",
		"fchmod",
		"fchmodat",
		"fchown",
		"fchown32",
		"fchownat",
		"fcntl",
		"fcntl64",
		"fdatasync",
		"fgetxattr",
		"flistxattr",
		"flock",
		"fork",
		"fremovexattr",
		"fsetxattr",
		"fstat",
		"fstat64",
		"fstatat64",
		"fstatfs",
		"fstatfs64",
		"fsync",
		"ftruncate",
		"ftruncate64",
		"futex",
		"futimesat",
		"getcpu",
		"getcwd",
		"getdents",
		"getdents64",
		"getegid",
		"getegid32",
		"geteuid",
		"geteuid32",
		"getgid",
		"getgid32",
		"getgroups",
		"getgroups32",
		"getitimer",
		"getpeername",
		"getpgid",
		"getpgrp",
		"getpid",
		"getppid",
		"getpriority",
		"getrandom",
		"getresgid",
		"getresgid32",
		"getresuid",
		"getresuid32",
		"getrlimit",
		"get_robust_list",
		"getrusage",
		"getsid",
		"getsockname",
		"getsockopt",
		"get_thread_area",
		"gettid",
		"gettimeofday",
		"getuid",
		"getuid32",
		"getxattr",
		"inotify_add_watch",
		"inotify_init",
		"inotify_init1",
		"inotify_rm_watch",
		"io_cancel",
		"ioctl",
		"io_destroy",
		"io_getevents",
		"ioprio_get",
		"ioprio_set",
		"io_setup",
		"io_submit",
		"ipc",
		"kill",
		"lchown",
		"lchown32",
		"lgetxattr",
		"link",
		"linkat",
		"listen",
		"listxattr",
		"llistxattr",
		"_llseek",
		"lremovexattr",
		"lseek",
		"lsetxattr",
		"lstat",
		"lstat64",
		"madvise",
		"memfd_create",
		"mincore",
		"mkdir",
		"mkdirat",
		"mknod",
		"mknodat",
		"mlock",
		"mlock2",
		"mlockall",
		"mmap",
		"mmap2",
		"mprotect",
		"mq_getsetattr",
		"mq_notify",
		"mq_open",
		"mq_timedreceive",
		"mq_timedsend",
		"mq_unlink",
		"mremap",
		"msgctl",
		"msgget",
		"msgrcv",
		"msgsnd",
		"msync",
		"munlock",
		"munlockall",
		"munmap",
		"nanosleep",
		"newfstatat",
		"_newselect",
		"open",
		"openat",
		"pause",
		"personality", // this is args-filtered by docker
		"pipe",
		"pipe2",
		"poll",
		"ppoll",
		"prctl",
		"pread64",
		"preadv",
		"prlimit64",
		"pselect6",
		"pwrite64",
		"pwritev",
		"read",
		"readahead",
		"readlink",
		"readlinkat",
		"readv",
		"recv",
		"recvfrom",
		"recvmmsg",
		"recvmsg",
		"remap_file_pages",
		"removexattr",
		"rename",
		"renameat",
		"renameat2",
		"restart_syscall",
		"rmdir",
		"rt_sigaction",
		"rt_sigpending",
		"rt_sigprocmask",
		"rt_sigqueueinfo",
		"rt_sigreturn",
		"rt_sigsuspend",
		"rt_sigtimedwait",
		"rt_tgsigqueueinfo",
		"sched_getaffinity",
		"sched_getattr",
		"sched_getparam",
		"sched_get_priority_max",
		"sched_get_priority_min",
		"sched_getscheduler",
		"sched_rr_get_interval",
		"sched_setaffinity",
		"sched_setattr",
		"sched_setparam",
		"sched_setscheduler",
		"sched_yield",
		"seccomp",
		"select",
		"semctl",
		"semget",
		"semop",
		"semtimedop",
		"send",
		"sendfile",
		"sendfile64",
		"sendmmsg",
		"sendmsg",
		"sendto",
		"setfsgid",
		"setfsgid32",
		"setfsuid",
		"setfsuid32",
		"setgid",
		"setgid32",
		"setgroups",
		"setgroups32",
		"setitimer",
		"setpgid",
		"setpriority",
		"setregid",
		"setregid32",
		"setresgid",
		"setresgid32",
		"setresuid",
		"setresuid32",
		"setreuid",
		"setreuid32",
		"setrlimit",
		"set_robust_list",
		"setsid",
		"setsockopt",
		"set_thread_area",
		"set_tid_address",
		"setuid",
		"setuid32",
		"setxattr",
		"shmat",
		"shmctl",
		"shmdt",
		"shmget",
		"shutdown",
		"sigaltstack",
		"signalfd",
		"signalfd4",
		"sigreturn",
		"socket",
		"socketcall",
		"socketpair",
		"splice",
		"stat",
		"stat64",
		"statfs",
		"statfs64",
		"symlink",
		"symlinkat",
		"sync",
		"sync_file_range",
		"syncfs",
		"sysinfo",
		"syslog",
		"tee",
		"tgkill",
		"time",
		"timer_create",
		"timer_delete",
		"timerfd_create",
		"timerfd_gettime",
		"timerfd_settime",
		"timer_getoverrun",
		"timer_gettime",
		"timer_settime",
		"times",
		"tkill",
		"truncate",
		"truncate64",
		"ugetrlimit",
		"umask",
		"uname",
		"unlink",
		"unlinkat",
		"utime",
		"utimensat",
		"utimes",
		"vfork",
		"vmsplice",
		"wait4",
		"waitid",
		"waitpid",
		"write",
		"writev",
		"arch_prctl",
		"modify_ldt",
		"chroot",
		"clone", // this is args-filtered by docker
	}
	// DockerDefaultSeccompBlacklist contains a default blacklist of syscalls,
	// used by docker for seccomp filtering.
	// See https://github.com/docker/docker/blob/master/docs/security/seccomp.md
	DockerDefaultSeccompBlacklist = []string{
		"acct",
		"add_key",
		"adjtimex",
		"bpf",
		"clock_adjtime",
		"clock_settime",
		// "clone", // this is args-filtered by docker
		"create_module",
		"delete_module",
		"finit_module",
		"get_kernel_syms",
		"get_mempolicy",
		"init_module",
		"ioperm",
		"iopl",
		"kcmp",
		"kexec_file_load",
		"kexec_load",
		"keyctl",
		"lookup_dcookie",
		"mbind",
		"mount",
		"move_pages",
		"name_to_handle_at",
		"nfsservctl",
		"open_by_handle_at",
		"perf_event_open",
		// "personality", // this is args-filtered by docker
		"pivot_root",
		"process_vm_readv",
		"process_vm_writev",
		"ptrace",
		"query_module",
		"quotactl",
		"reboot",
		"request_key",
		"set_mempolicy",
		"setns",
		"settimeofday",
		"stime",
		"swapon",
		"swapoff",
		"sysfs",
		"_sysctl",
		"umount",
		"umount2",
		"unshare",
		"uselib",
		"userfaultfd",
		"ustat",
		"vm86",
		"vm86old",
	}

	// RktDefaultSeccompBlacklist contains a default blacklist of syscalls,
	// used by rkt for seccomp filtering.
	RktDefaultSeccompBlacklist = DockerDefaultSeccompBlacklist
	// RktDefaultSeccompWhitelist contains a default whitelist of syscalls,
	// used by rkt for seccomp filtering.
	RktDefaultSeccompWhitelist = DockerDefaultSeccompWhitelist
)
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/google/gops/agent"
	"github.com/julienschmidt/httprouter"
	log "github.com/meifamily/logrus"
	"github.com/robfig/cron"

	ctrlr "github.com/meifamily/ptt-alertor/controllers"
	"github.com/meifamily/ptt-alertor/jobs"
	"github.com/meifamily/ptt-alertor/line"
	"github.com/meifamily/ptt-alertor/messenger"
	"github.com/meifamily/ptt-alertor/myutil"
	"github.com/meifamily/ptt-alertor/telegram"
)

var auth map[string]string

type myRouter struct {
	httprouter.Router
}

func (mr myRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.WithFields(log.Fields{
		"method": r.Method,
		"IP":     r.RemoteAddr,
		"URI":    r.URL.Path,
	}).Info("visit")
	mr.Router.ServeHTTP(w, r)
}

func newRouter() *myRouter {
	return &myRouter{
		Router: *httprouter.New(),
	}
}

func basicAuth(handle httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		user, password, hasAuth := r.BasicAuth()
		if hasAuth && user == auth["user"] && password == auth["password"] {
			handle(w, r, params)
		} else {
			w.Header().Set("WWW-Authenticate", "Basic realm=Restricted")
			http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
		}
	}
}

func main() {
	log.Info("Start Jobs")
	startJobs()
	// Web Server
	log.Info("Web Server Start on Port 9090")

	router := newRouter()
	router.NotFound = http.FileServer(http.Dir("public"))
	m := messenger.New()

	router.GET("/", ctrlr.Index)
	router.GET("/messenger", ctrlr.MessengerIndex)
	router.GET("/line", ctrlr.LineIndex)
	router.GET("/telegram", ctrlr.TelegramIndex)
	router.GET("/redirect/:sha1", ctrlr.Redirect)
	router.GET("/top", ctrlr.Top)
	router.GET("/docs", ctrlr.Docs)

	router.POST("/broadcast", basicAuth(ctrlr.Broadcast))

	// boards apis
	router.GET("/boards/:boardName/articles/:code", ctrlr.BoardArticle)
	router.GET("/boards/:boardName/articles", ctrlr.BoardArticleIndex)
	router.GET("/boards", ctrlr.BoardIndex)

	// keyword apis
	router.GET("/keyword/boards", ctrlr.KeywordBoards)

	// author apis
	router.GET("/author/boards", ctrlr.AuthorBoards)

	// pushsum apis
	router.GET("/pushsum/boards", ctrlr.PushSumBoards)

	// articles apis
	router.GET("/articles", ctrlr.ArticleIndex)

	// users apis
	router.GET("/users/:account", basicAuth(ctrlr.UserFind))
	router.GET("/users", basicAuth(ctrlr.UserAll))
	router.POST("/users", basicAuth(ctrlr.UserCreate))
	router.PUT("/users/:account", basicAuth(ctrlr.UserModify))

	// line
	router.POST("/line/callback", line.HandleRequest)
	router.POST("/line/notify/callback", line.CatchCallback)

	// facebook messenger
	router.GET("/messenger/webhook", m.Verify)
	router.POST("/messenger/webhook", m.Received)

	// telegram
	router.POST("/telegram/"+telegram.Token, telegram.HandleRequest)

	// gops agent
	if err := agent.Listen(&agent.Options{Addr: "127.0.0.1:9090", NoShutdownCleanup: false}); err != nil {
		log.Fatal(err)
	}

	err := http.ListenAndServe(":9090", router)
	if err != nil {
		log.Fatal("ListenAndServer ", err)
	}

}

func startJobs() {
	go jobs.NewChecker().Run()
	go jobs.NewPushListChecker().Run()
	go jobs.NewPushSumChecker().Run()
	c := cron.New()
	c.AddJob("@hourly", jobs.NewTop())
	c.AddJob("@every 3d", jobs.NewPushSumKeyReplacer())
	c.Start()
}

func init() {
	auth = myutil.Config("auth")
	jobs.NewTop().Run()
	// for initial app
	// jobs.NewCacheCleaner().Run()
	// jobs.NewGenerator().Run()
	// jobs.NewFetcher().Run()
}
<commit_msg>remove gops agent<commit_after>package main

import (
	"net/http"

	"github.com/julienschmidt/httprouter"
	log "github.com/meifamily/logrus"
	"github.com/robfig/cron"

	ctrlr "github.com/meifamily/ptt-alertor/controllers"
	"github.com/meifamily/ptt-alertor/jobs"
	"github.com/meifamily/ptt-alertor/line"
	"github.com/meifamily/ptt-alertor/messenger"
	"github.com/meifamily/ptt-alertor/myutil"
	"github.com/meifamily/ptt-alertor/telegram"
)

var auth map[string]string

type myRouter struct {
	httprouter.Router
}

func (mr myRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.WithFields(log.Fields{
		"method": r.Method,
		"IP":     r.RemoteAddr,
		"URI":    r.URL.Path,
	}).Info("visit")
	mr.Router.ServeHTTP(w, r)
}

func newRouter() *myRouter {
	return &myRouter{
		Router: *httprouter.New(),
	}
}

func basicAuth(handle httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		user, password, hasAuth := r.BasicAuth()
		if hasAuth && user == auth["user"] && password == auth["password"] {
			handle(w, r, params)
		} else {
			w.Header().Set("WWW-Authenticate", "Basic realm=Restricted")
			http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
		}
	}
}

func main() {
	log.Info("Start Jobs")
	startJobs()
	// Web Server
	log.Info("Web Server Start on Port 9090")

	router := newRouter()
	router.NotFound = http.FileServer(http.Dir("public"))
	m := messenger.New()

	router.GET("/", ctrlr.Index)
	router.GET("/messenger", ctrlr.MessengerIndex)
	router.GET("/line", ctrlr.LineIndex)
	router.GET("/telegram", ctrlr.TelegramIndex)
	router.GET("/redirect/:sha1", ctrlr.Redirect)
	router.GET("/top", ctrlr.Top)
	router.GET("/docs", ctrlr.Docs)

	router.POST("/broadcast", basicAuth(ctrlr.Broadcast))

	// boards apis
	router.GET("/boards/:boardName/articles/:code", ctrlr.BoardArticle)
	router.GET("/boards/:boardName/articles", ctrlr.BoardArticleIndex)
	router.GET("/boards", ctrlr.BoardIndex)

	// keyword apis
	router.GET("/keyword/boards", ctrlr.KeywordBoards)

	// author apis
	router.GET("/author/boards", ctrlr.AuthorBoards)

	// pushsum apis
	router.GET("/pushsum/boards", ctrlr.PushSumBoards)

	// articles apis
	router.GET("/articles", ctrlr.ArticleIndex)

	// users apis
	router.GET("/users/:account", basicAuth(ctrlr.UserFind))
	router.GET("/users", basicAuth(ctrlr.UserAll))
	router.POST("/users", basicAuth(ctrlr.UserCreate))
	router.PUT("/users/:account", basicAuth(ctrlr.UserModify))

	// line
	router.POST("/line/callback", line.HandleRequest)
	router.POST("/line/notify/callback", line.CatchCallback)

	// facebook messenger
	router.GET("/messenger/webhook", m.Verify)
	router.POST("/messenger/webhook", m.Received)

	// telegram
	router.POST("/telegram/"+telegram.Token, telegram.HandleRequest)

	err := http.ListenAndServe(":9090", router)
	if err != nil {
		log.Fatal("ListenAndServer ", err)
	}

}

func startJobs() {
	go jobs.NewChecker().Run()
	go jobs.NewPushListChecker().Run()
	go jobs.NewPushSumChecker().Run()
	c := cron.New()
	c.AddJob("@hourly", jobs.NewTop())
	c.AddJob("@every 3d", jobs.NewPushSumKeyReplacer())
	c.Start()
}

func init() {
	auth = myutil.Config("auth")
	jobs.NewTop().Run()
	// for initial app
	// jobs.NewCacheCleaner().Run()
	// jobs.NewGenerator().Run()
	// jobs.NewFetcher().Run()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"regexp"
	"reflect"
	"strings"
)

type config_paths struct {
	flannel_conf_path        string
	master_conf_path         string
	node_conf_path           string
	validity_check_conf_path string
}

type DefaultOpts struct {
	Flannel_opts         []string `yaml:"flannel_options"`
	Kubelet_opts         []string `yaml:"kubelet_options"`
	Kube_proxy_opts      []string `yaml:"kube_proxy_options"`
	Kube_apiserver_opts  []string `yaml:"kube_apiserver_options"`
	Kube_controller_opts []string `yaml:"kube_controller_manager_options"`
	Kube_scheduller_opts []string `yaml:"kube_scheduler_options"`
}

type ValidityOpts struct {
	Args_check struct {
			   Flannel_opts         []string `yaml:"flannel_options"`
			   Kubelet_opts         []string `yaml:"kubelet_options"`
			   Kube_proxy_opts      []string `yaml:"kube_proxy_options"`
			   Kube_apiserver_opts  []string `yaml:"kube_apiserver_options"`
			   Kube_controller_opts []string `yaml:"kube_controller_manager_options"`
			   Kube_scheduller_opts []string `yasml:"kube_scheduler_options"`
		   } `yaml: args_check`
}

//const relative_prefix_path = "/home/k8s_kdt/kdt/"
const relative_prefix_path = "D:\\go_src\\contrib\\ansible\\"

var Conf_paths = config_paths{
	flannel_conf_path:        "roles/flannel/defaults/main.yaml",
	master_conf_path:         "roles/master/defaults/main.yml",
	node_conf_path:           "roles/node/defaults/main.yml",
	validity_check_conf_path: "roles/validity-check/defaults/main.yml",
}

func main() {
	paths := [2]string{relative_prefix_path + Conf_paths.validity_check_conf_path, relative_prefix_path + Conf_paths.flannel_conf_path}
	fmt.Println(CompareArgs(paths, ValidityOpts{}, DefaultOpts{}, "flannel"))

}

func CompareArgs(file_path [2]string, validity_opts ValidityOpts, default_opts DefaultOpts, component_name string) (bool bool, err error) {
	fmt.Printf("Check %s args in path %s and path %s.\n", component_name, file_path[0], file_path[1])
	field_name := strings.Title(component_name + "_opts")
	validity_value, default_value, err := yaml_unmarshal(file_path, validity_opts, default_opts, field_name)
	if err != nil {
		return false, err
	}

	fmt.Printf("validity_value is %v\n", validity_value)
	fmt.Printf("default_value is %v\n", default_value)
	if isInclude(validity_value, default_value) && isInclude(default_value, validity_value) {
		return true, nil
	}
	return false, nil
}

func yaml_unmarshal(file_path [2]string, validity_opts ValidityOpts, default_opts DefaultOpts, field_name string) ([]string, []string, error) {
	data, err := ioutil.ReadFile(file_path[0])
	if err != nil {
		return []string{}, []string{}, fmt.Errorf("Reading config file failed: %v", err)
	}
	yaml.Unmarshal(data, &validity_opts)

	data, err = ioutil.ReadFile(file_path[1])
	if err != nil {
		return []string{}, []string{}, fmt.Errorf("Reading config file failed: %v", err)
	}
	yaml.Unmarshal(data, &default_opts)

	validity_value := reflect.ValueOf(validity_opts).FieldByName("Args_check").FieldByName(field_name).Interface()
	default_value := reflect.ValueOf(default_opts).FieldByName(field_name).Interface()
	val, _ := validity_value.([]string)
	def, _ := default_value.([]string)
	return val, def, nil
}

func isInclude(args0, args1 []string) bool {
	regx := regexp.MustCompile(`s*-+([^s=]+).*`)
	for _, arg0 := range args0 {
		include := false
		for _, arg1 := range args1 {
			if regx.FindStringSubmatch(arg0)[1] == regx.FindStringSubmatch(arg1)[1] {
				include = true
				break
			}
		}
		if !include {
			return include
		}
	}
	return true
}
<commit_msg>refactor<commit_after>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"regexp"
	"reflect"
	"strings"
	"os"
)

type config_paths struct {
	flannel_conf_path        string
	master_conf_path         string
	node_conf_path           string
	validity_check_conf_path string
}

type DefaultOpts struct {
	Flannel_opts         []string `yaml:"flannel_options"`
	Kubelet_opts         []string `yaml:"kubelet_options"`
	Kube_proxy_opts      []string `yaml:"kube_proxy_options"`
	Kube_apiserver_opts  []string `yaml:"kube_apiserver_options"`
	Kube_controller_opts []string `yaml:"kube_controller_manager_options"`
	Kube_scheduller_opts []string `yaml:"kube_scheduler_options"`
}

type ValidityOpts struct {
	Args_check struct {
			   Flannel_opts         []string `yaml:"flannel_options"`
			   Kubelet_opts         []string `yaml:"kubelet_options"`
			   Kube_proxy_opts      []string `yaml:"kube_proxy_options"`
			   Kube_apiserver_opts  []string `yaml:"kube_apiserver_options"`
			   Kube_controller_opts []string `yaml:"kube_controller_manager_options"`
			   Kube_scheduller_opts []string `yasml:"kube_scheduler_options"`
		   } `yaml: args_check`
}

//const relative_prefix_path = "/home/k8s_kdt/kdt/"
const relative_prefix_path = "D:\\go_src\\contrib\\ansible\\"

func main() {
	var conf_paths = config_paths{
		flannel_conf_path:        "roles/flannel/defaults/main.yaml",
		master_conf_path:         "roles/master/defaults/main.yml",
		node_conf_path:           "roles/node/defaults/main.yml",
		validity_check_conf_path: "roles/validity-check/defaults/main.yml",
	}

	paths := [2]string{conf_paths.validity_check_conf_path, conf_paths.flannel_conf_path}
	isEqual, _ := CompareArgs(paths, ValidityOpts{}, DefaultOpts{}, "flannel")
	fmt.Println("flannel flags keep consistency? ", isEqual)
	if !isEqual {
		os.Exit(-1)
	}
}

func CompareArgs(file_path [2]string, validity_opts ValidityOpts, default_opts DefaultOpts, component_name string) (bool bool, err error) {
	fmt.Printf("Check %s args in path %s and path %s.\n", component_name, file_path[0], file_path[1])
	field_name := strings.Title(component_name + "_opts")
	validity_value, default_value, err := yaml_unmarshal(file_path, validity_opts, default_opts, field_name)
	if err != nil {
		return false, err
	}

	fmt.Printf("%s validity args are %v\n", component_name, validity_value)
	fmt.Printf("%s default args are %v\n", component_name, default_value)
	if isInclude(validity_value, default_value) && isInclude(default_value, validity_value) {
		return true, nil
	}
	return false, nil
}

func yaml_unmarshal(file_path [2]string, validity_opts ValidityOpts, default_opts DefaultOpts, field_name string) ([]string, []string, error) {
	data, err := ioutil.ReadFile(relative_prefix_path + file_path[0])
	if err != nil {
		return []string{}, []string{}, fmt.Errorf("Reading config file failed: %v", err)
	}
	yaml.Unmarshal(data, &validity_opts)

	data, err = ioutil.ReadFile(relative_prefix_path + file_path[1])
	if err != nil {
		return []string{}, []string{}, fmt.Errorf("Reading config file failed: %v", err)
	}
	yaml.Unmarshal(data, &default_opts)

	validity_value := reflect.ValueOf(validity_opts).FieldByName("Args_check").FieldByName(field_name).Interface()
	default_value := reflect.ValueOf(default_opts).FieldByName(field_name).Interface()
	val, _ := validity_value.([]string)
	def, _ := default_value.([]string)
	return val, def, nil
}

func isInclude(args0, args1 []string) bool {
	regx := regexp.MustCompile(`s*-+([^s=]+).*`)
	for _, arg0 := range args0 {
		include := false
		for _, arg1 := range args1 {
			if regx.FindStringSubmatch(arg0)[1] == regx.FindStringSubmatch(arg1)[1] {
				include = true
				break
			}
		}
		if !include {
			return include
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

func main() {
	var args = os.Args
	var caskfile = "Caskfile"
	if len(args) > 1 {
		caskfile = args[1]
	}

	contents, err := ioutil.ReadFile(caskfile)
	if err != nil {
		exitUsage(err)
	}

	var brew = []string{"install"}
	var cask = []string{"cask", "install"}
	var tap = []string{"tap", "caskroom/cask"}

	for _, line := range strings.Split(string(contents), "\n") {
		commentPos := strings.Index(line, "#")
		// delete comment
		if commentPos >= 0 {
			line = line[:commentPos]
		}

		words := strings.Split(line, " ")

		// skip blank lines
		if len(words) < 1 {
			continue
		}

		// commands with no args
		if len(words) == 1 {
			continue // this should be error
		}

		switch words[0] {
		case "brew":
			brew = append(brew, words[1])
		case "cask":
			cask = append(cask, words[1])
		case "tap":
			tap = append(tap, words[1])
		default:
			continue // should be error (invalid command)
		}
	}

	for _, args := range [][]string{tap, brew, cask} {
		if len(args) > 2 || (args[0] == "install" && len(args) > 1) {
			cmd := exec.Command("brew", args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Run()
		}
	}
}

func exitUsage(errs ...error) {
	for _, err := range errs {
		fmt.Println(err.Error())
	}

	fmt.Println(`Usage:

beer-me <Caskfile>
`)
	os.Exit(1)
}
<commit_msg>Don't hardcode caskroom/cask under `brew tap` command<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

func main() {
	var args = os.Args
	var caskfile = "Caskfile"
	if len(args) > 1 {
		caskfile = args[1]
	}

	contents, err := ioutil.ReadFile(caskfile)
	if err != nil {
		exitUsage(err)
	}

	var brew = []string{"install"}
	var cask = []string{"cask", "install"}
	var tap = []string{"tap"}

	for _, line := range strings.Split(string(contents), "\n") {
		commentPos := strings.Index(line, "#")
		// delete comment
		if commentPos >= 0 {
			line = line[:commentPos]
		}

		words := strings.Split(line, " ")

		// skip blank lines
		if len(words) < 1 {
			continue
		}

		// commands with no args
		if len(words) == 1 {
			continue // this should be error
		}

		switch words[0] {
		case "brew":
			brew = append(brew, words[1])
		case "cask":
			cask = append(cask, words[1])
		case "tap":
			tap = append(tap, words[1])
		default:
			continue // should be error (invalid command)
		}
	}

	for _, args := range [][]string{tap, brew, cask} {
		if len(args) > 2 || (args[0] == "install" && len(args) > 1) {
			cmd := exec.Command("brew", args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Run()
		}
	}
}

func exitUsage(errs ...error) {
	for _, err := range errs {
		fmt.Println(err.Error())
	}

	fmt.Println(`Usage:

beer-me <Caskfile>
`)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

var DB *redis.Conn

func main() {
	environment := os.Getenv("ENVIRONMENT")
	if environment == "" {
		environment = "development"
	}

	config, err := ReadConfig("config/default.json", "config/"+environment+".json")
	if err != nil {
		log.Fatalln(err)
	}

	errorLogger, errorLogFile, err := NewErrorLogger(filepath.Join(config.LogDir, "errors"))
	if err != nil {
		log.Fatalln(err)
	}
	defer errorLogFile.Close()

	logFile, err := NewLogFile(config.LogDir)
	if err != nil {
		errorLogger.Fatalln(err)
	}
	defer logFile.Close()

	db, err := redis.DialTimeout(config.RedisNetwork, config.RedisAddr,
		config.MaxTimeout, config.MaxTimeout, config.MaxTimeout)
	if err != nil {
		errorLogger.Fatal(err)
	}
	defer db.Close()
	DB = &db

	router := mux.NewRouter()
	router.NotFoundHandler = http.HandlerFunc(NotFoundHandler)

	for _, route := range Routes {
		router.HandleFunc(route.URL, route.Handler).Methods(route.Method)
	}

	server := &http.Server{
		Addr:         config.ServerAddr,
		Handler:      NewLogHandler(logFile, router),
		ReadTimeout:  config.MaxTimeout,
		WriteTimeout: config.MaxTimeout,
	}

	if config.TLS != nil {
		err = server.ListenAndServeTLS(config.TLS.Cert, config.TLS.Key)
	} else {
		err = server.ListenAndServe()
	}
	if err != nil {
		errorLogger.Fatal(err)
	}
}
<commit_msg>Use strict slash routing<commit_after>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

var DB *redis.Conn

func main() {
	environment := os.Getenv("ENVIRONMENT")
	if environment == "" {
		environment = "development"
	}

	config, err := ReadConfig("config/default.json", "config/"+environment+".json")
	if err != nil {
		log.Fatalln(err)
	}

	errorLogger, errorLogFile, err := NewErrorLogger(filepath.Join(config.LogDir, "errors"))
	if err != nil {
		log.Fatalln(err)
	}
	defer errorLogFile.Close()

	logFile, err := NewLogFile(config.LogDir)
	if err != nil {
		errorLogger.Fatalln(err)
	}
	defer logFile.Close()

	db, err := redis.DialTimeout(config.RedisNetwork, config.RedisAddr,
		config.MaxTimeout, config.MaxTimeout, config.MaxTimeout)
	if err != nil {
		errorLogger.Fatal(err)
	}
	defer db.Close()
	DB = &db

	router := mux.NewRouter()
	router.StrictSlash(true)
	router.NotFoundHandler = http.HandlerFunc(NotFoundHandler)

	for _, route := range Routes {
		router.HandleFunc(route.URL, route.Handler).Methods(route.Method)
	}

	server := &http.Server{
		Addr:         config.ServerAddr,
		Handler:      NewLogHandler(logFile, router),
		ReadTimeout:  config.MaxTimeout,
		WriteTimeout: config.MaxTimeout,
	}

	if config.TLS != nil {
		err = server.ListenAndServeTLS(config.TLS.Cert, config.TLS.Key)
	} else {
		err = server.ListenAndServe()
	}
	if err != nil {
		errorLogger.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,
		MaxTime:  flushTimeout,
	}

	logger := &lib.LogHandler{
		Group:    "ecs-logs",
		Stream:   hostname,
		Hostname: hostname,
		Queue:    lib.NewMessageQueue(),
	}
	log.SetLevel(log.Level(level))
	log.SetHandler(multi.New(cli.New(os.Stdout), logger))

	expchan := time.Tick(flushTimeout / 2)
	msgchan := make(chan lib.Message, len(readers))
	sigchan := make(chan os.Signal, 1)
	counter := int32(len(readers))
	startReaders(readers, msgchan, &counter, hostname)

	for {
		select {
		case msg, ok := <-msgchan:
			now := time.Now()

			if !ok {
				log.Info("waiting for all write operations to complete")
				limits.Force = true
				flushAll(dests, store, limits, now, join)
				flushQueue(dests, store, logger.Queue, limits, now, join)
				join.Wait()
				return
			}

			_, stream := store.Add(msg, now)
			flush(dests, stream, limits, now, join)

		case <-logger.Queue.C:
			now := time.Now()
			flushQueue(dests, store, logger.Queue, limits, now, join)

		case <-expchan:
			now := time.Now()
			flushAll(dests, store, limits, now, join)
			removeExpired(dests, store, cacheTimeout, now)

		case sig := <-sigchan:
			log.WithFields(log.Fields{"signal": sig.String()}).Info("closing message readers")
			stopReaders(readers)
		}
	}
}

func getSources(names []string) (sources []source) {
	for i, src := range lib.GetSources(names...) {
		sources = append(sources, source{
			Source: src,
			name:   names[i],
		})
	}
	return
}

func getDestinations(names []string) (destinations []destination) {
	for i, dst := range lib.GetDestinations(names...) {
		destinations = append(destinations, destination{
			Destination: dst,
			name:        names[i],
		})
	}
	return
}

func openSources(sources []source) (readers []reader, err error) {
	readers = make([]reader, 0, len(sources))

	for _, source := range sources {
		if r, e := source.Open(); e != nil {
			log.WithFields(log.Fields{
				"source": source.name,
				"error":  e,
			}).Error("failed to open log source")
		} else {
			readers = append(readers, reader{
				Reader: r,
				name:   source.name,
			})
		}
	}

	if len(readers) == 0 {
		err = fmt.Errorf("no sources to read from")
	}

	return
}

func startReaders(readers []reader, msgchan chan<- lib.Message, counter *int32, hostname string) {
	for _, reader := range readers {
		go read(reader, msgchan, counter, hostname)
	}
}

func stopReaders(readers []reader) {
	for _, reader := range readers {
		reader.Close()
	}
}

func term(c chan<- lib.Message, counter *int32) {
	if atomic.AddInt32(counter, -1) == 0 {
		close(c)
	}
}

func read(r reader, c chan<- lib.Message, counter *int32, hostname string) {
	defer term(c, counter)
	for {
		var msg lib.Message
		var err error

		if msg, err = r.ReadMessage(); err != nil {
			if err == io.EOF {
				break
			}
			log.WithFields(log.Fields{
				"reader": r.name,
				"error":  err,
			}).Error("the message reader failed")
			continue
		}

		if len(msg.Group) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "group",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Stream) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "stream",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Event.Info.Host) == 0 {
			msg.Event.Info.Host = hostname
		}

		if msg.Event.Time == (time.Time{}) {
			msg.Event.Time = time.Now()
		}

		if msg.Event.Data == nil {
			msg.Event.Data = ecslogs.EventData{}
		}

		c <- msg
	}
}

func write(dest destination, group string, stream string, batch []lib.Message, join *sync.WaitGroup) {
	defer join.Done()

	var writer lib.Writer
	var err error

	if writer, err = dest.Open(group, stream); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
	defer writer.Close()

	if err = writer.WriteMessageBatch(batch); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
}

func flush(dests []destination, stream *lib.Stream, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	for {
		batch, reason := stream.Flush(limits, now)

		if len(batch) == 0 {
			break
		}

		// Ensure all messages in the batch are sorted. Checking if the batch is
		// sorted is an optimization since in most cases the batch will be sorted
		// because we're reading events that are generated live (checking for a
		// sorted list is O(N) vs O(N*log(N)) for sorting it).
		// There are cases where some log entries do appear unordered and this is
		// causing issues with CloudWatchLogs.
		if !sort.IsSorted(batch) {
			sort.Stable(batch)
		}

		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
			"count":  len(batch),
			"reason": reason,
		}).Info("flushing message batch")

		for _, dest := range dests {
			join.Add(1)
			go write(dest, stream.Group(), stream.Name(), batch, join)
		}
	}
}

func flushAll(dests []destination, store *lib.Store, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	store.ForEach(func(group *lib.Group) {
		group.ForEach(func(stream *lib.Stream) {
			flush(dests, stream, limits, now, join)
		})
	})
}

func flushQueue(dests []destination, store *lib.Store, queue *lib.MessageQueue, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	streams := make(map[string]*lib.Stream)

	for _, msg := range queue.Flush() {
		_, stream := store.Add(msg, now)
		key := stream.Group() + ":" + stream.Name()

		if streams[key] == nil {
			streams[key] = stream
		}
	}

	for _, stream := range streams {
		flush(dests, stream, limits, now, join)
	}
}

func removeExpired(dests []destination, store *lib.Store, cacheTimeout time.Duration, now time.Time) {
	for _, stream := range store.RemoveExpired(cacheTimeout, now) {
		for _, dest := range dests {
			dest.Close(stream.Group(), stream.Name())
		}
		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
		}).Info("removed expired stream")
	}
}

func logDropBatch(dest string, group string, stream string, err error, batch []lib.Message) {
	log.WithFields(log.Fields{
		"group":       group,
		"stream":      stream,
		"destination": dest,
		"error":       err,
		"count":       len(batch),
	}).Error("dropping message batch")

	for _, msg := range batch {
		log.WithFields(log.Fields{
			"group":  msg.Group,
			"stream": msg.Stream,
			"event":  msg.Event,
		}).Debug("dropped")
	}
}
<commit_msg>[]lib.Message => lib.MessageBatch<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,
		MaxTime:  flushTimeout,
	}

	logger := &lib.LogHandler{
		Group:    "ecs-logs",
		Stream:   hostname,
		Hostname: hostname,
		Queue:    lib.NewMessageQueue(),
	}
	log.SetLevel(log.Level(level))
	log.SetHandler(multi.New(cli.New(os.Stdout), logger))

	expchan := time.Tick(flushTimeout / 2)
	msgchan := make(chan lib.Message, len(readers))
	sigchan := make(chan os.Signal, 1)
	counter := int32(len(readers))
	startReaders(readers, msgchan, &counter, hostname)

	for {
		select {
		case msg, ok := <-msgchan:
			now := time.Now()

			if !ok {
				log.Info("waiting for all write operations to complete")
				limits.Force = true
				flushAll(dests, store, limits, now, join)
				flushQueue(dests, store, logger.Queue, limits, now, join)
				join.Wait()
				return
			}

			_, stream := store.Add(msg, now)
			flush(dests, stream, limits, now, join)

		case <-logger.Queue.C:
			now := time.Now()
			flushQueue(dests, store, logger.Queue, limits, now, join)

		case <-expchan:
			now := time.Now()
			flushAll(dests, store, limits, now, join)
			removeExpired(dests, store, cacheTimeout, now)

		case sig := <-sigchan:
			log.WithFields(log.Fields{"signal": sig.String()}).Info("closing message readers")
			stopReaders(readers)
		}
	}
}

func getSources(names []string) (sources []source) {
	for i, src := range lib.GetSources(names...) {
		sources = append(sources, source{
			Source: src,
			name:   names[i],
		})
	}
	return
}

func getDestinations(names []string) (destinations []destination) {
	for i, dst := range lib.GetDestinations(names...) {
		destinations = append(destinations, destination{
			Destination: dst,
			name:        names[i],
		})
	}
	return
}

func openSources(sources []source) (readers []reader, err error) {
	readers = make([]reader, 0, len(sources))

	for _, source := range sources {
		if r, e := source.Open(); e != nil {
			log.WithFields(log.Fields{
				"source": source.name,
				"error":  e,
			}).Error("failed to open log source")
		} else {
			readers = append(readers, reader{
				Reader: r,
				name:   source.name,
			})
		}
	}

	if len(readers) == 0 {
		err = fmt.Errorf("no sources to read from")
	}

	return
}

func startReaders(readers []reader, msgchan chan<- lib.Message, counter *int32, hostname string) {
	for _, reader := range readers {
		go read(reader, msgchan, counter, hostname)
	}
}

func stopReaders(readers []reader) {
	for _, reader := range readers {
		reader.Close()
	}
}

func term(c chan<- lib.Message, counter *int32) {
	if atomic.AddInt32(counter, -1) == 0 {
		close(c)
	}
}

func read(r reader, c chan<- lib.Message, counter *int32, hostname string) {
	defer term(c, counter)
	for {
		var msg lib.Message
		var err error

		if msg, err = r.ReadMessage(); err != nil {
			if err == io.EOF {
				break
			}
			log.WithFields(log.Fields{
				"reader": r.name,
				"error":  err,
			}).Error("the message reader failed")
			continue
		}

		if len(msg.Group) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "group",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Stream) == 0 {
			log.WithFields(log.Fields{
				"reader":  r.name,
				"missing": "stream",
			}).Warn("dropping message because the a required field wasn't set")
			continue
		}

		if len(msg.Event.Info.Host) == 0 {
			msg.Event.Info.Host = hostname
		}

		if msg.Event.Time == (time.Time{}) {
			msg.Event.Time = time.Now()
		}

		if msg.Event.Data == nil {
			msg.Event.Data = ecslogs.EventData{}
		}

		c <- msg
	}
}

func write(dest destination, group string, stream string, batch lib.MessageBatch, join *sync.WaitGroup) {
	defer join.Done()

	var writer lib.Writer
	var err error

	if writer, err = dest.Open(group, stream); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
	defer writer.Close()

	if err = writer.WriteMessageBatch(batch); err != nil {
		logDropBatch(dest.name, group, stream, err, batch)
		return
	}
}

func flush(dests []destination, stream *lib.Stream, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	for {
		batch, reason := stream.Flush(limits, now)

		if len(batch) == 0 {
			break
		}

		// Ensure all messages in the batch are sorted. Checking if the batch is
		// sorted is an optimization since in most cases the batch will be sorted
		// because we're reading events that are generated live (checking for a
		// sorted list is O(N) vs O(N*log(N)) for sorting it).
		// There are cases where some log entries do appear unordered and this is
		// causing issues with CloudWatchLogs.
		if !sort.IsSorted(batch) {
			sort.Stable(batch)
		}

		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
			"count":  len(batch),
			"reason": reason,
		}).Info("flushing message batch")

		for _, dest := range dests {
			join.Add(1)
			go write(dest, stream.Group(), stream.Name(), batch, join)
		}
	}
}

func flushAll(dests []destination, store *lib.Store, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	store.ForEach(func(group *lib.Group) {
		group.ForEach(func(stream *lib.Stream) {
			flush(dests, stream, limits, now, join)
		})
	})
}

func flushQueue(dests []destination, store *lib.Store, queue *lib.MessageQueue, limits lib.StreamLimits, now time.Time, join *sync.WaitGroup) {
	streams := make(map[string]*lib.Stream)

	for _, msg := range queue.Flush() {
		_, stream := store.Add(msg, now)
		key := stream.Group() + ":" + stream.Name()

		if streams[key] == nil {
			streams[key] = stream
		}
	}

	for _, stream := range streams {
		flush(dests, stream, limits, now, join)
	}
}

func removeExpired(dests []destination, store *lib.Store, cacheTimeout time.Duration, now time.Time) {
	for _, stream := range store.RemoveExpired(cacheTimeout, now) {
		for _, dest := range dests {
			dest.Close(stream.Group(), stream.Name())
		}
		log.WithFields(log.Fields{
			"group":  stream.Group(),
			"stream": stream.Name(),
		}).Info("removed expired stream")
	}
}

func logDropBatch(dest string, group string, stream string, err error, batch lib.MessageBatch) {
	log.WithFields(log.Fields{
		"group":       group,
		"stream":      stream,
		"destination": dest,
		"error":       err,
		"count":       len(batch),
	}).Error("dropping message batch")

	for _, msg := range batch {
		log.WithFields(log.Fields{
			"group":  msg.Group,
			"stream": msg.Stream,
			"event":  msg.Event,
		}).Debug("dropped")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/websocket"
	"github.com/hellodudu/comment/comt"
	"github.com/hellodudu/comment/task"
)

var comtAPI *comt.ComtAPI
var testChan chan interface{} = make(chan interface{}, 1)
var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func callBackTask(ts task.Tasker) {
	log.Println("task callback with reqnum:", ts.GetReq())
	ts.Write([]byte("It is done!"))
	testChan <- 1
}

func taskHandler(w http.ResponseWriter, r *http.Request) {
	comtAPI.AddHttpTask(w, r, callBackTask)
	<-testChan
	log.Printf("taskHandler over\n", w)
}

func wsHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
	}

	for {
		msgtype, p, err := conn.ReadMessage()
		if err != nil {
			log.Println(err)
			return
		}

		res := []byte("server recv:")
		if err := conn.WriteMessage(msgtype, append(res, p...)); err != nil {
			log.Println(err)
			return
		}
	}
}

func createAppHandler(w http.ResponseWriter, r *http.Request) {

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	newApp := &comt.App{}
	if err := json.Unmarshal(body, newApp); err != nil {
		log.Fatal(err)
	}

	// add app
	if err := comtAPI.AddNewApp(newApp); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	retBuf, retErr := json.Marshal(newApp)
	if retErr != nil {
		log.Fatal("create app response json marshal error!")
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(retBuf)
}

func main() {
	var err error
	if comtAPI, err = comt.NewComtAPI(); err != nil {
		log.Fatal(err)
	}
	defer comtAPI.Close()

	http.HandleFunc("/create_app", createAppHandler)
	http.HandleFunc("/task", taskHandler)
	http.HandleFunc("/ws", wsHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<commit_msg>add protocol buffer<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/golang/protobuf/proto"
	"github.com/gorilla/websocket"
	"github.com/hellodudu/comment/comt"
	"github.com/hellodudu/comment/proto"
	"github.com/hellodudu/comment/task"
)

var comtAPI *comt.ComtAPI
var testChan chan interface{} = make(chan interface{}, 1)
var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func callBackTask(ts task.Tasker) {
	log.Println("task callback with reqnum:", ts.GetReq())
	ts.Write([]byte("It is done!"))
	testChan <- 1

	// test proto
	// p := &tutorial.Person{
	// 	Id:    1234,
	// 	Name:  "John Doe",
	// 	Email: "jdoe@example.com",
	// 	Phones: []*tutorial.Person_PhoneNumber{
	// 		{Number: "555-4321", Type: tutorial.Person_HOME},
	// 	},
	// }
	book := &tutorial.AddressBook{}
	out, err := proto.Marshal(book)
	if err != nil {
		log.Fatalln("Failed to encode address book:", err)
	}

	if err := ioutil.WriteFile("address_book", out, 0644); err != nil {
		log.Fatalln("Failed to write address book:", err)
	}
}

func taskHandler(w http.ResponseWriter, r *http.Request) {
	comtAPI.AddHttpTask(w, r, callBackTask)
	<-testChan
	log.Printf("taskHandler over\n", w)
}

func wsHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
	}

	for {
		msgtype, p, err := conn.ReadMessage()
		if err != nil {
			log.Println(err)
			return
		}

		res := []byte("server recv:")
		if err := conn.WriteMessage(msgtype, append(res, p...)); err != nil {
			log.Println(err)
			return
		}
	}
}

func createAppHandler(w http.ResponseWriter, r *http.Request) {

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	newApp := &comt.App{}
	if err := json.Unmarshal(body, newApp); err != nil {
		log.Fatal(err)
	}

	// add app
	if err := comtAPI.AddNewApp(newApp); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	retBuf, retErr := json.Marshal(newApp)
	if retErr != nil {
		log.Fatal("create app response json marshal error!")
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(retBuf)
}

func main() {
	var err error
	if comtAPI, err = comt.NewComtAPI(); err != nil {
		log.Fatal(err)
	}
	defer comtAPI.Close()

	http.HandleFunc("/create_app", createAppHandler)
	http.HandleFunc("/task", taskHandler)
	http.HandleFunc("/ws", wsHandler)
	log.Fatal(http.ListenAndServe(":8080", nil))

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	_ "net/http/pprof" // import for side effects
	"os"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
)

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/hello/{name}", helloHandler)
	r.HandleFunc("/add_job", initiateTranscriptionJobHandler)

	// add middleware
	stderrLoggingHandler := func(http.Handler) http.Handler {
		return handlers.LoggingHandler(os.Stderr, r)
	}
	middlewareRouter := alice.New(handlers.CompressHandler, stderrLoggingHandler).Then(r)

	// serve http
	http.Handle("/", middlewareRouter)
	http.ListenAndServe(":8080", nil)
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	fmt.Fprintf(w, "Hello %s!", args["name"])
}

// initiateTranscriptionJobHandle takes a POST request containing a json object,
// decodes it into an audioData struct, and returns the struct json-encoded.
func initiateTranscriptionJobHandler(w http.ResponseWriter, r *http.Request) {
	var jsonData transcriptionJobData

	// unmarshal from the response body directly into our struct
	if err := json.NewDecoder(r.Body).Decode(&jsonData); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// return the struct encoded as json
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	json.NewEncoder(w).Encode(jsonData)
}

type transcriptionJobData struct {
	AudioURL string `json:"audioURL"`
}
<commit_msg>Changed StatusInternalServerError to StatusBadRequest<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	_ "net/http/pprof" // import for side effects
	"os"

	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
)

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/hello/{name}", helloHandler)
	r.HandleFunc("/add_job", initiateTranscriptionJobHandler)

	// add middleware
	stderrLoggingHandler := func(http.Handler) http.Handler {
		return handlers.LoggingHandler(os.Stderr, r)
	}
	middlewareRouter := alice.New(handlers.CompressHandler, stderrLoggingHandler).Then(r)

	// serve http
	http.Handle("/", middlewareRouter)
	http.ListenAndServe(":8080", nil)
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
	args := mux.Vars(r)
	fmt.Fprintf(w, "Hello %s!", args["name"])
}

// initiateTranscriptionJobHandle takes a POST request containing a json object,
// decodes it into an audioData struct, and returns appropriate message.
func initiateTranscriptionJobHandler(w http.ResponseWriter, r *http.Request) {
	var jsonData transcriptionJobData

	// unmarshal from the response body directly into our struct
	if err := json.NewDecoder(r.Body).Decode(&jsonData); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	fmt.Fprintf(w, "Accepted!")
}

type transcriptionJobData struct {
	AudioURL string `json:"audioURL"`
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/shxsun/goyaml"
)

var quitProgram = false

func renderTemplateFile(outfile string, tmplText string, data interface{}) (err error) {
	//tmpl, err := template.ParseFiles(tmplFile)
	tmpl, err := template.New("render").Parse(tmplText)
	if err != nil {
		return
	}
	fd, err := os.Create(outfile)
	if err != nil {
		return err
	}
	defer fd.Close()
	err = tmpl.Execute(fd, data)
	return
}

func renderWithDefault(tmplFile string, defaultTmpl []byte, data interface{}) (out string, err error) {
	buf := &bytes.Buffer{}
	if !fileExists(tmplFile) {
		ioutil.WriteFile(tmplFile, defaultTmpl, 0644)
	}
	tmpl, err := template.ParseFiles(tmplFile)
	if err != nil {
		return
	}
	err = tmpl.Execute(buf, data)
	return string(buf.Bytes()), err
}

func renderFile(outfile string, defaultTmpl []byte, data interface{}) (err error) {
	//if !fileExists(tmplFile) {
	//	ioutil.WriteFile(tmplFile, defaultTmpl, 0644)
	//}
	// ignore: tmplFile
	return renderTemplateFile(outfile, string(defaultTmpl), data)
}

func dumpFile(filename string, data interface{}) (err error) {
	out, err := goyaml.Marshal(data)
	if err != nil {
		return
	}
	return ioutil.WriteFile(filename, out, 0644)
}

func loadFile(filename string, data interface{}) (err error) {
	raw, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	return goyaml.Unmarshal(raw, data)
}

func ignore(info os.FileInfo) bool {
	if info.IsDir() {
		if info.Name() != "." && info.Name() != ".." &&
			strings.HasPrefix(info.Name(), ".") { // ignore hidden dir
			return true
		}
	} else {
		return strings.HasPrefix(info.Name(), ".")
	}
	return false
}

func currPathCfg(path string) *GlobalConfig {
	return mycnf
}

func pathWalk(path string, depth int) (files []string, err error) {
	files = make([]string, 0)
	path = filepath.Clean(path) // remove extra /
	baseNumSeps := strings.Count(path, string(os.PathSeparator))
	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pathDepth := strings.Count(path, string(os.PathSeparator)) - baseNumSeps
			cfgPath := filepath.Join(path, CONFIG_FILE)
			if fileExists(cfgPath) {
				localCfg := mycnf
				loadFile(cfgPath, localCfg)
			}
			if pathDepth > depth {
				return filepath.SkipDir
			}
			if ignore(info) {
				return filepath.SkipDir
			}
		} else if info.Mode().IsRegular() && !ignore(info) {
			if matched, _ := regexp.Match(mycnf.Include, []byte(info.Name())); matched {
				files = append(files, path)
			}
		}
		return nil
	})
	return
}

type GlobalConfig struct {
	Replacer string `yaml:"-"` //short:"i" description:"replacer"`
	Command  string `short:"c" description:"specify how to process each file"`
	Include  string `short:"I" long:"include-regex" description:"regex set which file can be included"`
	Path     string `short:"p" long:"path" description:"path for search"`
	Depth    int    `short:"d" long:"depth" description:"depth to travel directory tree"`
	Verbose  bool   `short:"v" long:"verbose" description:"show verbose output"`
	Timeout  string `short:"t" long:"timeout" description:"timeout for each exec"`
	Reload   bool   `short:"r" long:"reload" description:"reload all failed cmd, run again:"`
	Exclude  string `yaml:"-"`

	Version  bool     `yaml:"-"`
	InitYaml bool     `yaml:"-" long:"init" description:"create a sample .trival.yml and exit"`
	Result   string   `yaml:"html" long:"html" description:"output result as html"`
	Notify   []string `yaml:"notify" long:"notify" description:"notify people when task finish"`
}

var mycnf = &GlobalConfig{
	Depth:    0,
	Path:     "./",
	Replacer: "{}",
	Version:  false,
	Result:   "test.html",
	Timeout:  "30m",
	Command:  "./{}",
	Include:  ".*",
	Verbose:  false,
}

func init() {
	if fileExists(CONFIG_FILE) {
		loadFile(CONFIG_FILE, mycnf)
	}
	args, err := flags.Parse(mycnf)
	_, _ = args, err
	if err != nil {
		if er, ok := err.(*flags.Error); ok && er.Type == flags.ErrHelp {
			os.Exit(0)
		}
		log.Fatal(err)
	}
	if mycnf.InitYaml {
		dumpFile(CONFIG_FILE, mycnf)
		os.Exit(0)
	}
	if mycnf.Version {
		fmt.Println(VERSION)
		os.Exit(0)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

type TaskConfig struct {
	Replacer string
	Command  string
	Timeout  time.Duration
	Files    []string
}

type TaskResult struct {
	StartTime string
	TimeCost  time.Duration
	Command   string
	Filename  string
	Output    string // console out
	Source    string // source code
	Error     error
}

func Go(f func() error) chan error {
	ch := make(chan error)
	go func() {
		err := f()
		ch <- err
	}()
	return ch
}

func groupKill(cmd *exec.Cmd) (err error) {
	var pid, pgid int
	if cmd.Process != nil {
		pid = cmd.Process.Pid
		c := exec.Command("/bin/ps", "-o", "pgid", "-p", strconv.Itoa(pid), "--no-header")
		var out []byte
		out, err = c.Output()
		if err != nil {
			return
		}
		_, err = fmt.Sscanf(string(out), "%d", &pgid)
		if err != nil {
			return
		}
		err = exec.Command("/bin/kill", "-TERM", "-"+strconv.Itoa(pgid)).Run()
	}
	return
}

func work(cfg *TaskConfig) (results []TaskResult) {
	var err error
	results = []TaskResult{}
	for i, file := range cfg.Files {
		start := time.Now()
		r := TaskResult{
			StartTime: start.Format("15:04:05"),
			Filename:  file,
		}
		c := strings.Replace(cfg.Command, cfg.Replacer, file, -1)
		r.Command = c
		if quitProgram {
			r.Error = errors.New("not started, skiped for SIGINT interrupt")
			results = append(results, r)
			break
		}

		prefix := fmt.Sprintf("\r\033[36m>>>\033[0m %-5d", i)
		format := fmt.Sprintf(prefix+"%%-14v %-24s\n", c) //file) //file)
		// show current exec file
		fmt.Printf(prefix+"exec %s ...", strconv.Quote(file))
		if mycnf.Verbose {
			fmt.Printf("\n")
		}

		output := bytes.NewBuffer(nil)
		cmd := exec.Command("/bin/bash", "-c", c)
		if mycnf.Verbose {
			cmd.Stdout = io.MultiWriter(os.Stdout, output)
			cmd.Stderr = io.MultiWriter(os.Stderr, output)
		} else {
			cmd.Stdout = output
			cmd.Stderr = output
		}
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Setpgid = true

		// handle timeout
		if err = cmd.Start(); err == nil {
			select {
			case <-sigC:
				quitProgram = true
				groupKill(cmd)
				err = errors.New("signal: interrupt")
			case <-time.After(cfg.Timeout):
				//cmd.Process.Kill()
				groupKill(cmd)
				err = errors.New("signal: terminated")
			case err = <-Go(cmd.Wait):
			}
		}

		if err != nil {
			r.Error = err
			fmt.Printf(format, fmt.Sprintf("\033[33m"+"err: %s"+"\033[0m", err))
		} else {
			fmt.Printf(format, "\033[32m"+"success"+"\033[0m")
		}
		r.TimeCost = time.Now().Sub(start)
		r.Output = string(output.Bytes())
		r.Source = "unfinished(todo)"
		results = append(results, r)
	}
	return
}

func fileExists(filename string) bool {
	fi, err := os.Stat(filename)
	return err == nil && fi.Mode().IsRegular()
}

func selfPath() string {
	return filepath.Dir(os.Args[0])
}

const STATE_FILE = "state-travel.yml"
const CONFIG_FILE = ".travel.yml"

var cfgTree = map[string]*GlobalConfig{}

func main() {
	var err error
	var taskcfg = &TaskConfig{}
	var startTime = time.Now()

	cfgPath := filepath.Join(mycnf.Path, CONFIG_FILE)
	loadFile(cfgPath, mycnf)

	if mycnf.Reload && fileExists(STATE_FILE) {
		err = loadFile(STATE_FILE, taskcfg)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		files, err := pathWalk(mycnf.Path, mycnf.Depth)
		if err != nil {
			log.Fatal(err)
		}
		taskcfg.Command = mycnf.Command
		taskcfg.Replacer = mycnf.Replacer
		timeout, err := time.ParseDuration(mycnf.Timeout)
		if err != nil {
			log.Fatal(err)
		}
		taskcfg.Timeout = timeout
		taskcfg.Files = files
	}

	results := work(taskcfg)
	errfiles := []string{}
	errCnt, totCnt := 0, 0
	for _, r := range results {
		if r.Error != nil {
			errCnt++
			errfiles = append(errfiles, r.Filename)
		}
		totCnt++
	}
	defer func() {
		if errCnt != 0 {
			os.Exit(13) // 13 is a lucky number
		}
	}()

	// reder html
	data := map[string]interface{}{}
	data["StartTime"] = startTime.Format("2006-01-02 15:04:05")
	endTime := time.Now()
	data["TimeCost"] = endTime.Sub(startTime)
	data["Tasks"] = results
	hostname, _ := os.Hostname()
	data["Host"] = hostname
	data["Total"] = len(taskcfg.Files)
	data["FailCount"] = errCnt

	//htmlTmplPath := filepath.Join(selfPath(), ".html.tmpl")
	err = renderTemplateFile(mycnf.Result, defaultTemplate, data)
	if err != nil {
		log.Fatal(err)
	}
	htmlData, _ := ioutil.ReadFile(mycnf.Result)

	// save to restart again
	taskcfg.Files = errfiles
	err = dumpFile(STATE_FILE, taskcfg)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("summary: (total: %d fail: %d)\n", totCnt, errCnt)
	smsTmplPath := filepath.Join(selfPath(), ".sms.tmpl")
	if !quitProgram && len(mycnf.Notify) != 0 {
		msg, err := renderWithDefault(smsTmplPath, defaultSMSTemplate, data)
		if err != nil {
			log.Fatal(err)
		}
		err = sendNotify(msg, htmlData, mycnf.Notify...)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>add max fail support<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"text/template"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/shxsun/goyaml"
)

var quitProgram = false

func renderTemplateFile(outfile string, tmplText string, data interface{}) (err error) {
	//tmpl, err := template.ParseFiles(tmplFile)
	tmpl, err := template.New("render").Parse(tmplText)
	if err != nil {
		return
	}
	fd, err := os.Create(outfile)
	if err != nil {
		return err
	}
	defer fd.Close()
	err = tmpl.Execute(fd, data)
	return
}

func renderWithDefault(tmplFile string, defaultTmpl []byte, data interface{}) (out string, err error) {
	buf := &bytes.Buffer{}
	if !fileExists(tmplFile) {
		ioutil.WriteFile(tmplFile, defaultTmpl, 0644)
	}
	tmpl, err := template.ParseFiles(tmplFile)
	if err != nil {
		return
	}
	err = tmpl.Execute(buf, data)
	return string(buf.Bytes()), err
}

func renderFile(outfile string, defaultTmpl []byte, data interface{}) (err error) {
	//if !fileExists(tmplFile) {
	//	ioutil.WriteFile(tmplFile, defaultTmpl, 0644)
	//}
	// ignore: tmplFile
	return renderTemplateFile(outfile, string(defaultTmpl), data)
}

func dumpFile(filename string, data interface{}) (err error) {
	out, err := goyaml.Marshal(data)
	if err != nil {
		return
	}
	return ioutil.WriteFile(filename, out, 0644)
}

func loadFile(filename string, data interface{}) (err error) {
	raw, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	return goyaml.Unmarshal(raw, data)
}

func ignore(info os.FileInfo) bool {
	if info.IsDir() {
		if info.Name() != "." && info.Name() != ".." &&
			strings.HasPrefix(info.Name(), ".") { // ignore hidden dir
			return true
		}
	} else {
		return strings.HasPrefix(info.Name(), ".")
	}
	return false
}

func currPathCfg(path string) *GlobalConfig {
	return mycnf
}

func pathWalk(path string, depth int) (files []string, err error) {
	files = make([]string, 0)
	path = filepath.Clean(path) // remove extra /
	baseNumSeps := strings.Count(path, string(os.PathSeparator))
	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			pathDepth := strings.Count(path, string(os.PathSeparator)) - baseNumSeps
			cfgPath := filepath.Join(path, CONFIG_FILE)
			if fileExists(cfgPath) {
				localCfg := mycnf
				loadFile(cfgPath, localCfg)
			}
			if pathDepth > depth {
				return filepath.SkipDir
			}
			if ignore(info) {
				return filepath.SkipDir
			}
		} else if info.Mode().IsRegular() && !ignore(info) {
			if matched, _ := regexp.Match(mycnf.Include, []byte(info.Name())); matched {
				files = append(files, path)
			}
		}
		return nil
	})
	return
}

type GlobalConfig struct {
	Replacer string `yaml:"-"` //short:"i" description:"replacer"`
	Command  string `short:"c" description:"specify how to process each file"`
	Include  string `short:"I" long:"include-regex" description:"regex set which file can be included"`
	MaxFail  int    `long:"max-fail" description:"max failed case"`
	Path     string `short:"p" long:"path" description:"path for search"`
	Depth    int    `short:"d" long:"depth" description:"depth to travel directory tree"`
	Verbose  bool   `short:"v" long:"verbose" description:"show verbose output"`
	Timeout  string `short:"t" long:"timeout" description:"timeout for each exec"`
	Reload   bool   `short:"r" long:"failed" description:"reload all failed cmd, run again:"`
	Exclude  string `yaml:"-"`

	Version  bool     `yaml:"-"`
	InitYaml bool     `yaml:"-" long:"init" description:"create a sample .trival.yml and exit"`
	Result   string   `yaml:"html" long:"html" description:"output result as html"`
	Notify   []string `yaml:"notify" long:"notify" description:"notify people when task finish"`
}

var mycnf = &GlobalConfig{
	Depth:    0,
	Path:     "./",
	Replacer: "{}",
	Version:  false,
	Result:   "test.html",
	Timeout:  "30m",
	Command:  "./{}",
	Include:  ".*",
	Verbose:  false,
}

func init() {
	if fileExists(CONFIG_FILE) {
		loadFile(CONFIG_FILE, mycnf)
	}
	args, err := flags.Parse(mycnf)
	_, _ = args, err
	if err != nil {
		if er, ok := err.(*flags.Error); ok && er.Type == flags.ErrHelp {
			os.Exit(0)
		}
		log.Fatal(err)
	}
	if mycnf.InitYaml {
		dumpFile(CONFIG_FILE, mycnf)
		os.Exit(0)
	}
	if mycnf.Version {
		fmt.Println(VERSION)
		os.Exit(0)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

type TaskConfig struct {
	Replacer string
	Command  string
	Timeout  time.Duration
	Files    []string
}

type TaskResult struct {
	StartTime string
	TimeCost  time.Duration
	Command   string
	Filename  string
	Output    string // console out
	Source    string // source code
	Error     error
}

func Go(f func() error) chan error {
	ch := make(chan error)
	go func() {
		err := f()
		ch <- err
	}()
	return ch
}

func groupKill(cmd *exec.Cmd) (err error) {
	var pid, pgid int
	if cmd.Process != nil {
		pid = cmd.Process.Pid
		c := exec.Command("/bin/ps", "-o", "pgid", "-p", strconv.Itoa(pid), "--no-header")
		var out []byte
		out, err = c.Output()
		if err != nil {
			return
		}
		_, err = fmt.Sscanf(string(out), "%d", &pgid)
		if err != nil {
			return
		}
		err = exec.Command("/bin/kill", "-TERM", "-"+strconv.Itoa(pgid)).Run()
	}
	return
}

func work(cfg *TaskConfig) (results []TaskResult) {
	var err error
	var failcnt = 0
	results = []TaskResult{}
	for i, file := range cfg.Files {
		start := time.Now()
		r := TaskResult{
			StartTime: start.Format("15:04:05"),
			Filename:  file,
		}
		c := strings.Replace(cfg.Command, cfg.Replacer, file, -1)
		r.Command = c
		if mycnf.MaxFail != 0 && failcnt >= mycnf.MaxFail {
			quitProgram = true
		}
		if quitProgram {
			r.Error = errors.New("skip test")
			results = append(results, r)
			break
		}

		prefix := fmt.Sprintf("\r\033[36m>>>\033[0m %-5d", i)
		format := fmt.Sprintf(prefix+"%%-14v %-24s\n", c) //file) //file)
		// show current exec file
		fmt.Printf(prefix+"exec %s ...", strconv.Quote(file))
		if mycnf.Verbose {
			fmt.Printf("\n")
		}

		output := bytes.NewBuffer(nil)
		cmd := exec.Command("/bin/bash", "-c", c)
		if mycnf.Verbose {
			cmd.Stdout = io.MultiWriter(os.Stdout, output)
			cmd.Stderr = io.MultiWriter(os.Stderr, output)
		} else {
			cmd.Stdout = output
			cmd.Stderr = output
		}
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Setpgid = true

		// handle timeout
		if err = cmd.Start(); err == nil {
			select {
			case <-sigC:
				quitProgram = true
				groupKill(cmd)
				err = errors.New("signal: interrupt")
			case <-time.After(cfg.Timeout):
				//cmd.Process.Kill()
				groupKill(cmd)
				err = errors.New("signal: terminated")
			case err = <-Go(cmd.Wait):
			}
		}

		if err != nil {
			r.Error = err
			failcnt += 1
			fmt.Printf(format, fmt.Sprintf("\033[33m"+"err: %s"+"\033[0m", err))
		} else {
			fmt.Printf(format, "\033[32m"+"success"+"\033[0m")
		}
		r.TimeCost = time.Now().Sub(start)
		r.Output = string(output.Bytes())
		r.Source = "unfinished(todo)"
		results = append(results, r)
	}
	return
}

func fileExists(filename string) bool {
	fi, err := os.Stat(filename)
	return err == nil && fi.Mode().IsRegular()
}

func selfPath() string {
	return filepath.Dir(os.Args[0])
}

const STATE_FILE = "state-travel.yml"
const CONFIG_FILE = ".travel.yml"

var cfgTree = map[string]*GlobalConfig{}

func main() {
	var err error
	var taskcfg = &TaskConfig{}
	var startTime = time.Now()

	cfgPath := filepath.Join(mycnf.Path, CONFIG_FILE)
	loadFile(cfgPath, mycnf)

	if mycnf.Reload && fileExists(STATE_FILE) {
		err = loadFile(STATE_FILE, taskcfg)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		files, err := pathWalk(mycnf.Path, mycnf.Depth)
		if err != nil {
			log.Fatal(err)
		}
		taskcfg.Command = mycnf.Command
		taskcfg.Replacer = mycnf.Replacer
		timeout, err := time.ParseDuration(mycnf.Timeout)
		if err != nil {
			log.Fatal(err)
		}
		taskcfg.Timeout = timeout
		taskcfg.Files = files
	}

	results := work(taskcfg)
	errfiles := []string{}
	errCnt, totCnt := 0, 0
	for _, r := range results {
		if r.Error != nil {
			errCnt++
			errfiles = append(errfiles, r.Filename)
		}
		totCnt++
	}
	defer func() {
		if errCnt != 0 {
			os.Exit(13) // 13 is a lucky number
		}
	}()

	// reder html
	data := map[string]interface{}{}
	data["StartTime"] = startTime.Format("2006-01-02 15:04:05")
	endTime := time.Now()
	data["TimeCost"] = endTime.Sub(startTime)
	data["Tasks"] = results
	hostname, _ := os.Hostname()
	data["Host"] = hostname
	data["Total"] = len(taskcfg.Files)
	data["FailCount"] = errCnt

	//htmlTmplPath := filepath.Join(selfPath(), ".html.tmpl")
	err = renderTemplateFile(mycnf.Result, defaultTemplate, data)
	if err != nil {
		log.Fatal(err)
	}
	htmlData, _ := ioutil.ReadFile(mycnf.Result)

	// save to restart again
	taskcfg.Files = errfiles
	err = dumpFile(STATE_FILE, taskcfg)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("summary: (total: %d fail: %d)\n", totCnt, errCnt)
	smsTmplPath := filepath.Join(selfPath(), ".sms.tmpl")
	if !quitProgram && len(mycnf.Notify) != 0 {
		msg, err := renderWithDefault(smsTmplPath, defaultSMSTemplate, data)
		if err != nil {
			log.Fatal(err)
		}
		err = sendNotify(msg, htmlData, mycnf.Notify...)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
)

/*
	c	matches any literal character c
	.	matches any single character
	^	matches the beginning of the input string
	$	matches the end of the input string
	*	matches zero or more occurrences of the previous character

	3 concurrent stages
		1. Syntax Sieve
		2. Convert to reverse polish form
		3. Object code producer
*/

const (
	dollar = iota + 128
	star
	dot
	caret
	concat
)

func ascii(b byte) bool {
	return b == '\x00' || (b >= '\x07' && b <= '\x0D') || b == '\x1B' || (b >= '\x20' && b <= '\x7E')
}

func quantifiable(b byte) bool {
	return b != dollar && b != star && b != caret
}

func escape(b byte) byte {
	switch b {
	case '0':
		return '\x00'
	case 'a':
		return '\x07'
	case 'b':
		return '\x08'
	case 't':
		return '\x09'
	case 'n':
		return '\x0A'
	case 'v':
		return '\x0B'
	case 'f':
		return '\x0C'
	case 'r':
		return '\x0D'
	case 'e':
		return '\x1B'
	case '\\':
		return '\x5C'
	default:
		return b
	}

}

func meta(b byte) byte {
	switch b {
	case '$':
		return dollar
	case '*':
		return star
	case '.':
		return dot
	case '^':
		return caret
	default:
		return b
	}
}

type expression []byte

func (e expression) String() string {
	s := ""
	for _, v := range e {
		switch v {
		case dollar:
			s += "$"
		case star:
			s += "*"
		case dot:
			s += "."
		case caret:
			s += "^"
		case concat:
			s += " (concat) "
		case '$':
			s += "\\$"
		case '*':
			s += "\\*"
		case '.':
			s += "\\."
		case '^':
			s += "\\^"
		default:
			s += string(v)
		}
	}
	return s
}

// Syntactically validates the input expression.
// Either exits with a syntax error, or returns
// the expression with the "." to juxtapose expressions.
func validate(input string) expression {
	expr := make([]byte, 0, len(input))
	var prev byte
	var b byte
	for i := range input {
		b = input[i]
		if !ascii(b) {
			// check that b is a printable ascii character, otherwise log and exit
			log.Fatalln(b, "is not a printable ascii character")
		}
		if !quantifiable(prev) && b == '\x2A' {
			// if * is preceded by a non-quantifiable character, log and exit
			log.Fatalln("preceding token", prev, "is not quantifiable")
		}
		if prev == '\x5C' {
			// if the previous character in the expression was a backslash,
			// pop it and replace it with the escaped version of b
			expr = append(expr[:len(expr)-1], escape(b))
		} else if (i == len(input)-1) && (b == '\x5C') {
			// otherwise, if the last character is a backslash, log and exit
			log.Fatalln("cannot end with trailing backslash")
		} else {
			// otherwise, append all characters to the expression, taking into
			// account metacharacters like $, *, ., and ^.
			expr = append(expr, meta(b))
		}
		prev = expr[len(expr)-1]
		if prev == star {
			l := len(expr) - 2
			expr = append(expr, 0)
			copy(expr[l+1:], expr[l:])
			expr[l] = concat
		}
	}
	return expression(expr)
}

func main() {
	fmt.Println(validate(`a\**`))
}
<commit_msg>tpop robpike version<commit_after>package main

import "fmt"

/*
	c	matches any literal character c
	.	matches any single character
	^	matches the beginning of the input string
	$	matches the end of the input string
	*	matches zero or more occurrences of the previous character
*/

/* match: search for regexp anywhere in text */
func match(regexp, text []byte) bool {
	if regexp[0] == '^' {
		return matchhere(regexp[1:], text)
	}
	for i := range text {
		if matchhere(regexp, text[i:]) {
			return true
		}
	}
	return false
}

/* matchhere: search for regexp at beginning of text */
func matchhere(regexp, text []byte) bool {
	if len(regexp) == 0 {
		return true
	}
	if len(regexp) > 1 && regexp[1] == '*' {
		return matchstar(regexp[0], regexp[2:], text)
	}
	if regexp[0] == '$' && len(regexp) == 1 {
		return len(text) == 0
	}
	if len(text) != 0 && (regexp[0] == '.' || regexp[0] == text[0]) {
		return matchhere(regexp[1:], text[1:])
	}
	return false
}

/* matchstar: search for c*regexp at beginning of text */
func matchstar(c byte, regexp, text []byte) bool {
	i := 0
	for {
		if matchhere(regexp, text[i:]) {
			return true
		}
		if i == len(text) || (text[i] != c && c != '.') {
			break
		}
		i++
	}
	return false
}

// assumes input is correct
func main() {
	regexp := "da*b"
	text := "bcccb"
	fmt.Println(match([]byte(regexp), []byte(text)))
}
<|endoftext|>