<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/miekg/dns"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
)

var (
	http_client = http.Client{}
	dns_client  = dns.Client{}
)

type HTTPError int

func (e HTTPError) Error() string {
	return fmt.Sprintf("HTTP %d %s", e, http.StatusText(e.Code()))
}
func (e HTTPError) Code() int {
	return int(e)
}

func should_cache(path string) bool {
	if strings.HasSuffix(path, ".pkg.tar.xz") {
		return true
	}
	if strings.HasSuffix(path, ".rpm") {
		return true
	}
	if strings.Contains(path, "/repodata/") && (strings.HasSuffix(path, ".gz") ||
		strings.HasSuffix(path, ".bz2") || strings.HasSuffix(path, ".xz")) {
		return true
	}
	return false
}

func main() {

	var (
		listen string
		data   string
		host   string
	)

	flag.StringVar(&listen, "listen", ":80", "HTTP listen address")
	flag.StringVar(&data, "data", "/var/remirror", "Data storage path (data in here is public)")
	flag.StringVar(&host, "host", "9ex-dc-mirror", "This hosts name, so we can return a mirrorlist with ourselves")

	flag.Parse()

	fileserver := http.FileServer(http.Dir(data))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

		log.Println(r.Method + " http://" + r.Host + r.RequestURI)

		err := func() error {

			// Some special sauce mirrorlist handlers that will point to ourselves
			if r.Host == "mirrors.fedoraproject.org" {
				return fedora_mirrorlist(w, r, host)
			}
			if r.Host == "mirrorlist.centos.org" {
				return centos_mirrorlist(w, r, host)
			}

			// Now we guess the upstream from the URL
			upstream := ""

			if strings.HasPrefix(r.URL.Path, "/archlinux/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/centos/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/fedora/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/fedora-epel/") {
				upstream = "https://mirrors.xmission.com"
			}

			if upstream == "" {
				fmt.Println("no upstream found for url", r.URL.Path)
				return HTTPError(404)
			}

			local_path := ""

			if should_cache(r.URL.Path) {
				local_path = data + path.Clean(r.URL.Path)

				_, err := os.Stat(local_path)
				if err == nil {
					fileserver.ServeHTTP(w, r)
					return nil
				}
			}

			log.Println("-->", upstream+r.RequestURI)

			req, err := http.NewRequest("GET", upstream+r.RequestURI, nil)
			if err != nil {
				return err
			}

			for k, vs := range r.Header {
				if k != "Host" {
					for _, v := range vs {
						req.Header.Add(k, v)
					}
				}
			}

			resp, err := http_client.Do(req)
			if err != nil {
				return err
			}
			defer resp.Body.Close()

			out := io.Writer(w)

			tmp_path := ""

			if resp.StatusCode == 200 && local_path != "" {
				tmp, err := ioutil.TempFile(data, "remirror_tmp_")
				if err != nil {
					return err
				}
				tmp_path = tmp.Name()
				//fmt.Println("tmp", tmp_path)

				defer tmp.Close()
				defer os.Remove(tmp_path)

				out = io.MultiWriter(out, tmp)
			}

			for k, vs := range resp.Header {
				if k == "Accept-Ranges" {
					continue
				}
				for _, v := range vs {
					//fmt.Printf("proxy back header %#v\t%#v\n", k, v)
					w.Header().Add(k, v)
				}
			}

			w.Header().Set("Server", "remirror")
			w.WriteHeader(resp.StatusCode)

			n, err := io.Copy(out, resp.Body)
			if err != nil {
				log.Println(err)
				return nil
			}

			if n != resp.ContentLength {
				log.Printf("Short data returned from server (Content-Length %d received %d)\n", resp.ContentLength, n)
				return nil
			}

			if tmp_path != "" {
				os.MkdirAll(path.Dir(local_path), 0755)

				err = os.Rename(tmp_path, local_path)
				if err != nil {
					log.Println(err)
					return nil
				}
				log.Println(">:)")
			}

			return nil
		}()

		he, ok := err.(HTTPError)
		if ok {
			http.Error(w, he.Error(), he.Code())
			fmt.Println("\t\t", he.Error())
		} else if err != nil {
			http.Error(w, err.Error(), 500)
			fmt.Println("\t\t500 " + err.Error())
		}
	})

	log.Println("arch/fedora/centos mirror proxy listening on HTTP " + listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func centos_mirrorlist(w http.ResponseWriter, r *http.Request, host string) error {
	err := r.ParseForm()
	if err != nil {
		return err
	}

	release := r.Form.Get("release")
	repo := r.Form.Get("repo")
	arch := r.Form.Get("arch")

	if release == "7" {
		release = "7.2.1511"
	}

	w.Header().Set("Content-Type", "text/plain")

	us := "http://" + host + "/centos/" + release + "/" + repo + "/" + arch + "/"

	if _, err := io.WriteString(w, us); err != nil {
		log.Println(err)
		return nil
	}

	log.Println("===", us)
	return nil
}

func fedora_mirrorlist(w http.ResponseWriter, r *http.Request, host string) error {

	err := r.ParseForm()
	if err != nil {
		return err
	}

	repo := r.Form.Get("repo")
	arch := r.Form.Get("arch")

	upstream := "https://mirrors.fedoraproject.org" + r.RequestURI

	log.Println("---", upstream)

	req, err := http.NewRequest("GET", upstream, nil)
	if err != nil {
		return err
	}

	resp, err := http_client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return HTTPError(resp.StatusCode)
	}

	tmp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	s := string(tmp)

	start := strings.Index(s, `<resources maxconnections="1">`)
	end := strings.Index(s, `</resources>`)

	us := ""

	if start != -1 && end != -1 && repo == "epel-7" {
		us = `http://` + host + `/fedora-epel/7/` + arch + `/repodata/repomd.xml`
		s = s[:start] +
			`<resources maxconnections="1"><url protocol="http" type="http" location="US" preference="100">` +
			us +
			`</url>` +
			s[end:]
	}

	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	w.Header().Set("Content-Length", strconv.Itoa(len(s)))
	w.WriteHeader(200)
	if _, err := io.WriteString(w, s); err != nil {
		log.Println(err)
		return nil
	}

	if us != "" {
		log.Println("===", us)
	}
	return nil
}
<commit_msg>Remove vestigial dependency<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
)

var (
	http_client = http.Client{}
)

type HTTPError int

func (e HTTPError) Error() string {
	return fmt.Sprintf("HTTP %d %s", e, http.StatusText(e.Code()))
}
func (e HTTPError) Code() int {
	return int(e)
}

func should_cache(path string) bool {
	if strings.HasSuffix(path, ".pkg.tar.xz") {
		return true
	}
	if strings.HasSuffix(path, ".rpm") {
		return true
	}
	if strings.Contains(path, "/repodata/") && (strings.HasSuffix(path, ".gz") ||
		strings.HasSuffix(path, ".bz2") || strings.HasSuffix(path, ".xz")) {
		return true
	}
	return false
}

func main() {

	var (
		listen string
		data   string
		host   string
	)

	flag.StringVar(&listen, "listen", ":80", "HTTP listen address")
	flag.StringVar(&data, "data", "/var/remirror", "Data storage path (data in here is public)")
	flag.StringVar(&host, "host", "9ex-dc-mirror", "This hosts name, so we can return a mirrorlist with ourselves")

	flag.Parse()

	fileserver := http.FileServer(http.Dir(data))

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

		log.Println(r.Method + " http://" + r.Host + r.RequestURI)

		err := func() error {

			// Some special sauce mirrorlist handlers that will point to ourselves
			if r.Host == "mirrors.fedoraproject.org" {
				return fedora_mirrorlist(w, r, host)
			}
			if r.Host == "mirrorlist.centos.org" {
				return centos_mirrorlist(w, r, host)
			}

			// Now we guess the upstream from the URL
			upstream := ""

			if strings.HasPrefix(r.URL.Path, "/archlinux/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/centos/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/fedora/") {
				upstream = "https://mirrors.xmission.com"
			} else if strings.HasPrefix(r.URL.Path, "/fedora-epel/") {
				upstream = "https://mirrors.xmission.com"
			}

			if upstream == "" {
				fmt.Println("no upstream found for url", r.URL.Path)
				return HTTPError(404)
			}

			local_path := ""

			if should_cache(r.URL.Path) {
				local_path = data + path.Clean(r.URL.Path)

				_, err := os.Stat(local_path)
				if err == nil {
					fileserver.ServeHTTP(w, r)
					return nil
				}
			}

			log.Println("-->", upstream+r.RequestURI)

			req, err := http.NewRequest("GET", upstream+r.RequestURI, nil)
			if err != nil {
				return err
			}

			for k, vs := range r.Header {
				if k != "Host" {
					for _, v := range vs {
						req.Header.Add(k, v)
					}
				}
			}

			resp, err := http_client.Do(req)
			if err != nil {
				return err
			}
			defer resp.Body.Close()

			out := io.Writer(w)

			tmp_path := ""

			if resp.StatusCode == 200 && local_path != "" {
				tmp, err := ioutil.TempFile(data, "remirror_tmp_")
				if err != nil {
					return err
				}
				tmp_path = tmp.Name()
				//fmt.Println("tmp", tmp_path)

				defer tmp.Close()
				defer os.Remove(tmp_path)

				out = io.MultiWriter(out, tmp)
			}

			for k, vs := range resp.Header {
				if k == "Accept-Ranges" {
					continue
				}
				for _, v := range vs {
					//fmt.Printf("proxy back header %#v\t%#v\n", k, v)
					w.Header().Add(k, v)
				}
			}

			w.Header().Set("Server", "remirror")
			w.WriteHeader(resp.StatusCode)

			n, err := io.Copy(out, resp.Body)
			if err != nil {
				log.Println(err)
				return nil
			}

			if n != resp.ContentLength {
				log.Printf("Short data returned from server (Content-Length %d received %d)\n", resp.ContentLength, n)
				return nil
			}

			if tmp_path != "" {
				os.MkdirAll(path.Dir(local_path), 0755)

				err = os.Rename(tmp_path, local_path)
				if err != nil {
					log.Println(err)
					return nil
				}
				log.Println(">:)")
			}

			return nil
		}()

		he, ok := err.(HTTPError)
		if ok {
			http.Error(w, he.Error(), he.Code())
			fmt.Println("\t\t", he.Error())
		} else if err != nil {
			http.Error(w, err.Error(), 500)
			fmt.Println("\t\t500 " + err.Error())
		}
	})

	log.Println("arch/fedora/centos mirror proxy listening on HTTP " + listen)
	log.Fatal(http.ListenAndServe(listen, nil))
}

func centos_mirrorlist(w http.ResponseWriter, r *http.Request, host string) error {
	err := r.ParseForm()
	if err != nil {
		return err
	}

	release := r.Form.Get("release")
	repo := r.Form.Get("repo")
	arch := r.Form.Get("arch")

	if release == "7" {
		release = "7.2.1511"
	}

	w.Header().Set("Content-Type", "text/plain")

	us := "http://" + host + "/centos/" + release + "/" + repo + "/" + arch + "/"

	if _, err := io.WriteString(w, us); err != nil {
		log.Println(err)
		return nil
	}

	log.Println("===", us)
	return nil
}

func fedora_mirrorlist(w http.ResponseWriter, r *http.Request, host string) error {

	err := r.ParseForm()
	if err != nil {
		return err
	}

	repo := r.Form.Get("repo")
	arch := r.Form.Get("arch")

	upstream := "https://mirrors.fedoraproject.org" + r.RequestURI

	log.Println("---", upstream)

	req, err := http.NewRequest("GET", upstream, nil)
	if err != nil {
		return err
	}

	resp, err := http_client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return HTTPError(resp.StatusCode)
	}

	tmp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	s := string(tmp)

	start := strings.Index(s, `<resources maxconnections="1">`)
	end := strings.Index(s, `</resources>`)

	us := ""

	if start != -1 && end != -1 && repo == "epel-7" {
		us = `http://` + host + `/fedora-epel/7/` + arch + `/repodata/repomd.xml`
		s = s[:start] +
			`<resources maxconnections="1"><url protocol="http" type="http" location="US" preference="100">` +
			us +
			`</url>` +
			s[end:]
	}

	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	w.Header().Set("Content-Length", strconv.Itoa(len(s)))
	w.WriteHeader(200)
	if _, err := io.WriteString(w, s); err != nil {
		log.Println(err)
		return nil
	}

	if us != "" {
		log.Println("===", us)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
七牛本地上传客户端
$ qn_cli --help
*/
package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"golang.org/x/net/context"
	"qiniupkg.com/api.v7/kodo"
	"qiniupkg.com/api.v7/kodocli"
)

var ignorePaths = []string{
	".git", ".hg", ".svn", ".module-cache", ".bin",
}

type stringSlice []string

func (s *stringSlice) String() string {
	return fmt.Sprintf("%s", *s)
}
func (s *stringSlice) Set(value string) error {
	*s = append(*s, value)
	return nil
}

// 生成上传 token
func genUpToken(a *args, c *kodo.Client, key string) string {
	policy := kodo.PutPolicy{
		Scope: a.bucketName,
		// ReturnBody: `{"bucket": $(bucket),"key": $(key)}`,
		DetectMime: 1,
	}
	if key != "" {
		policy.SaveKey = key
		if a.overwrite {
			policy.Scope = policy.Scope + ":" + key
			policy.InsertOnly = 0
		}
	}
	return c.MakeUptoken(&policy)
}

// 上传本地文件
func uploadFile(
	uploader kodocli.Uploader, ctx context.Context, localFile, key, uptoken string) (ret *kodocli.PutRet, err error) {
	ret = &kodocli.PutRet{}
	if key == "" {
		err = uploader.PutFileWithoutKey(ctx, ret, uptoken, localFile, nil)
	} else {
		err = uploader.PutFile(ctx, ret, uptoken, key, localFile, nil)
	}
	return
}

// 自动生成文件名
func autoFileName(p string) (string, string, string) {
	dirname, name := path.Split(p)
	ext := path.Ext(name)
	return dirname, name, ext
}
func autoMD5FileName(p string) string {
	dirname, oldName, ext := autoFileName(p)
	now := int(time.Now().Nanosecond())
	hash := md5.Sum([]byte(
		strconv.Itoa(now),
	))
	newName := dirname + oldName + "_" + hex.EncodeToString(hash[:]) + ext
	return newName
}

func finalURL(bucketURL, key string) (url string) {
	return bucketURL + key
}

type args struct {
	bucketName  string
	bucketURL   string
	fileSlice   []string
	key         string
	autoName    bool
	autoMD5Name bool
	overwrite   bool
	saveDir     string
	verbose     bool
}

func parseArgs() *args {
	// 保存名称
	saveName := flag.String("n", "", "Save name")
	saveDir := flag.String("d", "", "Save dirname")
	autoName := flag.Bool("a", true, "Auto named saved files")
	autoMD5Name := flag.Bool("md5", false, "Auto named saved files use MD5 value")
	overwrite := flag.Bool("w", true, "Overwrite exists files")
	verbose := flag.Bool("v", false, "Verbose mode")
	var ignores stringSlice
	flag.Var(&ignores, "i", "ignores")

	flag.Parse()
	files := flag.Args()

	bucketName := os.Getenv("QINIU_BUCKET_NAME")
	bucketURL := os.Getenv("QINIU_BUCKET_URL")
	accessKey := os.Getenv("QINIU_ACCESS_KEY")
	secretKey := os.Getenv("QINIU_SECRET_KEY")
	if *verbose {
		fmt.Printf("bucketName: %s\n", bucketName)
		fmt.Printf("bucketURL: %s\n", bucketURL)
		fmt.Printf("accessKey: %s\n", accessKey)
		fmt.Printf("secretKey: %s\n", secretKey)
	}

	key := *saveName
	fileSlice := []string{}

	// 支持通配符
	for _, file := range files {
		matches, err := filepath.Glob(file)
		if err == nil {

			for _, path := range matches {
				// 遍历目录
				err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
					if err != nil {
						log.Print(err)
						return nil
					}

					// ignore ignorePaths
					for _, i := range ignorePaths {
						p := filepath.Base(path)
						if m, _ := filepath.Match(i, p); m {
							if info.IsDir() {
								return filepath.SkipDir
							}
							return nil
						}
					}
					if info.IsDir() {
						return nil
					}

					fileSlice = append(fileSlice, path)
					return nil
				})
				if err != nil {
					log.Print(err)
				}
			}
		}
	}
	if len(fileSlice) == 0 {
		flag.PrintDefaults()
		fmt.Println("need files: qn_cli FILE [FILE ...]")
		os.Exit(1)
	}

	// 配置 accessKey, secretKey
	kodo.SetMac(accessKey, secretKey)
	if len(ignores) != 0 {
		ignorePaths = append(ignorePaths, ignores...)
	}

	return &args{
		bucketName:  bucketName,
		bucketURL:   bucketURL,
		fileSlice:   fileSlice,
		key:         key,
		autoName:    *autoName,
		autoMD5Name: *autoMD5Name,
		overwrite:   *overwrite,
		saveDir:     *saveDir,
		verbose:     *verbose,
	}
}

func main() {
	a := parseArgs()
	if a.verbose {
		fmt.Println(a)
	}

	// 定义任务组
	var wg sync.WaitGroup

	// 上传文件
	for _, file := range a.fileSlice {
		// 增加一个任务
		wg.Add(1)
		// 使用 goroutine 异步执行上传任务
		go func(file string) {
			defer wg.Done() // 标记任务完成
			key := a.key
			zone := 0
			c := kodo.New(zone, nil)
			uploader := kodocli.NewUploader(zone, nil)
			ctx := context.Background()

			if a.autoMD5Name && key == "" {
				key = autoMD5FileName(file)
			} else if a.autoName && key == "" {
				key = file
			}
			if a.saveDir != "" {
				key = path.Join(a.saveDir, key)
			}
			token := genUpToken(a, c, key)

			// 上传文件
			ret, err := uploadFile(uploader, ctx, file, key, token)
			if err != nil {
				if a.verbose {
					fmt.Printf("%s: %s ✕\n", file, err)
				} else {
					fmt.Printf("%s ✕\n", file)
				}
				log.Fatal(err)
			} else {
				url := finalURL(a.bucketURL, ret.Key)
				if a.verbose {
					fmt.Printf("%s: %s ✓\n", file, url)
				} else {
					fmt.Printf("%s\n", url)
				}
			}
		}(file)
	}

	// 等待所有任务完成
	wg.Wait()
}
<commit_msg>把遍历目录的操作提取为一个函数<commit_after>/*
七牛本地上传客户端
$ qn_cli --help
*/
package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"golang.org/x/net/context"
	"qiniupkg.com/api.v7/kodo"
	"qiniupkg.com/api.v7/kodocli"
)

var ignorePaths = []string{
	".git", ".hg", ".svn", ".module-cache", ".bin",
}

type stringSlice []string

func (s *stringSlice) String() string {
	return fmt.Sprintf("%s", *s)
}
func (s *stringSlice) Set(value string) error {
	*s = append(*s, value)
	return nil
}

// 生成上传 token
func genUpToken(a *args, c *kodo.Client, key string) string {
	policy := kodo.PutPolicy{
		Scope: a.bucketName,
		// ReturnBody: `{"bucket": $(bucket),"key": $(key)}`,
		DetectMime: 1,
	}
	if key != "" {
		policy.SaveKey = key
		if a.overwrite {
			policy.Scope = policy.Scope + ":" + key
			policy.InsertOnly = 0
		}
	}
	return c.MakeUptoken(&policy)
}

// 上传本地文件
func uploadFile(
	uploader kodocli.Uploader, ctx context.Context, localFile, key, uptoken string) (ret *kodocli.PutRet, err error) {
	ret = &kodocli.PutRet{}
	if key == "" {
		err = uploader.PutFileWithoutKey(ctx, ret, uptoken, localFile, nil)
	} else {
		err = uploader.PutFile(ctx, ret, uptoken, key, localFile, nil)
	}
	return
}

// 自动生成文件名
func autoFileName(p string) (string, string, string) {
	dirname, name := path.Split(p)
	ext := path.Ext(name)
	return dirname, name, ext
}

func autoMD5FileName(p string) string {
	dirname, oldName, ext := autoFileName(p)
	now := int(time.Now().Nanosecond())
	hash := md5.Sum([]byte(
		strconv.Itoa(now),
	))
	newName := dirname + oldName + "_" + hex.EncodeToString(hash[:]) + ext
	return newName
}

func walkFiles(files []string, ignorePaths []string) (fileSlice []string) {
	for _, file := range files {
		matches, err := filepath.Glob(file)
		if err == nil {

			for _, path := range matches {
				// 遍历目录
				err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
					if err != nil {
						log.Print(err)
						return nil
					}

					// ignore ignorePaths
					for _, i := range ignorePaths {
						p := filepath.Base(path)
						if m, _ := filepath.Match(i, p); m {
							if info.IsDir() {
								return filepath.SkipDir
							}
							return nil
						}
					}
					if info.IsDir() {
						return nil
					}

					fileSlice = append(fileSlice, path)
					return nil
				})
				if err != nil {
					log.Print(err)
				}
			}
		}
	}

	return
}

func finalURL(bucketURL, key string) (url string) {
	return bucketURL + key
}

type args struct {
	bucketName  string
	bucketURL   string
	fileSlice   []string
	key         string
	autoName    bool
	autoMD5Name bool
	overwrite   bool
	saveDir     string
	verbose     bool
}

func parseArgs() *args {
	// 保存名称
	saveName := flag.String("n", "", "Save name")
	saveDir := flag.String("d", "", "Save dirname")
	autoName := flag.Bool("a", true, "Auto named saved files")
	autoMD5Name := flag.Bool("md5", false, "Auto named saved files use MD5 value")
	overwrite := flag.Bool("w", true, "Overwrite exists files")
	verbose := flag.Bool("v", false, "Verbose mode")
	var ignores stringSlice
	flag.Var(&ignores, "i", "ignores")

	flag.Parse()
	files := flag.Args()

	bucketName := os.Getenv("QINIU_BUCKET_NAME")
	bucketURL := os.Getenv("QINIU_BUCKET_URL")
	accessKey := os.Getenv("QINIU_ACCESS_KEY")
	secretKey := os.Getenv("QINIU_SECRET_KEY")
	if *verbose {
		fmt.Printf("bucketName: %s\n", bucketName)
		fmt.Printf("bucketURL: %s\n", bucketURL)
		fmt.Printf("accessKey: %s\n", accessKey)
		fmt.Printf("secretKey: %s\n", secretKey)
	}

	key := *saveName
	// 支持通配符
	fileSlice := walkFiles(files, ignorePaths)

	if len(fileSlice) == 0 {
		flag.PrintDefaults()
		fmt.Println("need files: qn_cli FILE [FILE ...]")
		os.Exit(1)
	}

	// 配置 accessKey, secretKey
	kodo.SetMac(accessKey, secretKey)
	if len(ignores) != 0 {
		ignorePaths = append(ignorePaths, ignores...)
	}

	return &args{
		bucketName:  bucketName,
		bucketURL:   bucketURL,
		fileSlice:   fileSlice,
		key:         key,
		autoName:    *autoName,
		autoMD5Name: *autoMD5Name,
		overwrite:   *overwrite,
		saveDir:     *saveDir,
		verbose:     *verbose,
	}
}

func main() {
	a := parseArgs()
	if a.verbose {
		fmt.Println(a)
	}

	// 定义任务组
	var wg sync.WaitGroup

	// 上传文件
	for _, file := range a.fileSlice {
		// 增加一个任务
		wg.Add(1)
		// 使用 goroutine 异步执行上传任务
		go func(file string) {
			defer wg.Done() // 标记任务完成
			key := a.key
			zone := 0
			c := kodo.New(zone, nil)
			uploader := kodocli.NewUploader(zone, nil)
			ctx := context.Background()

			if a.autoMD5Name && key == "" {
				key = autoMD5FileName(file)
			} else if a.autoName && key == "" {
				key = file
			}
			if a.saveDir != "" {
				key = path.Join(a.saveDir, key)
			}
			token := genUpToken(a, c, key)

			// 上传文件
			ret, err := uploadFile(uploader, ctx, file, key, token)
			if err != nil {
				if a.verbose {
					fmt.Printf("%s: %s ✕\n", file, err)
				} else {
					fmt.Printf("%s ✕\n", file)
				}
				log.Fatal(err)
			} else {
				url := finalURL(a.bucketURL, ret.Key)
				if a.verbose {
					fmt.Printf("%s: %s ✓\n", file, url)
				} else {
					fmt.Printf("%s\n", url)
				}
			}
		}(file)
	}

	// 等待所有任务完成
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config         Config
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels             []string
	DBConn               string
	Nick                 string
	Ident                string
	FullName             string
	FlickrAPIKey         string
	WolframAPIKey        string
	OpenWeatherMapAPIKey string
	IRCPass              string
	RebuildWords         bool
	Commands             []struct {
		Channel  string
		Commands []struct {
			Name string
			Text string
		}
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

func getCommand(line *irc.Line) string {
	splitmessage := strings.Split(line.Text(), " ")
	cmd := strings.TrimSpace(splitmessage[0])
	return cmd
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error converting page to utf8:", err)
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("Error reading posted link:", err)
		return
	}
	respbody = append(respbody, restofbody...)
	query, err := goquery.NewDocumentFromReader(bytes.NewReader(respbody))
	if err != nil {
		log.Println("Error parsing HTML tree:", err)
		return
	}
	title := query.Find("title").Text()
	title = strings.TrimSpace(title)
	if len(title) == 0 || !utf8.ValidString(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	hostNick := fmt.Sprintf(" (%s / %s)", postedUrl.Host, nick)
	formattedTitle := html.UnescapeString(title)
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	if len(formattedTitle) > conn.Config().SplitLen-len(hostNick)-1 {
		formattedTitle = formattedTitle[:conn.Config().SplitLen-len(hostNick)-1]
	}
	formattedTitle = formattedTitle + hostNick
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(conn *irc.Conn, line *irc.Line) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, line.Target(), line.Text(), line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, line.Text())
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(conn *irc.Conn, line *irc.Line) {
	if strings.HasPrefix(line.Text(), "#") {
		return
	}
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(line.Text(), -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(line.Target(), item, conn, line.Nick)
	}
}

func cst(conn *irc.Conn, line *irc.Line) {
	if line.Nick != "sadbox" || getCommand(line) != "!cst" {
		return
	}
	go conn.Privmsg(line.Target(), "\u00039,13#CSTMASTERRACE")
}

// Commands that are read in from the config file
func configCommands(conn *irc.Conn, line *irc.Line) {
	splitmessage := strings.Split(line.Text(), " ")
AllConfigs:
	for _, commandConfig := range config.Commands {
		if commandConfig.Channel == line.Target() || commandConfig.Channel == "default" {
			for _, command := range commandConfig.Commands {
				if getCommand(line) == command.Name {
					var response string
					if len(splitmessage) >= 2 {
						response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
					} else {
						response = command.Text
					}
					go conn.Privmsg(line.Target(), response)
					break AllConfigs
				}
			}
		}
	}
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	numcommands := 0
	for _, commandConfig := range config.Commands {
		for _, command := range commandConfig.Commands {
			numcommands++
			log.Printf("%d %s/%s: %s", numcommands, commandConfig.Channel, command.Name, command.Text)
		}
	}
	log.Printf("Found %d commands", numcommands)
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	// Handle all the things
	c.HandleFunc(irc.PRIVMSG, logMessage)
	c.HandleFunc(irc.ACTION, logMessage)

	c.HandleFunc(irc.PRIVMSG, checkForUrl)
	c.HandleFunc(irc.ACTION, checkForUrl)

	c.HandleFunc(irc.PRIVMSG, haata)
	c.HandleFunc(irc.PRIVMSG, wolfram)
	c.HandleFunc(irc.PRIVMSG, meeba)
	c.HandleFunc(irc.PRIVMSG, markov)
	c.HandleFunc(irc.PRIVMSG, dance)
	c.HandleFunc(irc.PRIVMSG, cst)
	c.HandleFunc(irc.PRIVMSG, roll)
	c.HandleFunc(irc.PRIVMSG, btc)
	c.HandleFunc(irc.PRIVMSG, lastSeen)
	c.HandleFunc(irc.PRIVMSG, showWeather)
	c.HandleFunc(irc.PRIVMSG, showQuote)
	c.HandleFunc(irc.PRIVMSG, configCommands)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<commit_msg>minor change to link-posting format<commit_after>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/PuerkitoBio/goquery"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config         Config
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels             []string
	DBConn               string
	Nick                 string
	Ident                string
	FullName             string
	FlickrAPIKey         string
	WolframAPIKey        string
	OpenWeatherMapAPIKey string
	IRCPass              string
	RebuildWords         bool
	Commands             []struct {
		Channel  string
		Commands []struct {
			Name string
			Text string
		}
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

func getCommand(line *irc.Line) string {
	splitmessage := strings.Split(line.Text(), " ")
	cmd := strings.TrimSpace(splitmessage[0])
	return cmd
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("Error converting page to utf8:", err)
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("Error reading posted link:", err)
		return
	}
	respbody = append(respbody, restofbody...)
	query, err := goquery.NewDocumentFromReader(bytes.NewReader(respbody))
	if err != nil {
		log.Println("Error parsing HTML tree:", err)
		return
	}
	title := query.Find("title").Text()
	title = strings.TrimSpace(title)
	if len(title) == 0 || !utf8.ValidString(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	hostNick := fmt.Sprintf(" (%s)", postedUrl.Host)
	formattedTitle := html.UnescapeString(title)
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	if len(formattedTitle) > conn.Config().SplitLen-len(hostNick)-1 {
		formattedTitle = formattedTitle[:conn.Config().SplitLen-len(hostNick)-1]
	}
	formattedTitle = formattedTitle + hostNick
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(conn *irc.Conn, line *irc.Line) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, line.Target(), line.Text(), line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, line.Text())
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(conn *irc.Conn, line *irc.Line) {
	if strings.HasPrefix(line.Text(), "#") {
		return
	}
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(line.Text(), -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(line.Target(), item, conn, line.Nick)
	}
}

func cst(conn *irc.Conn, line *irc.Line) {
	if line.Nick != "sadbox" || getCommand(line) != "!cst" {
		return
	}
	go conn.Privmsg(line.Target(), "\u00039,13#CSTMASTERRACE")
}

// Commands that are read in from the config file
func configCommands(conn *irc.Conn, line *irc.Line) {
	splitmessage := strings.Split(line.Text(), " ")
AllConfigs:
	for _, commandConfig := range config.Commands {
		if commandConfig.Channel == line.Target() || commandConfig.Channel == "default" {
			for _, command := range commandConfig.Commands {
				if getCommand(line) == command.Name {
					var response string
					if len(splitmessage) >= 2 {
						response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
					} else {
						response = command.Text
					}
					go conn.Privmsg(line.Target(), response)
					break AllConfigs
				}
			}
		}
	}
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	numcommands := 0
	for _, commandConfig := range config.Commands {
		for _, command := range commandConfig.Commands {
			numcommands++
			log.Printf("%d %s/%s: %s", numcommands, commandConfig.Channel, command.Name, command.Text)
		}
	}
	log.Printf("Found %d commands", numcommands)
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	// Handle all the things
	c.HandleFunc(irc.PRIVMSG, logMessage)
	c.HandleFunc(irc.ACTION, logMessage)

	c.HandleFunc(irc.PRIVMSG, checkForUrl)
	c.HandleFunc(irc.ACTION, checkForUrl)

	c.HandleFunc(irc.PRIVMSG, haata)
	c.HandleFunc(irc.PRIVMSG, wolfram)
	c.HandleFunc(irc.PRIVMSG, meeba)
	c.HandleFunc(irc.PRIVMSG, markov)
	c.HandleFunc(irc.PRIVMSG, dance)
	c.HandleFunc(irc.PRIVMSG, cst)
	c.HandleFunc(irc.PRIVMSG, roll)
	c.HandleFunc(irc.PRIVMSG, btc)
	c.HandleFunc(irc.PRIVMSG, lastSeen)
	c.HandleFunc(irc.PRIVMSG, showWeather)
	c.HandleFunc(irc.PRIVMSG, showQuote)
	c.HandleFunc(irc.PRIVMSG, configCommands)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/gorilla/mux"
	"github.com/spf13/viper"
)

type Program struct {
	URL      string
	Duration time.Duration
}

// InitializeConfig loads our configuration using Viper package.
func InitializeConfig() {

	viper.SetConfigType("yaml")
	viper.SetConfigName("config")

	viper.AddConfigPath("$HOME/.gotator")
	viper.AddConfigPath(".")

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	viper.SetDefault("debug", false)

	viper.SetEnvPrefix("gorotator") // will be uppercased automatically
	viper.BindEnv("debug")
	viper.BindEnv("firefox_ip")
	viper.BindEnv("firefox_port")
	viper.BindEnv("gotator_port")

	if !viper.IsSet("firefox_ip") || !viper.IsSet("firefox_port") {
		fmt.Fprintln(os.Stderr, "Configuration error.  Both FIREFOX_IP and FIREFOX_PORT must be set via either config or environment.")
		os.Exit(1)
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("\nConfig file changed:", e.Name)
		skip <- struct{}{}
		fmt.Printf("Content will change immediately.\n\n")
	})

}

// Loads a list of programs.
// A program consists of a list things to display on the rotator along
// with a number of seconds to display each one before moving on.
func loadProgramList(filename string) []Program {

	fmt.Printf("Loading programs from %s\n", filename)

	var list []Program

	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	webpages := string(bytes)

	r := csv.NewReader(strings.NewReader(webpages))
	r.LazyQuotes = true

	for {
		var p Program
		record, err := r.Read()
		if err == io.EOF {
			fmt.Printf("Finished loading programs.\n\n")
			break
		}
		if err != nil {
			log.Printf("Error reading line from program file: %s.  Abandoning attempt to read programs.\n", filename)
			log.Fatal(err)
		}
		p.URL = record[0]
		p.Duration, err = time.ParseDuration(record[1])
		if err != nil {
			fmt.Println("Program rejected.  Invalid duration.")
		}

		fmt.Printf("  Loaded program %.50s to show for %s.\n", p.URL, p.Duration)
		list = append(list, p)
	}

	return list
}

func runProgram(program Program) {

	ip := viper.Get("FIREFOX_IP")
	port := viper.GetInt("FIREFOX_PORT")

	constr := fmt.Sprintf("%s:%d", ip, port)

	conn, err := net.Dial("tcp", constr)
	if err != nil {
		fmt.Printf("Error making network connection to: %s\n", constr)
		fmt.Println("It is possible Firefox needs to be started or restarted.")
		fmt.Println("Pausing for 30s")
		time.Sleep(30 * time.Second) // wait 30 seconds to slow retries
		return
	}

	fmt.Printf("Running program for %s:  %s -> RESULT: ", program.Duration, program.URL)
	fmt.Fprintf(conn, "window.location='%s'\n", program.URL)
	status, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		fmt.Println("ERROR - URL didn't load as desired.")
	}

	var statusParsed interface{}
	err = json.Unmarshal([]byte(status), &statusParsed)

	m := statusParsed.(map[string]interface{})

	if m["result"] == program.URL {
		fmt.Println("OK")
	} else {
		fmt.Println("ERROR - URL didn't load as desired.")
	}

	select {
	case <-time.After(program.Duration):
		// Do nothing.
		Unpause()
	case <-skip:
		fmt.Println("Current program skipped")
		return
	}
}

func Pause() {
	mu.Lock()
	pause = true
	mu.Unlock()
}

func Unpause() {
	mu.Lock()
	pause = false
	mu.Unlock()
}

func IsPaused() bool {
	mu.Lock()
	defer mu.Unlock()

	return pause == true
}

func LoadAndRunLoop() {

	// Load and run the acctive program_file indefinately
	for {
		// We pull filename inside the loop because the
		// configuration can change while our program is running.
		filename := viper.GetString("program_file")
		for IsPaused() {
			fmt.Println("Paused.")
			time.Sleep(1 * time.Second)
		}

		pl := loadProgramList(filename)

		for _, p := range pl {
			for IsPaused() {
				fmt.Println("Program list is paused.")
				time.Sleep(1 * time.Second)
			}
			runProgram(p)
		}

		fmt.Printf("\nLooping back to play program list from beginning\n\n")
	}

}

func PlayHandler(w http.ResponseWriter, r *http.Request) {

	r.ParseForm()
	var p Program
	p.URL = r.Form.Get("url")
	fmt.Printf("URL: %s\n", p.URL)

	d := r.Form.Get("duration")
	fmt.Printf("Duration: %s\n", d)

	var err error
	p.Duration, err = time.ParseDuration(r.Form.Get("duration"))
	if err != nil {
		w.Write([]byte("Program rejected.  Invalid duration.\n"))
		return
	}

	// Needs validation...

	// Now do something with the program.. play it?

	// Stop normal rotation
	Pause()
	skip <- struct{}{}

	go runProgram(p)
	w.Write([]byte("Program accepted\n"))

}

func PauseHandler(w http.ResponseWriter, r *http.Request) {
	Pause()
	log.Println("Pausing from web request")
	w.Write([]byte("Ok, paused.\n"))
}

func ResumeHandler(w http.ResponseWriter, r *http.Request) {
	Unpause()
	log.Println("Unpausing from web request")
	w.Write([]byte("Ok, unpaused.\n"))
}
func SkipHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Skippingfrom web request")
	Unpause()
	skip <- struct{}{}

	w.Write([]byte("Skipping current programming and resume program list runner from web request.\n"))
}

func readKeyboardLoop() {
	for {
		os.Stdin.Read(make([]byte, 1)) // read a single byte
		fmt.Printf(" >> Got keyboard input, that means you want to move to the next program.  Can do! << \n\n")
		Unpause()
		skip <- struct{}{}
	}
}

// Control channel to stop running programs immediately (yes, global)

var skip = make(chan struct{})
var exitprogram = make(chan struct{})
var pause bool
var mu = &sync.Mutex{}
var version = "0.0.5"

func main() {

	if len(os.Args) > 1 {
		if os.Args[1] == "version" {
			fmt.Println("Gotator version:", version)
			os.Exit(0)
		}
	}
	fmt.Println("Starting gotator: version", version)
	Unpause()

	InitializeConfig()

	go LoadAndRunLoop()
	go readKeyboardLoop()

	if viper.IsSet("apienabled") && viper.Get("apienabled") == true {
		listen_port := ":8080"
		if viper.IsSet("gotator_port") {
			listen_port = ":" + viper.GetString("gotator_port")
		}

		fmt.Printf("Starting API server on port %s.  Notice:  This allows UNAUTHENTICATED remote control of Firefox. set 'apienabled: false' in config.yaml to disable.\n",
			listen_port)

		r := mux.NewRouter()
		r.HandleFunc("/play", PlayHandler)
		r.HandleFunc("/pause", PauseHandler)
		r.HandleFunc("/resume", ResumeHandler)
		r.HandleFunc("/skip", SkipHandler)

		go log.Fatal(http.ListenAndServe(listen_port, r))
	} else {
		fmt.Println("notice: rest API not enabled in configuration and will be unavailable.  set 'apienabled: true' in config.yaml if you want to use it.\n")
		// If we aren't doing http.ListenAndServe() we need to block here or else gotator would exit immediately
		<-exitprogram
	}

}
<commit_msg>Change to use log over fmt.Print<commit_after>package main

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/gorilla/mux"
	"github.com/spf13/viper"
)

type Program struct {
	URL      string
	Duration time.Duration
}

// InitializeConfig loads our configuration using Viper package.
func InitializeConfig() {

	viper.SetConfigType("yaml")
	viper.SetConfigName("config")

	viper.AddConfigPath("$HOME/.gotator")
	viper.AddConfigPath(".")

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	viper.SetDefault("debug", false)

	viper.SetEnvPrefix("gorotator") // will be uppercased automatically
	viper.BindEnv("debug")
	viper.BindEnv("firefox_ip")
	viper.BindEnv("firefox_port")
	viper.BindEnv("gotator_port")

	if !viper.IsSet("firefox_ip") || !viper.IsSet("firefox_port") {
		fmt.Fprintln(os.Stderr, "Configuration error.  Both FIREFOX_IP and FIREFOX_PORT must be set via either config or environment.")
		os.Exit(1)
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("\nConfig file changed:", e.Name)
		skip <- struct{}{}
		fmt.Printf("Content will change immediately.\n\n")
	})

}

// Loads a list of programs.
// A program consists of a list things to display on the rotator along
// with a number of seconds to display each one before moving on.
func loadProgramList(filename string) []Program {

	log.Printf("Loading programs from %s\n", filename)

	var list []Program

	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	webpages := string(bytes)

	r := csv.NewReader(strings.NewReader(webpages))
	r.LazyQuotes = true

	for {
		var p Program
		record, err := r.Read()
		if err == io.EOF {
			log.Println("Finished loading programs.")
			break
		}
		if err != nil {
			log.Printf("Error reading line from program file: %s.  Abandoning attempt to read programs.\n", filename)
			log.Fatal(err)
		}
		p.URL = record[0]
		p.Duration, err = time.ParseDuration(record[1])
		if err != nil {
			log.Println("Program rejected.  Invalid duration.")
		}

		log.Printf("  Loaded program %.50s to show for %s.\n", p.URL, p.Duration)
		list = append(list, p)
	}

	return list
}

func runProgram(program Program) {

	ip := viper.Get("FIREFOX_IP")
	port := viper.GetInt("FIREFOX_PORT")

	constr := fmt.Sprintf("%s:%d", ip, port)

	conn, err := net.Dial("tcp", constr)
	if err != nil {
		fmt.Printf("Error making network connection to: %s\n", constr)
		fmt.Println("It is possible Firefox needs to be started or restarted.")
		fmt.Println("Pausing for 30s")
		time.Sleep(30 * time.Second) // wait 30 seconds to slow retries
		return
	}

	fmt.Printf("Running program for %s:  %s -> RESULT: ", program.Duration, program.URL)
	fmt.Fprintf(conn, "window.location='%s'\n", program.URL)
	status, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		fmt.Println("ERROR - URL didn't load as desired.")
	}

	var statusParsed interface{}
	err = json.Unmarshal([]byte(status), &statusParsed)

	m := statusParsed.(map[string]interface{})

	if m["result"] == program.URL {
		fmt.Println("OK")
	} else {
		fmt.Println("ERROR - URL didn't load as desired.")
	}

	select {
	case <-time.After(program.Duration):
		// Do nothing.
		Unpause()
	case <-skip:
		log.Println("Current program skipped")

		return
	}
}

func Pause() {
	mu.Lock()
	pause = true
	mu.Unlock()
}

func Unpause() {
	mu.Lock()
	pause = false
	mu.Unlock()
}

func IsPaused() bool {
	mu.Lock()
	defer mu.Unlock()

	return pause == true
}

func LoadAndRunLoop() {

	// Load and run the acctive program_file indefinately
	for {
		// We pull filename inside the loop because the
		// configuration can change while our program is running.
		filename := viper.GetString("program_file")
		for IsPaused() {
			fmt.Println("Paused.")
			time.Sleep(1 * time.Second)
		}

		pl := loadProgramList(filename)

		for _, p := range pl {
			for IsPaused() {
				// log.Println("Program list is paused.")
				time.Sleep(1 * time.Second)
			}
			runProgram(p)
		}

		log.Println("Looping back to play program list from beginning")
	}

}

func PlayHandler(w http.ResponseWriter, r *http.Request) {

	r.ParseForm()
	var p Program
	p.URL = r.Form.Get("url")
	log.Printf("URL: %s\n", p.URL)

	d := r.Form.Get("duration")
	log.Printf("Duration: %s\n", d)

	var err error
	p.Duration, err = time.ParseDuration(r.Form.Get("duration"))
	if err != nil {
		w.Write([]byte("Program rejected.  Invalid duration.\n"))
		return
	}

	// Needs validation...

	// Now do something with the program.. play it?

	// Stop normal rotation
	Pause()
	skip <- struct{}{}

	go runProgram(p)
	w.Write([]byte("Program accepted\n"))

}

func PauseHandler(w http.ResponseWriter, r *http.Request) {
	Pause()
	log.Println("Pausing from web request")
	w.Write([]byte("Ok, paused.\n"))
}

func ResumeHandler(w http.ResponseWriter, r *http.Request) {
	Unpause()
	log.Println("Unpausing from web request")
	w.Write([]byte("Ok, unpaused.\n"))
}
func SkipHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Skippingfrom web request")
	Unpause()
	skip <- struct{}{}

	w.Write([]byte("Skipping current programming and resume program list runner from web request.\n"))
}

func readKeyboardLoop() {
	for {
		os.Stdin.Read(make([]byte, 1)) // read a single byte
		log.Printf(" >> Got keyboard input, that means you want to move to the next program.  Can do! << \n\n")
		Unpause()
		skip <- struct{}{}
	}
}

// Control channel to stop running programs immediately (yes, global)

var skip = make(chan struct{})
var exitprogram = make(chan struct{})
var pause bool
var mu = &sync.Mutex{}
var version = "0.0.5"

func main() {

	if len(os.Args) > 1 {
		if os.Args[1] == "version" {
			fmt.Println("Gotator version:", version)
			os.Exit(0)
		}
	}
	fmt.Println("Starting gotator: version", version)
	Unpause()

	InitializeConfig()

	go LoadAndRunLoop()
	go readKeyboardLoop()

	if viper.IsSet("apienabled") && viper.Get("apienabled") == true {
		listen_port := ":8080"
		if viper.IsSet("gotator_port") {
			listen_port = ":" + viper.GetString("gotator_port")
		}

		log.Printf("Starting API server on port %s.  Notice:  This allows UNAUTHENTICATED remote control of Firefox. set 'apienabled: false' in config.yaml to disable.\n",
			listen_port)

		r := mux.NewRouter()
		r.HandleFunc("/play", PlayHandler)
		r.HandleFunc("/pause", PauseHandler)
		r.HandleFunc("/resume", ResumeHandler)
		r.HandleFunc("/skip", SkipHandler)

		go log.Fatal(http.ListenAndServe(listen_port, r))
	} else {
		log.Println("notice: rest API not enabled in configuration and will be unavailable.  set 'apienabled: true' in config.yaml if you want to use it.\n")
		// If we aren't doing http.ListenAndServe() we need to block here or else gotator would exit immediately
		<-exitprogram
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	if len(os.Args) < 2 {
		os.Stdout.WriteString("GPIO No should be specified.\n")
		os.Exit(1)
	}

	gpio := os.Args[1]

	f, err := os.Create("/sys/class/gpio/export")

	if err != nil {
		panic(err)
	}

	f.WriteString(gpio)

	f, err = os.Create(fmt.Sprintf("/sys/class/gpio/gpio%s/direction", gpio))

	if err != nil {
		panic(err)
	}

	f.WriteString("out")

	f, err = os.Create(fmt.Sprintf("/sys/class/gpio/gpio%s/value", gpio))

	if err != nil {
		panic(err)
	}

	f.WriteString("1")

	time.Sleep(1 * time.Second)

	f.WriteString("0")
}
<commit_msg>Update main.go<commit_after>package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	if len(os.Args) < 2 {
		os.Stderr.WriteString("GPIO No should be specified.\n")
		os.Exit(1)
	}

	gpio := os.Args[1]

	f, err := os.Create("/sys/class/gpio/export")
	if err != nil {
		panic(err)
	}

	f.WriteString(gpio)

	f, err = os.Create(fmt.Sprintf("/sys/class/gpio/gpio%s/direction", gpio))
	if err != nil {
		panic(err)
	}

	f.WriteString("out")

	f, err = os.Create(fmt.Sprintf("/sys/class/gpio/gpio%s/value", gpio))
	if err != nil {
		panic(err)
	}

	f.WriteString("1")

	time.Sleep(3 * time.Second)

	f.WriteString("0")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/cli"
)

var (
	ui     *cli.ColoredUi
	regLen = 32
	binStr string
	value  int64
)

// field range
type fRange struct {
	start int
	end   int
}

func initUi() error {
	ui = new(cli.ColoredUi)
	if ui == nil {
		fmt.Printf("error of ui\n")
		return errors.New("failed to new cli")
	}

	bui := new(cli.BasicUi)
	bui.Reader = os.Stdin
	bui.Writer = os.Stdout
	bui.ErrorWriter = os.Stderr

	ui.Ui = bui
	ui.OutputColor = cli.UiColorNone
	ui.InfoColor = cli.UiColorGreen
	ui.ErrorColor = cli.UiColorRed
	ui.WarnColor = cli.UiColorYellow

	return nil
}

func checkBinStr() bool {
	if binStr == "" {
		ui.Info("empty value. Use 'value' to update.")
		return false
	}
	return true
}

func updateBit(input string, set bool) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(input)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	binByte := []byte(binStr)
	c := byte('0')
	if set {
		c = '1'
	}
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = c
	}

	// update global variable
	binStr = string(binByte)
}

func showReg(input string) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(input)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	start_index := regLen - 1 - r.end
	end_index := regLen - 1 - r.start
	subbin := binStr[start_index : end_index+1]

	outputTriFormat(os.Stdout, subbin)
}

func updateValue(s string) {
	val, err := parseInt(s)
	if err != nil {
		ui.Error(fmt.Sprintf("convert to Int failed: %v", err))
		return
	}

	s = strconv.FormatInt(val, 2)
	l := len(s)
	bin := strings.Repeat("0", regLen-l)
	binStr = bin + s
}

func writeFiled(rStr, vStr string) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(rStr)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	val, err := parseInt(vStr)
	if err != nil {
		ui.Error(fmt.Sprintf("convert to Int failed: %v", err))
		return
	}

	max := (2 << uint(r.end-r.start)) - 1
	if val < 0 || int(val) > max {
		ui.Error(fmt.Sprint("val is out of range [%d, %d]", 0, max))
		return
	}

	s := strconv.FormatInt(val, 2)
	l := len(s)
	sub := strings.Repeat("0", r.end-r.start+1-l)
	sub = sub + s
	subByte := []byte(sub)
	fmt.Println(sub)

	binByte := []byte(binStr)
	j := r.end - r.start
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = subByte[j]
		j--
	}

	// update global variable
	binStr = string(binByte)
}

func handleInput(input string) (exit bool) {
	exit = false
	cmdline := strings.Fields(input)

	if len(cmdline) == 0 {
		printUsage()
		return
	}

	switch cmdline[0] {
	case "exit":
		exit = true
		return
	case "help", "h":
		printUsage()
	case "print", "p":
		outputTriFormat(os.Stdout, binStr)
	case "value", "v":
		if len(cmdline) < 2 {
			ui.Error("Needs argument: <range>")
			return
		}
		updateValue(cmdline[1])
		outputTriFormat(os.Stdout, binStr)
	case "set", "s", "clear", "c":
		if len(cmdline) < 2 {
			ui.Error("Needs argument: <range>")
			return
		}
		set := true
		if cmdline[0] == "clear" || cmdline[0] == "c" {
			set = false
		}
		updateBit(cmdline[1], set)
		outputTriFormat(os.Stdout, binStr)
	case "write", "w":
		if len(cmdline) < 3 {
			ui.Error("Needs arguments: <range> <val>")
		}
		writeFiled(cmdline[1], cmdline[2])
		outputTriFormat(os.Stdout, binStr)
	default:
		showReg(cmdline[0])
	}

	return
}

func printUsage() {
	ui.Output("Usage:")
	ui.Output("  [h]elp          : print this message.")
	ui.Output("  [p]rint         : print input value.")
	ui.Output("  [v]alue <val>   : input value.")
	ui.Output("  [s]et <bit>     : set <bit> to 1.")
	ui.Output("  [c]lear <bit>   : clear <bit> to 0.")
	ui.Output("  [w]rite <r> <v> : write val <v> into field range <r>.")
	ui.Output("  <range>         : read the value of field range <range>, like 1 or 2:3.")
	ui.Output("  exit            : exit this program.")
}

func main() {
	flag.IntVar(&regLen, "l", 32, "register length.")
	flag.Parse()

	if len(os.Args) == 2 && (os.Args[1] == "help" || os.Args[1] == "-h") {
		flag.Usage()
		os.Exit(0)
	}

	if err := initUi(); err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	for {
		input, err := ui.Ask("\n>>>")
		if err != nil {
			fmt.Println(err)
		}

		if handleInput(input) {
			break
		}
	}
}
<commit_msg>rework 'exit'<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/cli"
)

var (
	ui     *cli.ColoredUi
	regLen = 32
	binStr string
	value  int64
)

// field range
type fRange struct {
	start int
	end   int
}

func initUi() error {
	ui = new(cli.ColoredUi)
	if ui == nil {
		fmt.Printf("error of ui\n")
		return errors.New("failed to new cli")
	}

	bui := new(cli.BasicUi)
	bui.Reader = os.Stdin
	bui.Writer = os.Stdout
	bui.ErrorWriter = os.Stderr

	ui.Ui = bui
	ui.OutputColor = cli.UiColorNone
	ui.InfoColor = cli.UiColorGreen
	ui.ErrorColor = cli.UiColorRed
	ui.WarnColor = cli.UiColorYellow

	return nil
}

func checkBinStr() bool {
	if binStr == "" {
		ui.Info("empty value. Use 'value' to update.")
		return false
	}
	return true
}

func updateBit(input string, set bool) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(input)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	binByte := []byte(binStr)
	c := byte('0')
	if set {
		c = '1'
	}
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = c
	}

	// update global variable
	binStr = string(binByte)
}

func showReg(input string) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(input)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	start_index := regLen - 1 - r.end
	end_index := regLen - 1 - r.start
	subbin := binStr[start_index : end_index+1]

	outputTriFormat(os.Stdout, subbin)
}

func updateValue(s string) {
	val, err := parseInt(s)
	if err != nil {
		ui.Error(fmt.Sprintf("convert to Int failed: %v", err))
		return
	}

	s = strconv.FormatInt(val, 2)
	l := len(s)
	bin := strings.Repeat("0", regLen-l)
	binStr = bin + s
}

func writeFiled(rStr, vStr string) {
	if !checkBinStr() {
		return
	}

	r, err := getRange(rStr)
	if err != nil {
		ui.Error(fmt.Sprintf("parse range start index failed, %v", err))
		return
	}

	val, err := parseInt(vStr)
	if err != nil {
		ui.Error(fmt.Sprintf("convert to Int failed: %v", err))
		return
	}

	max := (2 << uint(r.end-r.start)) - 1
	if val < 0 || int(val) > max {
		ui.Error(fmt.Sprint("val is out of range [%d, %d]", 0, max))
		return
	}

	s := strconv.FormatInt(val, 2)
	l := len(s)
	sub := strings.Repeat("0", r.end-r.start+1-l)
	sub = sub + s
	subByte := []byte(sub)
	fmt.Println(sub)

	binByte := []byte(binStr)
	j := r.end - r.start
	for i := r.start; i <= r.end; i++ {
		binByte[regLen-1-i] = subByte[j]
		j--
	}

	// update global variable
	binStr = string(binByte)
}

func handleInput(input string) (exit bool) {
	exit = false
	cmdline := strings.Fields(input)

	if len(cmdline) == 0 {
		printUsage()
		return
	}

	switch cmdline[0] {
	case "exit":
		exit = true
	case "help", "h":
		printUsage()
	case "print", "p":
		outputTriFormat(os.Stdout, binStr)
	case "value", "v":
		if len(cmdline) < 2 {
			ui.Error("Needs argument: <range>")
			return
		}
		updateValue(cmdline[1])
		outputTriFormat(os.Stdout, binStr)
	case "set", "s", "clear", "c":
		if len(cmdline) < 2 {
			ui.Error("Needs argument: <range>")
			return
		}
		set := true
		if cmdline[0] == "clear" || cmdline[0] == "c" {
			set = false
		}
		updateBit(cmdline[1], set)
		outputTriFormat(os.Stdout, binStr)
	case "write", "w":
		if len(cmdline) < 3 {
			ui.Error("Needs arguments: <range> <val>")
		}
		writeFiled(cmdline[1], cmdline[2])
		outputTriFormat(os.Stdout, binStr)
	default:
		showReg(cmdline[0])
	}

	return
}

func printUsage() {
	ui.Output("Usage:")
	ui.Output("  [h]elp          : print this message.")
	ui.Output("  [p]rint         : print input value.")
	ui.Output("  [v]alue <val>   : input value.")
	ui.Output("  [s]et <bit>     : set <bit> to 1.")
	ui.Output("  [c]lear <bit>   : clear <bit> to 0.")
	ui.Output("  [w]rite <r> <v> : write val <v> into field range <r>.")
	ui.Output("  <range>         : read the value of field range <range>, like 1 or 2:3.")
	ui.Output("  exit            : exit this program.")
}

func main() {
	flag.IntVar(&regLen, "l", 32, "register length.")
	flag.Parse()

	if len(os.Args) == 2 && (os.Args[1] == "help" || os.Args[1] == "-h") {
		flag.Usage()
		os.Exit(0)
	}

	if err := initUi(); err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	for {
		input, err := ui.Ask("\n>>>")
		if err != nil {
			fmt.Println(err)
		}

		if handleInput(input) {
			break
		}
	}
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2016 Aya Tokikaze
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"
)

const (
	cliName        = "oui"
	cliDescription = "search vender information for OUI(Organizationally Unique Identifier)"
	version        = "v0.2.0-dev"
)

func main() {
	sc := bufio.NewScanner(os.Stdin)

	app := cli.NewApp()
	app.Name = cliName
	app.Usage = cliDescription
	app.Version = version
	app.Flags = []cli.Flag{
		cli.HelpFlag,
		cli.BoolFlag{
			Name:  "input, i",
			Usage: "use standard input",
		},
	}
	app.Action = func(c *cli.Context) error {
		if c.NArg() == 0 && !c.Bool("input") {
			cli.ShowAppHelp(c)
		} else {
			data := InitMalData()

			var mac string
			if c.Bool("input") {
				if sc.Scan() {
					mac = sc.Text()
				}
			} else {
				mac = c.Args()[0]
			}

			mac = strings.Replace(mac, ":", "", -1)
			mac = strings.Replace(mac, "-", "", -1)
			for i := 0; i < len(data); i++ {
				if data[i].Hex == strings.ToUpper(mac[0:6]) {
					fmt.Println(data[i].OrgName)
					break
				}
			}
		}
		return nil
	}
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} <Address> [options]

VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
`
	app.Run(os.Args)
}
<commit_msg>Support verbose option<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2016 Aya Tokikaze
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/codegangsta/cli"
)

const (
	cliName        = "oui"
	cliDescription = "search vender information for OUI(Organizationally Unique Identifier)"
	version        = "v0.2.0-dev"
)

var (
	verbose bool
	input   bool
)

func main() {
	sc := bufio.NewScanner(os.Stdin)

	app := cli.NewApp()
	app.Name = cliName
	app.Usage = cliDescription
	app.Version = version
	app.Flags = []cli.Flag{
		cli.HelpFlag,
		cli.BoolFlag{
			Name:        "verbose, v",
			Usage:       "print detailed information",
			Destination: &verbose,
		},
		cli.BoolFlag{
			Name:        "input, i",
			Usage:       "use standard input",
			Destination: &input,
		},
	}
	app.Action = func(c *cli.Context) error {

		if c.NArg() == 0 && !input {
			cli.ShowAppHelp(c)
		} else {
			data := InitMalData()

			var mac string
			if input {
				if sc.Scan() {
					mac = sc.Text()
				}
			} else {
				mac = c.Args()[0]
			}

			mac = strings.Replace(mac, ":", "", -1)
			mac = strings.Replace(mac, "-", "", -1)
			for i := range data {
				if data[i].Hex == strings.ToUpper(mac[0:6]) {
					if verbose {
						split := []string{mac[0:2], mac[2:4], mac[4:6]}
						fmt.Printf("OUI/%s :      %s\nOrganization :  %s\nAddress :       %s\n", data[i].Registry, strings.Join(split, "-"), data[i].OrgName, data[i].OrgAddress)
					} else {
						fmt.Println(data[i].OrgName)
					}
					break
				}
			}
		}
		return nil
	}
	cli.VersionFlag = cli.BoolFlag{
		Name:  "version",
		Usage: "print oui version",
	}
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} <Address> [options]

VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}
`
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
)

func main() {
	// Setup flags
	statsPtr := flag.Bool("stats", false, "show stats and usage of `r`")
	commandsPtr := flag.Bool("commands", false, "show all commands that `r` will track")
	addPtr := flag.String("add", "", "show stats and usage of `r`")
	flag.Parse()

	// Check if `stats` flag is passed
	if *statsPtr {
		stats()
		os.Exit(0)
	}

	// Check if `commands` flag is passed
	if *commandsPtr {
		commands, err := listCommands()
		if err != nil {
			log.Fatal(err)
		}

		for _, c := range commands {
			fmt.Println(c)
		}

		os.Exit(0)
	}

	// Check if `add` flag is passed
	if *addPtr != "" {
		args := strings.Split(*addPtr, ":")
		err := add(args[0], args[1])
		if err != nil {
			log.Fatal(err)
		}
	}
}

// setupDB verifies and creates boltDB in ~ home folder
func setupDB() error {
	// TODO setup boltdb
	fmt.Println("setup boltdb")

	return nil
}

// add checks if command being passed is in the listCommands
// then stores the command and workding directory
func add(path string, promptCmd string) error {
	// get the first command in the promptCmd string
	cmd := strings.Split(promptCmd, " ")[0]

	commands, err := listCommands()
	if err != nil {
		return err
	}

	containsCmd := func() bool {
		for _, c := range commands {
			// check first command against list of commands
			if c == cmd {
				return true
			}
		}
		return false
	}

	// check if the command is valid
	if !containsCmd() {
		return nil
	}

	fmt.Printf("adding. cmd: %s, path: %s \n", promptCmd, path)

	return nil
}

// listCommands use $PATH to find directories
// Then reads each directory and looks for executables
func listCommands() ([]string, error) {
	// Split $PATH directories into slice
	paths := strings.Split(os.Getenv("PATH"), ":")
	var commands []string

	// created buffered error chan
	errc := make(chan error, 1)

	// sync go routines
	var wg sync.WaitGroup

	// find commands appends results to commands slice
	findCommands := func(p string) {
		defer wg.Done()

		files, err := ioutil.ReadDir(p)
		if err != nil {
			errc <- err // write err into error chan
			return
		}

		for _, f := range files {
			m := f.Mode()

			// Check if file is executable
			if m&0111 != 0 {
				commands = append(commands, f.Name())
			}
		}

		errc <- nil // write nil into error chan
	}

	// Check each path for commands
	for _, p := range paths {
		wg.Add(1)
		go findCommands(p)

		// read any error that is in error chan
		if err := <-errc; err != nil {
			return nil, err
		}
	}

	wg.Wait() // Wait for the paths to be checked

	return commands, nil
}

// stats TODO print stats and usage of r
func stats() {
	fmt.Println("stats")
}
<commit_msg>Using readline package in place of bash complete<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/chzyer/readline"
)

func main() {
	// Setup flags
	statsPtr := flag.Bool("stats", false, "show stats and usage of `r`")
	completePtr := flag.String("complete", "", "show all results for `r`")
	addPtr := flag.String("add", "", "show stats and usage of `r`")
	flag.Parse()

	// Check if `stats` flag is passed
	if *statsPtr {
		stats()
		os.Exit(0)
	}

	// Check if `results` flag is passed
	if *completePtr != "" {
		results := showResults(*completePtr)
		for _, result := range results {
			fmt.Println(result)
		}
		os.Exit(0)
	}

	// Check if `add` flag is passed
	if *addPtr != "" {
		args := strings.Split(*addPtr, ":")
		err := add(args[0], args[1])
		if err != nil {
			log.Fatal(err)
		}

		os.Exit(0)
	}

	readLine()
}

func readLine() {
	// create completer from results
	results := showResults("r")
	var pcItems []*readline.PrefixCompleter
	for _, result := range results {
		pcItems = append(pcItems, readline.PcItem(result))
	}
	var completer = readline.NewPrefixCompleter(pcItems...)

	rl, err := readline.NewEx(&readline.Config{
		Prompt:       "> ",
		AutoComplete: completer,
	})
	if err != nil {
		panic(err)
	}
	defer rl.Close()

	for {
		line, err := rl.Readline()
		if err != nil { // io.EOF
			break
		}
		println(line)
	}
}

// setupDB verifies and creates boltDB in ~ home folder
func setupDB() error {
	// TODO setup boltdb
	fmt.Println("setup boltdb")

	return nil
}

func showResults(input string) []string {
	results := []string{"git status", "git clone", "go install", "cd ~", "cd $GOPATH/src/github.com/jesselucas", "ls -la"}

	if input == "r" {
		return results
	}

	// filter
	fmt.Println("filtered: ", input)
	var filtered []string
	for _, result := range results {
		if strings.HasPrefix(result, input) {
			filtered = append(filtered, result)
		}
	}

	return filtered

}

// add checks if command being passed is in the listCommands
// then stores the command and workding directory
func add(path string, promptCmd string) error {
	// get the first command in the promptCmd string
	cmd := strings.Split(promptCmd, " ")[0]

	commands, err := listCommands()
	if err != nil {
		return err
	}

	containsCmd := func() bool {
		for _, c := range commands {
			// check first command against list of commands
			if c == cmd {
				return true
			}
		}
		return false
	}

	// check if the command is valid
	if !containsCmd() {
		return nil
	}

	fmt.Printf("adding. cmd: %s, path: %s \n", promptCmd, path)

	return nil
}

// listCommands use $PATH to find directories
// Then reads each directory and looks for executables
func listCommands() ([]string, error) {
	// Split $PATH directories into slice
	paths := strings.Split(os.Getenv("PATH"), ":")
	var commands []string

	// created buffered error chan
	errc := make(chan error, 1)

	// sync go routines
	var wg sync.WaitGroup

	// find commands appends results to commands slice
	findCommands := func(p string) {
		defer wg.Done()

		files, err := ioutil.ReadDir(p)
		if err != nil {
			errc <- err // write err into error chan
			return
		}

		for _, f := range files {
			m := f.Mode()

			// Check if file is executable
			if m&0111 != 0 {
				commands = append(commands, f.Name())
			}
		}

		errc <- nil // write nil into error chan
	}

	// Check each path for commands
	for _, p := range paths {
		wg.Add(1)
		go findCommands(p)

		// read any error that is in error chan
		if err := <-errc; err != nil {
			return nil, err
		}
	}

	wg.Wait() // Wait for the paths to be checked

	return commands, nil
}

// stats TODO print stats and usage of r
func stats() {
	fmt.Println("stats")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/juju/errors"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"

	compute "google.golang.org/api/compute/v1"
)

var (
	flagDisk = flag.String("disk", "", "disk name to attach to the instance")
	flagPath = flag.String("path", "", "path in the current instance to mount the disk")
)

func main() {
	if err := runSafe(); err != nil {
		log.Println(errors.ErrorStack(err))
	}
}

func runSafe() error {
	flag.Parse()

	if *flagDisk == "" || *flagPath == "" {
		return errors.NotValidf("--disk and --path are required")
	}

	log.Println(" [*] Attaching disk", *flagDisk, "to the instance in path: ", *flagPath)

	client := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.ComputeTokenSource(""),
		},
	}
	service, err := compute.New(client)
	if err != nil {
		return errors.Trace(err)
	}

	log.Println(" > Get metadata...")
	project, err := getMetadata("project/project-id")
	if err != nil {
		return errors.Trace(err)
	}

	rawZone, err := getMetadata("instance/zone")
	if err != nil {
		return errors.Trace(err)
	}
	parts := strings.Split(rawZone, "/")
	zone := parts[len(parts)-1]

	rawInstanceName, err := getMetadata("instance/hostname")
	if err != nil {
		return errors.Trace(err)
	}
	parts = strings.Split(rawInstanceName, ".")
	instanceName := parts[0]

	log.Println(" > Check disk name is correct...")
	if err := checkDiskExists(service, project, zone); err != nil {
		return errors.Trace(err)
	}

	log.Println(" > Check if there is another instance with the disk...")
	instance, err := findAttachedInstance(service, project, zone)
	if err != nil {
		return errors.Trace(err)
	}
	if instance != "" {
		log.Println(" > Deattaching disk from instance:", instance)
		if err := detachDisk(service, project, zone, instance); err != nil {
			return errors.Trace(err)
		}
	}

	log.Println(" > Attach disk to this instance...")
	if err := attachDisk(service, project, zone, instanceName); err != nil {
		return errors.Trace(err)
	}

	log.Println(" [*] Disk attached successfully!")

	return nil
}

func getMetadata(path string) (string, error) {
	u := fmt.Sprintf("http://metadata.google.internal/computeMetadata/v1/%s", path)

	resp, err := http.Get(u)
	if err != nil {
		return "", errors.Trace(err)
	}
	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", errors.Trace(err)
	}

	return string(content), nil
}

func checkDiskExists(service *compute.Service, project, zone string) error {
	_, err := service.Disks.Get(project, zone, *flagDisk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func findAttachedInstance(service *compute.Service, project, zone string) (string, error) {
	instances, err := service.Instances.List(project, zone).Do()
	if err != nil {
		return "", errors.Trace(err)
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			if disk.DeviceName == *flagDisk {
				return instance.Name, nil
			}
		}
	}

	return "", nil
}

func detachDisk(service *compute.Service, project, zone, instance string) error {
	_, err := service.Instances.DetachDisk(project, zone, instance, *flagDisk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func attachDisk(service *compute.Service, project, zone, instance string) error {
	disk := &compute.AttachedDisk{
		DeviceName: *flagDisk,
		Source:     fmt.Sprintf("https://content.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, *flagDisk),
	}
	_, err := service.Instances.AttachDisk(project, zone, instance, disk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}
<commit_msg>Add newline break<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/juju/errors"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"

	compute "google.golang.org/api/compute/v1"
)

var (
	flagDisk = flag.String("disk", "", "disk name to attach to the instance")
	flagPath = flag.String("path", "", "path in the current instance to mount the disk")
)

func main() {
	if err := runSafe(); err != nil {
		log.Println(errors.ErrorStack(err))
	}
}

func runSafe() error {
	flag.Parse()

	if *flagDisk == "" || *flagPath == "" {
		return errors.NotValidf("--disk and --path are required")
	}

	log.Println(" [*] Attaching disk", *flagDisk, "to the instance in path: ", *flagPath, "\n")

	client := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.ComputeTokenSource(""),
		},
	}
	service, err := compute.New(client)
	if err != nil {
		return errors.Trace(err)
	}

	log.Println(" > Get metadata...")
	project, err := getMetadata("project/project-id")
	if err != nil {
		return errors.Trace(err)
	}

	rawZone, err := getMetadata("instance/zone")
	if err != nil {
		return errors.Trace(err)
	}
	parts := strings.Split(rawZone, "/")
	zone := parts[len(parts)-1]

	rawInstanceName, err := getMetadata("instance/hostname")
	if err != nil {
		return errors.Trace(err)
	}
	parts = strings.Split(rawInstanceName, ".")
	instanceName := parts[0]

	log.Println(" > Check disk name is correct...")
	if err := checkDiskExists(service, project, zone); err != nil {
		return errors.Trace(err)
	}

	log.Println(" > Check if there is another instance with the disk...")
	instance, err := findAttachedInstance(service, project, zone)
	if err != nil {
		return errors.Trace(err)
	}
	if instance != "" {
		log.Println(" > Deattaching disk from instance:", instance)
		if err := detachDisk(service, project, zone, instance); err != nil {
			return errors.Trace(err)
		}
	}

	log.Println(" > Attach disk to this instance...")
	if err := attachDisk(service, project, zone, instanceName); err != nil {
		return errors.Trace(err)
	}

	log.Println(" [*] Disk attached successfully!")

	return nil
}

func getMetadata(path string) (string, error) {
	u := fmt.Sprintf("http://metadata.google.internal/computeMetadata/v1/%s", path)

	resp, err := http.Get(u)
	if err != nil {
		return "", errors.Trace(err)
	}
	defer resp.Body.Close()

	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", errors.Trace(err)
	}

	return string(content), nil
}

func checkDiskExists(service *compute.Service, project, zone string) error {
	_, err := service.Disks.Get(project, zone, *flagDisk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func findAttachedInstance(service *compute.Service, project, zone string) (string, error) {
	instances, err := service.Instances.List(project, zone).Do()
	if err != nil {
		return "", errors.Trace(err)
	}
	for _, instance := range instances.Items {
		for _, disk := range instance.Disks {
			if disk.DeviceName == *flagDisk {
				return instance.Name, nil
			}
		}
	}

	return "", nil
}

func detachDisk(service *compute.Service, project, zone, instance string) error {
	_, err := service.Instances.DetachDisk(project, zone, instance, *flagDisk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func attachDisk(service *compute.Service, project, zone, instance string) error {
	disk := &compute.AttachedDisk{
		DeviceName: *flagDisk,
		Source:     fmt.Sprintf("https://content.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s", project, zone, *flagDisk),
	}
	_, err := service.Instances.AttachDisk(project, zone, instance, disk).Do()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"
	"github.com/gfwBreakers/gopac/cmd/build"
	"github.com/gfwBreakers/gopac/cmd/serve"
)

const APP_VER = "0.0.0"

var app = cli.NewApp()

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	app.Name = "gopac"
	app.Usage = "Generate proxy auto-config rules and host them."
	app.Version = APP_VER
	app.Commands = append(app.Commands,
		cli.Command{
			Name:   "build",
			Usage:  "Generate proxy auto-config rules",
			Action: build.Action,
			Flags: []cli.Flag{
				cli.StringFlag{"proxy, x", "SOCKS5 127.0.0.1:8964; SOCKS 127.0.0.1:8964; DIRECT", "Examples: SOCKS5 127.0.0.1:8964; SOCKS 127.0.0.1:8964; PROXY 127.0.0.1:6489"},
			},
		},
		cli.Command{
			Name:   "serve",
			Usage:  "Start pac server",
			Action: serve.Action,
			Flags: []cli.Flag{
				cli.IntFlag{"port, p", 0, "Pac Server Port [OPTIONAL], examples: 8970"},
			},
		},
	)
}

func main() {
	app.Run(os.Args)
}
<commit_msg>tweak port flat type<commit_after>package main

import (
	"os"
	"runtime"

	"github.com/codegangsta/cli"
	"github.com/gfwBreakers/gopac/cmd/build"
	"github.com/gfwBreakers/gopac/cmd/serve"
)

const APP_VER = "0.0.0"

var app = cli.NewApp()

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	app.Name = "gopac"
	app.Usage = "Generate proxy auto-config rules and host them."
	app.Version = APP_VER
	app.Commands = append(app.Commands,
		cli.Command{
			Name:   "build",
			Usage:  "Generate proxy auto-config rules",
			Action: build.Action,
			Flags: []cli.Flag{
				cli.StringFlag{"proxy, x", "SOCKS5 127.0.0.1:8964; SOCKS 127.0.0.1:8964; DIRECT", "Examples: SOCKS5 127.0.0.1:8964; SOCKS 127.0.0.1:8964; PROXY 127.0.0.1:6489"},
			},
		},
		cli.Command{
			Name:   "serve",
			Usage:  "Start pac server",
			Action: serve.Action,
			Flags: []cli.Flag{
				cli.StringFlag{"port, p", "0", "Pac Server Port [OPTIONAL], examples: 8970"},
			},
		},
	)
}

func main() {
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/urfave/cli"
	"github.com/ak1t0/flame/crawler"
	"github.com/ak1t0/flame/format"
	"github.com/ak1t0/flame/reader"
	"log"
	"os"
)

var Version string = "0.0.1"

func main() {
	app := cli.NewApp()
	app.Name = "flame"
	app.Usage = "crawl onion services"
	app.Version = Version
	app.Author = "ak1t0"
	app.Email = "aktoo3097@gmail.com"
	app.Commands = Commands

	app.Run(os.Args)
}

var Commands = []cli.Command{
	commandScan,
}

var commandScan = cli.Command{
	Name: "scan",
	Usage: "Scan onion services",
	Aliases: []string{"s"},
	Action: doScan,
	Flags: []cli.Flag {
		cli.StringFlag{
			Name: "f",
			Usage: "Select log file",
		},
	},
}

func doScan(c *cli.Context) error {
	var target string
	if c.String("f") != "" {
		target = c.String("f")
	} else {
		target = "log.json"
	}

	parsed := reader.ReadJson(target)
	r := crawler.Scan(format.NewOnionLogs(parsed))

	log.Println(r)

	return nil
}

<commit_msg>gofmt<commit_after>package main

import (
	"github.com/ak1t0/flame/crawler"
	"github.com/ak1t0/flame/format"
	"github.com/ak1t0/flame/reader"
	"github.com/urfave/cli"
	"log"
	"os"
)

var Version string = "0.0.1"

func main() {
	app := cli.NewApp()
	app.Name = "flame"
	app.Usage = "crawl onion services"
	app.Version = Version
	app.Author = "ak1t0"
	app.Email = "aktoo3097@gmail.com"
	app.Commands = Commands

	app.Run(os.Args)
}

var Commands = []cli.Command{
	commandScan,
}

var commandScan = cli.Command{
	Name:    "scan",
	Usage:   "Scan onion services",
	Aliases: []string{"s"},
	Action:  doScan,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "f",
			Usage: "Select log file",
		},
	},
}

func doScan(c *cli.Context) error {
	var target string
	if c.String("f") != "" {
		target = c.String("f")
	} else {
		target = "log.json"
	}

	parsed := reader.ReadJson(target)
	r := crawler.Scan(format.NewOnionLogs(parsed))

	log.Println(r)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/mattn/go-sqlite3"
)

const (
	ADDRESS = "https://p.pantsu.cat/"
	LENGTH  = 6
	TEXT    = "$ <command> | curl -F 'p=<-' " + ADDRESS + "\n"
	PORT    = ":9900"
)

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("sqlite3", "./database.db")
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func save(raw []byte) string {
	paste := raw[86 : len(raw)-46]

	s := generateName()
	db, err := sql.Open("sqlite3", "./database.db")
	check(err)
	stmt, err := db.Prepare("INSERT INTO pastebin(id, data) values(?,?)")
	_, err = stmt.Exec(s, html.EscapeString(string(paste)))
	check(err)
	db.Close()

	return s
}

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		param1 := html.EscapeString(r.URL.Query().Get("p"))
		param2 := html.EscapeString(r.URL.Query().Get("lang"))
		db, err := sql.Open("sqlite3", "./database.db")
		var s string
		err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
		check(err)
		db.Close()

		if param1 != "" {
			if param2 != "" {
				highlight := pygments.Highlight(html.UnescapeString(s), param2, "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
				io.WriteString(w, highlight)

			} else {
				io.WriteString(w, html.UnescapeString(s))
			}
		} else {
			io.WriteString(w, TEXT)
		}
	case "POST":
		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}
		name := save(buf)
		io.WriteString(w, ADDRESS+"?p="+name+"\n")
	case "DELETE":
		// Remove the record.
	}
}

func main() {
	http.HandleFunc("/", pasteHandler)
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<commit_msg>Add sql no row handling<commit_after>package main

import (
	"database/sql"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/dchest/uniuri"
	"github.com/ewhal/pygments"
	_ "github.com/mattn/go-sqlite3"
)

const (
	ADDRESS = "https://p.pantsu.cat/"
	LENGTH  = 6
	TEXT    = "$ <command> | curl -F 'p=<-' " + ADDRESS + "\n"
	PORT    = ":9900"
)

func check(err error) {
	if err != nil {
		fmt.Println(err)
	}
}

func generateName() string {
	s := uniuri.NewLen(LENGTH)
	db, err := sql.Open("sqlite3", "./database.db")
	check(err)

	query, err := db.Query("select id from pastebin")
	for query.Next() {
		var id string
		err := query.Scan(&id)
		if err != nil {

		}
		if id == s {
			generateName()
		}
	}
	db.Close()

	return s

}
func save(raw []byte) string {
	paste := raw[86 : len(raw)-46]

	s := generateName()
	db, err := sql.Open("sqlite3", "./database.db")
	check(err)
	stmt, err := db.Prepare("INSERT INTO pastebin(id, data) values(?,?)")
	_, err = stmt.Exec(s, html.EscapeString(string(paste)))
	check(err)
	db.Close()

	return s
}

func pasteHandler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		param1 := html.EscapeString(r.URL.Query().Get("p"))
		param2 := html.EscapeString(r.URL.Query().Get("lang"))
		db, err := sql.Open("sqlite3", "./database.db")
		var s string
		err = db.QueryRow("select data from pastebin where id=?", param1).Scan(&s)
		db.Close()
		if err == sql.ErrNoRows {
			io.WriteString(w, "Error invalid paste")
		} else {
			check(err)
		}

		if param1 != "" {
			if param2 != "" {
				highlight := pygments.Highlight(html.UnescapeString(s), param2, "html", "full, style=autumn,linenos=True, lineanchors=True,anchorlinenos=True,", "utf-8")
				io.WriteString(w, highlight)

			} else {
				io.WriteString(w, html.UnescapeString(s))
			}
		} else {
			io.WriteString(w, TEXT)
		}
	case "POST":
		buf, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), 500)
		}
		name := save(buf)
		io.WriteString(w, ADDRESS+"?p="+name+"\n")
	case "DELETE":
		// Remove the record.
	}
}

func main() {
	http.HandleFunc("/", pasteHandler)
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	log "github.com/Sirupsen/logrus"
	"github.com/mhausenblas/reshifter/pkg/etcd"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	backupTotal *prometheus.CounterVec
)

func init() {
	backupTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "dev",
			Subsystem: "app_server",
			Name:      "backup_total",
			Help:      "The count of backup attempts.",
		},
		[]string{"outcome"},
	)
	prometheus.MustRegister(backupTotal)
}

func main() {
	go api()
	go ui()
	select {}
}

func api() {
	http.Handle("/metrics", promhttp.Handler())
	http.HandleFunc("/v1/version", versionHandler)
	http.HandleFunc("/v1/backup", backupHandler)
	http.HandleFunc("/v1/restore", restoreHandler)
	log.Println("Serving API from /v1")
	_ = http.ListenAndServe(":8080", nil)
}

func versionHandler(w http.ResponseWriter, r *http.Request) {
	version := "0.1"
	fmt.Fprintf(w, "ReShifter in version %s", version)
}

func backupHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	ep := etcd.Endpoint{
		Version: "2",
		URL:     "localhost:2379",
	}
	outcome := "success"
	b, err := etcd.Backup(ep.URL)
	if err != nil {
		outcome = "failed"
		log.Error(err)
	}
	log.Infof("Created backup from %s in %s", ep.URL, b)
	_ = json.NewEncoder(w).Encode(ep)
	backupTotal.WithLabelValues(outcome).Inc()
}

func restoreHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	ep := etcd.Endpoint{
		Version: "2",
		URL:     "localhost:2379",
	}
	// cwd, _ := os.Getwd()
	afile := r.URL.Query().Get("archive")
	// b, err := etcd.Restore(afile, cwd, ep.URL)
	// if err != nil {
	// 	log.Error(err)
	// }
	log.Infof("Restored from %s to %s", afile, ep.URL)
	_ = json.NewEncoder(w).Encode(ep)
}

func ui() {
	fs := http.FileServer(http.Dir("ui"))
	http.Handle("/", fs)
	log.Println("Serving UI from /")
	_ = http.ListenAndServe(":8080", nil)
}
<commit_msg>fixes restore main<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/mhausenblas/reshifter/pkg/etcd"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	backupTotal *prometheus.CounterVec
)

func init() {
	backupTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "dev",
			Subsystem: "app_server",
			Name:      "backup_total",
			Help:      "The count of backup attempts.",
		},
		[]string{"outcome"},
	)
	prometheus.MustRegister(backupTotal)
}

func main() {
	go api()
	go ui()
	select {}
}

func api() {
	http.Handle("/metrics", promhttp.Handler())
	http.HandleFunc("/v1/version", versionHandler)
	http.HandleFunc("/v1/backup", backupHandler)
	http.HandleFunc("/v1/restore", restoreHandler)
	log.Println("Serving API from /v1")
	_ = http.ListenAndServe(":8080", nil)
}

func versionHandler(w http.ResponseWriter, r *http.Request) {
	version := "0.1"
	fmt.Fprintf(w, "ReShifter in version %s", version)
}

func backupHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	ep := etcd.Endpoint{
		Version: "2",
		URL:     "localhost:2379",
	}
	outcome := "success"
	b, err := etcd.Backup(ep.URL)
	if err != nil {
		outcome = "failed"
		log.Error(err)
	}
	log.Infof("Created backup from %s in %s", ep.URL, b)
	_ = json.NewEncoder(w).Encode(ep)
	backupTotal.WithLabelValues(outcome).Inc()
}

func restoreHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	ep := etcd.Endpoint{
		Version: "2",
		URL:     "localhost:2379",
	}
	cwd, _ := os.Getwd()
	afile := r.URL.Query().Get("archive")
	err := etcd.Restore(afile, cwd, ep.URL)
	if err != nil {
		log.Error(err)
	}
	log.Infof("Restored from %s to %s", afile, ep.URL)
	_ = json.NewEncoder(w).Encode(ep)
}

func ui() {
	fs := http.FileServer(http.Dir("ui"))
	http.Handle("/", fs)
	log.Println("Serving UI from /")
	_ = http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/gocarina/gocsv"
	"github.com/sjwhitworth/golearn/base"
	"github.com/sjwhitworth/golearn/evaluation"
	"github.com/sjwhitworth/golearn/knn"
	"io/ioutil"
)

type inputRow struct {
	Value string `csv:"value"`
	Type  string `csv:"type"`
}

func main() {

	inputFile, err := ioutil.ReadFile("input.csv")
	if err != nil {
		panic(err)
	}
	var iData []inputRow
	if err := gocsv.Unmarshal(bytes.NewReader(inputFile), &iData); err != nil {
		panic(err)
	}

	var dataset []extractedRow
	for _, v := range iData {
		row := extractFeatures(v.Value)
		row.Category = v.Type
		dataset = append(dataset, row)
	}

	csv, err := gocsv.MarshalBytes(dataset)
	if err != nil {
		panic(err)
	}
	ioutil.WriteFile("extracted_data.csv", csv, 0644)

	rawData, err := base.ParseCSVToInstances("extracted_data.csv", true)
	if err != nil {
		panic(err)
	}

	cls := knn.NewKnnClassifier("euclidean", "linear", 5)
	trainData, testData := base.InstancesTrainTestSplit(rawData, 0.70)

	cls.Fit(trainData)

	predictions, err := cls.Predict(testData)
	if err != nil {
		panic(err)
	}

	confusionMat, err := evaluation.GetConfusionMatrix(testData, predictions)
	if err != nil {
		panic(fmt.Sprintf("Unable to get confusion matrix: %s", err.Error()))
	}
	fmt.Println(evaluation.GetSummary(confusionMat))

}
<commit_msg>add example<commit_after>package main

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"github.com/gocarina/gocsv"
	"github.com/sjwhitworth/golearn/base"
	"github.com/sjwhitworth/golearn/evaluation"
	"github.com/sjwhitworth/golearn/knn"
	"io/ioutil"
)

type inputRow struct {
	Value string `csv:"value"`
	Type  string `csv:"type"`
}

func main() {

	inputFile, err := ioutil.ReadFile("input.csv")
	if err != nil {
		panic(err)
	}
	var iData []inputRow
	if err := gocsv.Unmarshal(bytes.NewReader(inputFile), &iData); err != nil {
		panic(err)
	}

	var dataset []extractedRow
	for _, v := range iData {
		row := extractFeatures(v.Value)
		row.Category = v.Type
		dataset = append(dataset, row)
	}

	csvData, err := gocsv.MarshalBytes(dataset)
	if err != nil {
		panic(err)
	}
	ioutil.WriteFile("extracted_data.csv", csvData, 0644)

	rawData, err := base.ParseCSVToInstances("extracted_data.csv", true)
	if err != nil {
		panic(err)
	}

	cls := knn.NewKnnClassifier("euclidean", "linear", 5)
	trainData, testData := base.InstancesTrainTestSplit(rawData, 0.70)

	cls.Fit(trainData)

	predictions, err := cls.Predict(testData)
	if err != nil {
		panic(err)
	}

	confusionMat, err := evaluation.GetConfusionMatrix(testData, predictions)
	if err != nil {
		panic(fmt.Sprintf("Unable to get confusion matrix: %s", err.Error()))
	}
	fmt.Println(evaluation.GetSummary(confusionMat))

	// checking with different example

	exampleInputFile, err := ioutil.ReadFile("example.csv")
	if err != nil {
		panic(err)
	}
	var eiData []inputRow
	if err := gocsv.Unmarshal(bytes.NewReader(exampleInputFile), &eiData); err != nil {
		panic(err)
	}
	var edataset []extractedRow
	for _, v := range eiData {
		row := extractFeatures(v.Value)
		row.Category = v.Type
		edataset = append(edataset, row)
	}

	ecsvData, err := gocsv.MarshalBytes(edataset)
	if err != nil {
		panic(err)
	}
	err = ioutil.WriteFile("example_extracted_data.csv", ecsvData, 0644)
	if err != nil {
		panic(err)
	}

	exampleData, err := base.ParseCSVToTemplatedInstances("example_extracted_data.csv", true, rawData)
	if err != nil {
		panic(err)
	}
	check, err := cls.Predict(exampleData)
	if err != nil {
		panic(err)
	}

	exampleFile, err := ioutil.ReadFile("example.csv")
	if err != nil {
		panic(err)
	}

	r := csv.NewReader(bytes.NewReader(exampleFile))
	records, err := r.ReadAll()
	if err != nil {
		panic(err)
	}

	_, size := check.Size()
	for i := 0; i < size; i++ {
		headers := records[0]
		fmt.Printf("%v:%v, type: %v\n", headers[0], records[i+1][0], check.RowString(i))
	}

}
<|endoftext|>