<commit_before>package main

import (
    "os"
    "os/exec"

    "errors"

    "io/ioutil"

    "log"

//    "fmt"
    "strings"
    "runtime"
    "strconv" // For Itoa
//    "encoding/csv"
    "encoding/json"

    "github.com/go-martini/martini"
)

func main() {
    m := martini.Classic()

    // CPU count
    m.Get("/sh/numberofcores.php", func () ([]byte, error) {
        return json.Marshal(runtime.NumCPU())
    })

    // Server's hostname
    m.Get("/sh/hostname.php", func () ([]byte, error) {
        host, err := os.Hostname()

        if err != nil {
            return nil, err
        }

        return json.Marshal(host)
    })

    // PS
    m.Get("/sh/ps.php", func () ([]byte, error) {
        // Run uptime command
        rawOutput, err := exec.Command("ps", "aux").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(parseCommandTable(rawOutput, 1, 1))
    })

    m.Get("/sh/df.php", func () ([]byte, error) {
        // Run uptime command
        rawOutput, err := exec.Command("df", "-Ph").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(parseCommandTable(rawOutput, 1, 1))
    })

    m.Get("/sh/time.php", func () ([]byte, error) {
        raw, err := exec.Command("date").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(string(raw[:]))
    })

    m.Get("/sh/issue.php", func () ([]byte, error) {
        raw, err := exec.Command("uname", "-rsm").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(string(raw[:]))
    })

    m.Get("/sh/users.php", func () ([]byte, error) {
        data, err := ioutil.ReadFile("/etc/passwd")

        if err != nil {
            return nil, err
        }

        lines := strings.Split(string(data), "\n")

        // Output records
        var records [][]string

        for _, line := range lines {
            parts := strings.Split(line, ":")

            // Skip bad or empty lines
            if len(parts) != 7 {
                log.Println(len(parts))
                continue
            }

            // Parse base 10, 16 bit UID integer
            uid, err := strconv.ParseInt(parts[2], 10, 16)

            // Error parsing UID
            if err != nil {
                continue
            }

            userType := "user"

            // Check if system user
            if uid <= 499 {
                userType = "system"
            }

            user := []string{
                // User type
                userType,
                // Username
                parts[0],
                // Home directory
                parts[6],
            }

            records = append(records, user)
        }

        return json.Marshal(records)
    })

    m.Get("/sh/online.php", func () ([]byte, error) {
        raw, err := exec.Command("w").Output()

        if err != nil {
            return nil, err
        }

        // We'll add all the parsed lines here
        var entries [][]string

        // Skip first and last line of output
        for _, entry := range parseCommandTable(raw, 2, 1) {
            entries = append(entries, []string{
                // User
                entry[0],
                // From
                entry[2],
                // Login at
                entry[3],
                // Idle
                entry[4],
            })
        }

        return json.Marshal(entries)
    })

    m.Get("/sh/loadavg.php", func () ([]byte, error) {
        raw, err := exec.Command("w").Output()

        if err != nil {
            return nil, err
        }

        lines := strings.Split(string(raw[:]), "\n")
        headers := strings.Fields(lines[0])

        var cpuLoads [][2]string

        // Last 3 headers are CPU loads, clean them up
        for _, load := range headers[len(headers)-3:] {
            // Remove trailing coma if there
            cleanLoad := strings.Split(string(load), ",")[0]

            loadFloat, err := strconv.ParseFloat(cleanLoad, 32)

            if err != nil {
                continue
            }

            loadPercentage := int(loadFloat * 100) / runtime.NumCPU()

            cpuLoads = append(cpuLoads, [2]string{
                cleanLoad,
                strconv.Itoa(loadPercentage),
            })
        }

        return json.Marshal(cpuLoads)
    })

    m.Get("/sh/ping.php", func () ([]byte, error) {
        HOSTS := [4]string{
            "wikipedia.org",
            "github.com",
            "google.com",
            "gnu.org",
        }

        var entries [][]string

        for _, host := range HOSTS {
            ping, err := pingHost(host, 2)

            if err != nil {
                continue
            }

            entries = append(entries, []string{
                host,
                ping,
            })
        }

        return json.Marshal(entries)
    })

    m.Get("/sh/netstat.php", func () ([]byte, error) {
        raw, err := exec.Command("netstat", "-nt").Output()

        if err != nil {
            return nil, err
        }

        // Get entries as a table
        entries := parseCommandTable(raw, 2, 1)

        // Connections we care about
        var connections [][]string

        // Filter by protocol
        for _, entry := range entries {
            // Skip bad entries or non tcp
            if len(entry) != 6 || (entry[0] != "tcp" && entry[0] != "tcp4") {
                continue
            }

            connections = append(connections, entry)
        }

        // Count connections from a machine
        connectionCount := make(map[string]int)

        // Extract
        for _, conn := range connections {
            // Foreign Address
            remoteAddr := conn[4]

            // Increment counter
            connectionCount[remoteAddr]++
        }

        // Expected output by client
        var output [][2]string

        for remoteAddr, count := range connectionCount {
            // Compensate for difference between OSX and linux
            parts := strings.Split(strings.Replace(remoteAddr, ",", ".", 1), ".")
            ip := strings.Join(parts[:4], ".")
            // port := parts[4]

            output = append(output, [2]string{
                strconv.Itoa(count),
                ip,
            })
        }

        return json.Marshal(output)
    })

    // Serve static files
    m.Get("/.*", martini.Static(""))

    m.Run()
}

func parseCommandTable(rawOutput []byte, headerCount int, footerCount int) [][]string {
    // Convert output to a string (it's not binary data, so this is ok)
    output := string(rawOutput[:])

    // We'll add all the parsed lines here
    var entries [][]string

    // Lines of output
    lines := strings.Split(output, "\n")

    // Skip first and last line of output
    for _, str := range lines[headerCount:len(lines)-footerCount] {
        entries = append(entries, strings.Fields(str))
    }

    return entries
}

// Get average pingTime to a host
func pingHost(hostname string, pingCount int) (string, error) {
    raw, err := exec.Command("ping", "-c", strconv.Itoa(pingCount), hostname).Output()

    if err != nil {
        return "", err
    }

    lines := strings.Split(string(raw[:]), "\n")

    if len(lines) < 2 {
        return "", errors.New("Bad output for ping command")
    }

    // Get 2nd last line
    lastLine := lines[len(lines)-2]

    // Extract average ping time as a string
    pingTime := strings.Split(strings.Split(lastLine, "=")[1], "/")[1]

    return pingTime, nil
}
<commit_msg>Add where command<commit_after>package main

import (
    "os"
    "os/exec"

    "errors"

    "io/ioutil"

    "log"

//    "fmt"
    "strings"
    "runtime"
    "strconv" // For Itoa
//    "encoding/csv"
    "encoding/json"

    "github.com/go-martini/martini"
)

func main() {
    m := martini.Classic()

    // CPU count
    m.Get("/sh/numberofcores.php", func () ([]byte, error) {
        return json.Marshal(runtime.NumCPU())
    })

    // Server's hostname
    m.Get("/sh/hostname.php", func () ([]byte, error) {
        host, err := os.Hostname()

        if err != nil {
            return nil, err
        }

        return json.Marshal(host)
    })

    // PS
    m.Get("/sh/ps.php", func () ([]byte, error) {
        // Run uptime command
        rawOutput, err := exec.Command("ps", "aux").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(parseCommandTable(rawOutput, 1, 1))
    })

    m.Get("/sh/df.php", func () ([]byte, error) {
        // Run uptime command
        rawOutput, err := exec.Command("df", "-Ph").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(parseCommandTable(rawOutput, 1, 1))
    })

    m.Get("/sh/time.php", func () ([]byte, error) {
        raw, err := exec.Command("date").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(string(raw[:]))
    })

    m.Get("/sh/issue.php", func () ([]byte, error) {
        raw, err := exec.Command("uname", "-rsm").Output()

        if err != nil {
            return nil, err
        }

        return json.Marshal(string(raw[:]))
    })

    m.Get("/sh/users.php", func () ([]byte, error) {
        data, err := ioutil.ReadFile("/etc/passwd")

        if err != nil {
            return nil, err
        }

        lines := strings.Split(string(data), "\n")

        // Output records
        var records [][]string

        for _, line := range lines {
            parts := strings.Split(line, ":")

            // Skip bad or empty lines
            if len(parts) != 7 {
                log.Println(len(parts))
                continue
            }

            // Parse base 10, 16 bit UID integer
            uid, err := strconv.ParseInt(parts[2], 10, 16)

            // Error parsing UID
            if err != nil {
                continue
            }

            userType := "user"

            // Check if system user
            if uid <= 499 {
                userType = "system"
            }

            user := []string{
                // User type
                userType,
                // Username
                parts[0],
                // Home directory
                parts[6],
            }

            records = append(records, user)
        }

        return json.Marshal(records)
    })

    m.Get("/sh/online.php", func () ([]byte, error) {
        raw, err := exec.Command("w").Output()

        if err != nil {
            return nil, err
        }

        // We'll add all the parsed lines here
        var entries [][]string

        // Skip first and last line of output
        for _, entry := range parseCommandTable(raw, 2, 1) {
            entries = append(entries, []string{
                // User
                entry[0],
                // From
                entry[2],
                // Login at
                entry[3],
                // Idle
                entry[4],
            })
        }

        return json.Marshal(entries)
    })

    m.Get("/sh/loadavg.php", func () ([]byte, error) {
        raw, err := exec.Command("w").Output()

        if err != nil {
            return nil, err
        }

        lines := strings.Split(string(raw[:]), "\n")
        headers := strings.Fields(lines[0])

        var cpuLoads [][2]string

        // Last 3 headers are CPU loads, clean them up
        for _, load := range headers[len(headers)-3:] {
            // Remove trailing coma if there
            cleanLoad := strings.Split(string(load), ",")[0]

            loadFloat, err := strconv.ParseFloat(cleanLoad, 32)

            if err != nil {
                continue
            }

            loadPercentage := int(loadFloat * 100) / runtime.NumCPU()

            cpuLoads = append(cpuLoads, [2]string{
                cleanLoad,
                strconv.Itoa(loadPercentage),
            })
        }

        return json.Marshal(cpuLoads)
    })

    m.Get("/sh/ping.php", func () ([]byte, error) {
        HOSTS := [4]string{
            "wikipedia.org",
            "github.com",
            "google.com",
            "gnu.org",
        }

        var entries [][]string

        for _, host := range HOSTS {
            ping, err := pingHost(host, 2)

            if err != nil {
                continue
            }

            entries = append(entries, []string{
                host,
                ping,
            })
        }

        return json.Marshal(entries)
    })

    m.Get("/sh/netstat.php", func () ([]byte, error) {
        raw, err := exec.Command("netstat", "-nt").Output()

        if err != nil {
            return nil, err
        }

        // Get entries as a table
        entries := parseCommandTable(raw, 2, 1)

        // Connections we care about
        var connections [][]string

        // Filter by protocol
        for _, entry := range entries {
            // Skip bad entries or non tcp
            if len(entry) != 6 || (entry[0] != "tcp" && entry[0] != "tcp4") {
                continue
            }

            connections = append(connections, entry)
        }

        // Count connections from a machine
        connectionCount := make(map[string]int)

        // Extract
        for _, conn := range connections {
            // Foreign Address
            remoteAddr := conn[4]

            // Increment counter
            connectionCount[remoteAddr]++
        }

        // Expected output by client
        var output [][2]string

        for remoteAddr, count := range connectionCount {
            // Compensate for difference between OSX and linux
            parts := strings.Split(strings.Replace(remoteAddr, ",", ".", 1), ".")
            ip := strings.Join(parts[:4], ".")
            // port := parts[4]

            output = append(output, [2]string{
                strconv.Itoa(count),
                ip,
            })
        }

        return json.Marshal(output)
    })

    m.Get("/sh/where.php", func () ([]byte, error) {
        SOFTWARE := []string{
            "php",
            "node",
            "mysql",
            "vim",
            "python",
            "ruby",
            "java",
            "apache2",
            "nginx",
            "openssl",
            "vsftpd",
            "make",
        }

        var entries [][2]string

        for _, bin := range SOFTWARE {
            path, err := exec.LookPath(bin)

            if err != nil {
                path = "Not Installed"
            }

            entries = append(entries, [2]string{
                bin,
                path,
            })
        }

        return json.Marshal(entries)
    })

    // Serve static files
    m.Get("/.*", martini.Static(""))

    m.Run()
}

func parseCommandTable(rawOutput []byte, headerCount int, footerCount int) [][]string {
    // Convert output to a string (it's not binary data, so this is ok)
    output := string(rawOutput[:])

    // We'll add all the parsed lines here
    var entries [][]string

    // Lines of output
    lines := strings.Split(output, "\n")

    // Skip first and last line of output
    for _, str := range lines[headerCount:len(lines)-footerCount] {
        entries = append(entries, strings.Fields(str))
    }

    return entries
}

// Get average pingTime to a host
func pingHost(hostname string, pingCount int) (string, error) {
    raw, err := exec.Command("ping", "-c", strconv.Itoa(pingCount), hostname).Output()

    if err != nil {
        return "", err
    }

    lines := strings.Split(string(raw[:]), "\n")

    if len(lines) < 2 {
        return "", errors.New("Bad output for ping command")
    }

    // Get 2nd last line
    lastLine := lines[len(lines)-2]

    // Extract average ping time as a string
    pingTime := strings.Split(strings.Split(lastLine, "=")[1], "/")[1]

    return pingTime, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "http://wsdecks.com/"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type cardsConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string) (string, error) {
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	cardList, err := filepath.Glob(dirPath + "*.gif")
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	// PI/S40-056
	for _, card := range cardList {
		card = strings.Replace(card, dirPath, "", 1)
		card = strings.Replace(card, "_", "-", 1)
		card = strings.Replace(card, "_", "/", 1)
		ex := strings.Split(card, ".")[0]
		out.WriteString(ex + "\n")
	}
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		url := hoTcURL + scanner.Text()
		fmt.Println(url)
		doc, err := goquery.NewDocument(url)
		if err != nil {
			fmt.Println(err)
		}
		textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
		if err != nil {
			fmt.Println(err)
		}
		textHTML = strings.Replace(textHTML, "
", "
", -1)
		// html.UnescapeString(textHTML)
		card := cardStruc{ID: scanner.Text(), Translation: html.UnescapeString(textHTML)}

		translations = append(translations, card)
		// json.Marshal(card)
		// doc.Find(".card3").Get(2)
	}
	return translations
}

func getCardsConfig(link string) (cardsConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	cardsConfig := cardsConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return cardsConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return cardsConfig, nil
}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			cardsConfig, err := getCardsConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(cardsConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(cardsConfig.Dir, 0744)
				doc.Find(cardsConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if cardsConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if cardsConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(cardsConfig.Dir, path.Base(url))
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))
						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(cardsConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				for _, file := range files {
					absPath := filepath.Join(cardsConfig.Dir, file.Name())
					urlPath := lowCostSystemToURL(absPath)
					result = append(result, urlPath)
				}

			}
			createCardsCodeFile(cardsConfig.Dir)
			// getTranslationHotC(cardsConfig.Dir)
		}

		wg.Wait()
		fmt.Printf("Finish")
		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<commit_msg>Create code file when all files are save<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	"github.com/vincent-petithory/dataurl"
)

const yuyuteiURL = "http://yuyu-tei.jp/"
const wsDeckURL = "http://wsdecks.com/"
const hoTcURL = "http://www.heartofthecards.com/code/cardlist.html?card=WS_"

// Prox struct
type Prox struct {
	// target url of reverse proxy
	target *url.URL
	// instance of Go ReverseProxy thatwill do the job for us
	proxy *httputil.ReverseProxy
}

type siteConfig struct {
	Name   string
	Filter string
}

type cardsConfig struct {
	Dir  string
	Site siteConfig
}

type cardStruc struct {
	ID          string
	Translation string
}

// New proxy
func New(target string) *Prox {
	url, _ := url.Parse(target)
	// you should handle error on parsing
	return &Prox{target: url, proxy: httputil.NewSingleHostReverseProxy(url)}
}

func (p *Prox) handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("X-GoProxy", "GoProxy")
	// call to magic method from ReverseProxy object
	p.proxy.ServeHTTP(w, r)
}
func lowCostSystemToURL(syspath string) string {
	return strings.Replace(syspath, "\\", "/", -1)
}

func convertToJpg(filePath string) {
	// convert -density 150 -trim to_love-ru_darkness_2nd_trial_deck.pdf -quality 100 -sharpen 0x1.0 love.jpg
	cmd := exec.Command("convert", "-density", "150", "-trim", filePath, "-quality", "100", "-sharpen", "0x1.0", filePath+".jpg")
	err := cmd.Start()
	if err != nil {
		fmt.Println(err)
	}
	err = cmd.Wait()
}

func createCardsCodeFile(dirPath string) (string, error) {
	//TODO Do nothing if file exists
	dirPath += "/"
	out, err := os.Create(dirPath + "codes.txt")
	defer out.Close()
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	cardList, err := filepath.Glob(dirPath + "*.gif")
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	// PI/S40-056
	for _, card := range cardList {
		card = strings.Replace(card, dirPath, "", 1)
		card = strings.Replace(card, "_", "-", 1)
		card = strings.Replace(card, "_", "/", 1)
		ex := strings.Split(card, ".")[0]
		out.WriteString(ex + "\n")
	}
	return out.Name(), nil
}

func getTranslationHotC(codesPath string) []cardStruc {
	translations := []cardStruc{}
	file, err := os.Open(codesPath + "/codes.txt")
	scanner := bufio.NewScanner(file)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println("getTranslationHotC")
	for scanner.Scan() {
		// fmt.Println(scanner.Text())
		url := hoTcURL + scanner.Text()
		fmt.Println(url)
		doc, err := goquery.NewDocument(url)
		if err != nil {
			fmt.Println(err)
		}
		textHTML, err := doc.Find(".cards3").Slice(2, 3).Html()
		if err != nil {
			fmt.Println(err)
		}
		textHTML = strings.Replace(textHTML, "
", "
", -1)
		// html.UnescapeString(textHTML)
		card := cardStruc{ID: scanner.Text(), Translation: html.UnescapeString(textHTML)}

		translations = append(translations, card)
		// json.Marshal(card)
		// doc.Find(".card3").Get(2)
	}
	return translations
}

func getCardsConfig(link string) (cardsConfig, error) {
	uid := ""
	site := siteConfig{}

	if strings.Contains(link, yuyuteiURL) {
		site.Name = "yuyutei"
		site.Filter = ".card_list_box" + " .image img"
		parsedURL, _ := url.Parse(link)
		values, _ := url.ParseQuery(parsedURL.RawQuery)
		uid = values.Get("ver")
	} else if strings.Contains(link, wsDeckURL) {
		site.Name = "wsdeck"
		site.Filter = ".wscard" + " img"
		uid = filepath.Base(link)
	}
	dir := filepath.Join("static", site.Name, uid)
	cardsConfig := cardsConfig{Dir: dir, Site: site}
	if site.Filter == "" {
		return cardsConfig, fmt.Errorf("Url is not supported %v", link)
	}

	return cardsConfig, nil
}

func main() {
	proxy := New("http://localhost:8080")
	os.MkdirAll(filepath.Join("static", "yuyutei"), 0744)
	os.MkdirAll(filepath.Join("static", "wsdeck"), 0744)
	// static := http.FileServer(http.Dir("./"))
	http.HandleFunc("/", proxy.handle)

	http.HandleFunc("/translationimages", func(w http.ResponseWriter, r *http.Request) {
		file := r.PostFormValue("file")
		filename := r.PostFormValue("filename")
		uid := strings.Replace(filename, filepath.Ext(filename), "", 1)
		dir := filepath.Join("static", uid)
		filePath := filepath.Join(dir, filename)

		data, err := dataurl.DecodeString(file)
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		// fmt.Println(dataURL.Data)
		os.MkdirAll(dir, 0777)
		ioutil.WriteFile(filePath, data.Data, 0644)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			convertToJpg(filePath)
		}
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		listJpg, err := filepath.Glob(filePath + "*.jpg")
		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		for i, jpgfile := range listJpg {
			listJpg[i] = lowCostSystemToURL(jpgfile)
		}

		b, err := json.Marshal(listJpg)

		if err != nil {
			fmt.Println(err)
			http.Error(w, err.Error(), 500)
			return
		}

		w.Write(b)
	})

	http.HandleFunc("/cardimages", func(w http.ResponseWriter, r *http.Request) {
		var wg sync.WaitGroup

		result := []string{}
		link := r.PostFormValue("url")

		if link != "" {
			doc, err := goquery.NewDocument(link)
			imageURL := ""
			cardsConfig, err := getCardsConfig(link)

			if err != nil {
				fmt.Println(err)
			}
			if _, err := os.Stat(cardsConfig.Dir); os.IsNotExist(err) {
				os.MkdirAll(cardsConfig.Dir, 0744)
				doc.Find(cardsConfig.Site.Filter).Each(func(i int, s *goquery.Selection) {
					wg.Add(1)
					val, _ := s.Attr("src")
					if cardsConfig.Site.Name == "yuyutei" {
						big := strings.Replace(val, "90_126", "front", 1)
						imageURL = yuyuteiURL + big
					} else if cardsConfig.Site.Name == "wsdeck" {
						imageURL = wsDeckURL + val
					}

					go func(url string) {
						defer wg.Done()
						// fmt.Println("dir : ", dir)
						fileName := filepath.Join(cardsConfig.Dir, path.Base(url))
						out, err := os.Create(fileName)
						if err != nil {
							fmt.Println(err)
						}
						defer out.Close()
						reps, err := http.Get(url)

						if err != nil {
							fmt.Println(err)
						}

						file, err := io.Copy(out, reps.Body)
						if err != nil {
							fmt.Println(err)
						}

						fmt.Println("File", file)
						// fmt.Printf("Link: n-%d __ %v%v\n", i, imageURL, uid)
						defer reps.Body.Close()
						// fmt.Println("image url: ", strings.Replace(fileName, "\\", "/", 1))
						result = append(result, lowCostSystemToURL(fileName))
					}(imageURL)
				})
			} else {
				files, err := ioutil.ReadDir(cardsConfig.Dir)
				if err != nil {
					fmt.Println(err)
				}
				for _, file := range files {
					absPath := filepath.Join(cardsConfig.Dir, file.Name())
					urlPath := lowCostSystemToURL(absPath)
					result = append(result, urlPath)
				}

			}
			// getTranslationHotC(cardsConfig.Dir)
			wg.Wait()
			fmt.Printf("Finish")
			createCardsCodeFile(cardsConfig.Dir)
		}

		b, err := json.Marshal(result)

		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		w.Write(b)
	})

	http.HandleFunc("/static/", func(w http.ResponseWriter, r *http.Request) {
		// fmt.Println("static", r.URL.Path[1:])
		http.ServeFile(w, r, r.URL.Path[1:])
	})

	http.ListenAndServe(":8010", nil)
}
<|endoftext|>


<commit_before>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"math"
	"fmt"
	"os"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
	//WhisperData: "..",
}

type WhisperFetchResponse struct {
	Name        string      `json:"name"`
	StartTime   int         `json:"startTime"`
	StopTime    int         `json:"stopTime"`
	StepTime    int         `json:"stepTime"`
	Values      []float64   `json:"values"`
	IsAbsent    []bool      `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name        string      `json:"name"`
	Paths       []string    `json:"paths"`
}

var log Logger

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	files := make([]string, 0)
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace + 1:rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace + 1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse {
			Name:		glob,
			Paths:		make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for glob %s: %s", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	log.Info("find: %d hits for %s", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		log.Debug("failed to %s", err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		log.Debug("fromTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		log.Debug("untilTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if (w != nil) {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)",
				http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		log.Error("failed to fetch points from %s: %s", path, err)
		http.Error(wr, "Fetching data points failed",
				http.StatusInternalServerError)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse {
			Name:		metric,
			StartTime:	points.FromTime(),
			StopTime:	points.UntilTime(),
			StepTime:	points.Step(),
			Values:		make([]float64, len(values)),
			IsAbsent:	make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for %s: %s", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	log.Info("served %d points for %s", len(values), metric)
	return
}

func main() {
	log = NewOutputLogger(INFO);

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	log.Info("listening on port 8080")
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal("%s", err)
	}
	log.Info("stopped")
}

// Simple wrapper to enable/disable lots of spam
type Logger interface {
	Info(format string, a ...interface{})
	Warn(format string, a ...interface{})
	Error(format string, a ...interface{})
	Fatal(format string, a ...interface{})
	Debug(format string, a ...interface{})
}

type LogLevel int
const (
	FATAL LogLevel = 0
	ERROR LogLevel = 1
	WARN  LogLevel = 2
	INFO  LogLevel = 3
	DEBUG LogLevel = 4
)
type outputLogger struct {
	level LogLevel
	out *os.File
	err *os.File
}

func NewOutputLogger(level LogLevel) *outputLogger {
	r := new(outputLogger)
	r.level = level;
	r.out = os.Stdout;
	r.err = os.Stderr;

	return r
}

func (l *outputLogger) Debug(format string, a ...interface{}) {
	if l.level >= DEBUG {
		l.out.WriteString(fmt.Sprintf("DEBUG: " + format + "\n", a...))
	}
}

func (l *outputLogger) Info(format string, a ...interface{}) {
	if l.level >= INFO {
		l.out.WriteString(fmt.Sprintf("INFO: " + format + "\n", a...))
	}
}

func (l *outputLogger) Warn(format string, a ...interface{}) {
	if l.level >= WARN {
		l.out.WriteString(fmt.Sprintf("WARN: " + format + "\n", a...))
	}
}

func (l *outputLogger) Error(format string, a ...interface{}) {
	if l.level >= ERROR {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
}

func (l *outputLogger) Fatal(format string, a ...interface{}) {
	if l.level >= FATAL {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
	os.Exit(1)
}

<commit_msg>find: bail out if we have nothing to search for<commit_after>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"net/http"
	"encoding/json"
	"path/filepath"
	"strings"
	"strconv"
	"math"
	"fmt"
	"os"
)

var config = struct {
	WhisperData	string
}{
	WhisperData: "/var/lib/carbon/whisper",
	//WhisperData: "..",
}

type WhisperFetchResponse struct {
	Name        string      `json:"name"`
	StartTime   int         `json:"startTime"`
	StopTime    int         `json:"stopTime"`
	StepTime    int         `json:"stepTime"`
	Values      []float64   `json:"values"`
	IsAbsent    []bool      `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name        string      `json:"name"`
	Paths       []string    `json:"paths"`
}

var log Logger

func findHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	if glob == "" {
		log.Warn("dropping invalid request (query=): %s", req.URL.RequestURI())
		http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	files := make([]string, 0)
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace + 1:rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace + 1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData + "/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p) - 4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse {
			Name:		glob,
			Paths:		make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for glob %s: %s", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	log.Info("find: %d hits for %s", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
				format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
				http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		log.Debug("failed to %s", err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		log.Debug("fromTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		log.Debug("untilTime (%s) invalid: %s (in %s)",
				from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if (w != nil) {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)",
				http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		log.Error("failed to fetch points from %s: %s", path, err)
		http.Error(wr, "Fetching data points failed",
				http.StatusInternalServerError)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse {
			Name:		metric,
			StartTime:	points.FromTime(),
			StopTime:	points.UntilTime(),
			StepTime:	points.Step(),
			Values:		make([]float64, len(values)),
			IsAbsent:	make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for %s: %s", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	log.Info("served %d points for %s", len(values), metric)
	return
}

func main() {
	log = NewOutputLogger(INFO);

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	log.Info("listening on port 8080")
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal("%s", err)
	}
	log.Info("stopped")
}

// Simple wrapper to enable/disable lots of spam
type Logger interface {
	Info(format string, a ...interface{})
	Warn(format string, a ...interface{})
	Error(format string, a ...interface{})
	Fatal(format string, a ...interface{})
	Debug(format string, a ...interface{})
}

type LogLevel int
const (
	FATAL LogLevel = 0
	ERROR LogLevel = 1
	WARN  LogLevel = 2
	INFO  LogLevel = 3
	DEBUG LogLevel = 4
)
type outputLogger struct {
	level LogLevel
	out *os.File
	err *os.File
}

func NewOutputLogger(level LogLevel) *outputLogger {
	r := new(outputLogger)
	r.level = level;
	r.out = os.Stdout;
	r.err = os.Stderr;

	return r
}

func (l *outputLogger) Debug(format string, a ...interface{}) {
	if l.level >= DEBUG {
		l.out.WriteString(fmt.Sprintf("DEBUG: " + format + "\n", a...))
	}
}

func (l *outputLogger) Info(format string, a ...interface{}) {
	if l.level >= INFO {
		l.out.WriteString(fmt.Sprintf("INFO: " + format + "\n", a...))
	}
}

func (l *outputLogger) Warn(format string, a ...interface{}) {
	if l.level >= WARN {
		l.out.WriteString(fmt.Sprintf("WARN: " + format + "\n", a...))
	}
}

func (l *outputLogger) Error(format string, a ...interface{}) {
	if l.level >= ERROR {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
}

func (l *outputLogger) Fatal(format string, a ...interface{}) {
	if l.level >= FATAL {
		l.err.WriteString(fmt.Sprintf("ERROR: " + format + "\n", a...))
	}
	os.Exit(1)
}

<|endoftext|>


<commit_before>package main

// http://tsdb.kronos.d:4242/#start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95&o=&yrange=%5B0:%5D&wxh=1420x608&style=linespoint
// http://tsdb.kronos.d:4242/api/query/?start=2016/02/09-08:00:00&end=2016/02/09-11:00:00&m=mimmax:php.timers.mongo.p95

// http://tsdb.kronos.d:4242/#start=2016/02/03-09:23:00&end=2016/02/10-09:23:58&m=mimmax:gauges.do.sphinx.lag&o=&yrange=%5B0:%5D&wxh=1380x636&style=linespoint
import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"time"
)

const tsdbHost = "http://tsdb.kronos.d:4242"

const tpl = `<!DOCTYPE html>

		<title>chart</title>

`

type tsdbResponse struct {
	Metric        string             `json:"metric"`
	Tags          map[string]string  `json:"Tags"`
	AggregateTags []string           `json:"aggregateTags"`
	DataPoints    map[string]float64 `json:"dps"`
}

type DataPoint struct {
	Time  time.Time
	Value float64
}

type DataPoints []DataPoint

func (dps DataPoints) Len() int           { return len(dps) }
func (dps DataPoints) Swap(i, j int)      { dps[i], dps[j] = dps[j], dps[i] }
func (dps DataPoints) Less(i, j int) bool { return dps[i].Time.Before(dps[j].Time) }

type chartData struct {
	Date    string  `json:"date"`
	Value   float64 `json:"val"`
	Average float64 `json:"avg"`
	Sigma   float64 `json:"sigma"`
}

/*
	{
		"date": "2016-02-09 09:20:00",
		"id": "anomaly-01",
		"lineAlpha": 0.5,
		"lineColor": "#FF0000",
		"lineThickness": 5
	}
*/
type anomalyData struct {
	ID        string  `json:"id"`
	Date      string  `json:"date",omitempty`
	Alpha     float64 `json:"lineAlpha",omitempty`
	Color     string  `json:"lineColor"`
	Thickness int64   `json:"lineThickness"`
	Value     int64   `json:"value",omitempty`
	Axis      string  `json:"valueAxis",omitempty`
}

func main() {
	port := "80"
	if os.Getenv("PORT") != "" {
		port = os.Getenv("PORT")
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()

		start := r.URL.Query().Get("start")
		end := r.URL.Query().Get("end")
		m := r.URL.Query().Get("m")
		periodRaw := r.URL.Query().Get("period")

		if m == "" {
			fmt.Fprintln(w, "Берём урл от tsdb, заменяем хост и порт на текущий, вместо '/#start' делаем '/?start'")
			return
		}

		if periodRaw == "" {
			periodRaw = "120"
		}
		period, err := strconv.Atoi(periodRaw)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to atoi: %v", err), http.StatusInternalServerError)
		}

		chart, anomalyes, err := detectAnomalyes(start, end, m, period)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to detect: %v", err), http.StatusInternalServerError)
		}

		anomalyes = append(anomalyes, anomalyData{
			ID:        "guide-1",
			Color:     "#CC0000",
			Thickness: 3,
			Value:     1,
			Axis:      "ValueAxis-2",
		})

		anomalyes = append(anomalyes, anomalyData{
			ID:        "guide-1",
			Color:     "#CC0000",
			Thickness: 3,
			Value:     -1,
			Axis:      "ValueAxis-2",
		})

		chartBytes, err := json.Marshal(chart)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to Marshal: %v", err), http.StatusInternalServerError)
		}

		anomalyesBytes, err := json.Marshal(anomalyes)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to Marshal: %v", err), http.StatusInternalServerError)
		}

		fmt.Fprintf(w, tpl, string(anomalyesBytes), string(chartBytes))

		log.Printf("%s %q %v\n", r.Method, r.URL.String(), time.Since(t))
	})
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func detectAnomalyes(start, end, m string, period int) ([]chartData, []anomalyData, error) {
	path := url.Values{}
	// path.Add("start", "2016/02/09-08:00:00")
	// path.Add("end", "2016/02/09-11:00:00")
	// path.Add("m", "mimmax:php.timers.mongo.p95")
	path.Add("start", start)
	path.Add("end", end)
	path.Add("m", m)

	resp, err := http.Get(fmt.Sprintf("%s/api/query/?%s", tsdbHost, path.Encode()))
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get data from TSDB: %v", err)
	}

	var data []tsdbResponse
	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
		return nil, nil, fmt.Errorf("failed to decode TSDB response: %v", err)
	}

	dps := make(DataPoints, 0)
	for ts, val := range data[0].DataPoints {
		timestamp, err := strconv.Atoi(ts)
		if err != nil {
			continue
		}
		dp := DataPoint{
			Time:  time.Unix(int64(timestamp), 0),
			Value: val,
		}
		dps = append(dps, dp)
	}

	sort.Sort(dps)

	var movingAverage float64
	var sigma float64
	vals := make([]float64, period)

	chart := make([]chartData, 0)
	anomalyes := make([]anomalyData, 0)

	loc, _ := time.LoadLocation("Asia/Novosibirsk")
	for idx, cdp := range dps {
		if idx <= period {
			continue
		}
		points := dps[idx-period : idx]
		for i, dp := range points {
			vals[i] = dp.Value
		}

		movingAverage = average(vals)
		sigma = test3Sigma(vals)
		anomalyIdx := 0
		if sigma > 1.2 {
			anomalyIdx++
			anomalyes = append(anomalyes, anomalyData{
				ID:        fmt.Sprintf("anomaly-%d", anomalyIdx),
				Date:      cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
				Alpha:     0.5,
				Color:     "#FF0000",
				Thickness: 5,
			})
		}
		//fmt.Printf("%v,%v,%v,%v\n", cdp.Time.Format("2006-01-02 15:04:05"), cdp.Value, movingAverage, sigma)

		chart = append(chart, chartData{
			cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
			cdp.Value,
			movingAverage,
			sigma,
		})
	}

	return chart, anomalyes, nil
}

func average(vals []float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		sum += vals[i]
	}
	return sum / float64(len(vals))
}

// Get the standard deviation of float64 values, with
// an input average.
func stddev(vals []float64, avg float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		dis := vals[i] - avg
		sum += dis * dis
	}
	return math.Sqrt(sum / float64(len(vals)))
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func test3Sigma(vals []float64) float64 {
	if len(vals) == 0 {
		return 0
	}

	// Values average and standard deviation.
	avg := average(vals)
	std := stddev(vals, avg)
	last := average(vals[len(vals)-3:])

	if std == 0 {
		switch {
		case last == avg:
			return 0
		case last > avg:
			return 1
		case last < avg:
			return -1
		}
		return 0
	}
	// 3-sigma
	return math.Abs(last-avg) / (3 * std)
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"time"
)

const tsdbHost = "http://tsdb.kronos.d:4242"

const tpl = `<!DOCTYPE html>

		<title>chart</title>

`

type tsdbResponse struct {
	Metric        string             `json:"metric"`
	Tags          map[string]string  `json:"Tags"`
	AggregateTags []string           `json:"aggregateTags"`
	DataPoints    map[string]float64 `json:"dps"`
}

type DataPoint struct {
	Time  time.Time
	Value float64
}

type DataPoints []DataPoint

func (dps DataPoints) Len() int           { return len(dps) }
func (dps DataPoints) Swap(i, j int)      { dps[i], dps[j] = dps[j], dps[i] }
func (dps DataPoints) Less(i, j int) bool { return dps[i].Time.Before(dps[j].Time) }

type chartData struct {
	Date    string  `json:"date"`
	Value   float64 `json:"val"`
	Average float64 `json:"avg"`
	Sigma   float64 `json:"sigma"`
}

/*
	{
		"date": "2016-02-09 09:20:00",
		"id": "anomaly-01",
		"lineAlpha": 0.5,
		"lineColor": "#FF0000",
		"lineThickness": 5
	}
*/
type anomalyData struct {
	ID        string  `json:"id"`
	Date      string  `json:"date",omitempty`
	Alpha     float64 `json:"lineAlpha",omitempty`
	Color     string  `json:"lineColor"`
	Thickness int64   `json:"lineThickness"`
	Value     int64   `json:"value",omitempty`
	Axis      string  `json:"valueAxis",omitempty`
}

func main() {
	port := "80"
	if os.Getenv("PORT") != "" {
		port = os.Getenv("PORT")
	}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		t := time.Now()

		start := r.URL.Query().Get("start")
		end := r.URL.Query().Get("end")
		m := r.URL.Query().Get("m")
		periodRaw := r.URL.Query().Get("period")

		if m == "" {
			fmt.Fprintln(w, "Берём урл от tsdb, заменяем хост и порт на текущий, вместо '/#start' делаем '/?start'")
			return
		}

		if periodRaw == "" {
			periodRaw = "120"
		}
		period, err := strconv.Atoi(periodRaw)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to atoi: %v", err), http.StatusInternalServerError)
		}

		chart, anomalyes, err := detectAnomalyes(start, end, m, period)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to detect: %v", err), http.StatusInternalServerError)
		}

		anomalyes = append(anomalyes, anomalyData{
			ID:        "guide-1",
			Color:     "#CC0000",
			Thickness: 3,
			Value:     1,
			Axis:      "ValueAxis-2",
		})

		anomalyes = append(anomalyes, anomalyData{
			ID:        "guide-1",
			Color:     "#CC0000",
			Thickness: 3,
			Value:     -1,
			Axis:      "ValueAxis-2",
		})

		chartBytes, err := json.Marshal(chart)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to Marshal: %v", err), http.StatusInternalServerError)
		}

		anomalyesBytes, err := json.Marshal(anomalyes)
		if err != nil {
			http.Error(w, fmt.Sprintf("Failed to Marshal: %v", err), http.StatusInternalServerError)
		}

		fmt.Fprintf(w, tpl, string(anomalyesBytes), string(chartBytes))

		log.Printf("%s %q %v\n", r.Method, r.URL.String(), time.Since(t))
	})
	log.Fatal(http.ListenAndServe(":"+port, nil))
}

func detectAnomalyes(start, end, metric string, period int) ([]chartData, []anomalyData, error) {
	loc, _ := time.LoadLocation("Asia/Novosibirsk")
	startTime, err := time.ParseInLocation("2006/01/02-15:04:05", start, loc)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse start date: %v", err)
	}
	endTime, err := time.ParseInLocation("2006/01/02-15:04:05", end, loc)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse end date: %v", err)
	}

	dps, err := getDataPoints(tsdbHost, metric, startTime, endTime)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get data from tsdb: %v", err)
	}

	var movingAverage float64
	var sigma float64
	vals := make([]float64, period)

	chart := make([]chartData, 0)
	anomalyes := make([]anomalyData, 0)

	for idx, cdp := range dps {
		if idx <= period {
			continue
		}
		points := dps[idx-period : idx]
		for i, dp := range points {
			vals[i] = dp.Value
		}

		movingAverage = average(vals)
		sigma = test3Sigma(vals)
		anomalyIdx := 0
		if sigma > 1.2 {
			anomalyIdx++
			anomalyes = append(anomalyes, anomalyData{
				ID:        fmt.Sprintf("anomaly-%d", anomalyIdx),
				Date:      cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
				Alpha:     0.5,
				Color:     "#FF0000",
				Thickness: 5,
			})
		}
		//fmt.Printf("%v,%v,%v,%v\n", cdp.Time.Format("2006-01-02 15:04:05"), cdp.Value, movingAverage, sigma)

		chart = append(chart, chartData{
			cdp.Time.In(loc).Format("2006-01-02 15:04:05"),
			cdp.Value,
			movingAverage,
			sigma,
		})
	}

	return chart, anomalyes, nil
}

func average(vals []float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		sum += vals[i]
	}
	return sum / float64(len(vals))
}

// Get the standard deviation of float64 values, with
// an input average.
func stddev(vals []float64, avg float64) float64 {
	var sum float64
	for i := 0; i < len(vals); i++ {
		dis := vals[i] - avg
		sum += dis * dis
	}
	return math.Sqrt(sum / float64(len(vals)))
}

// Calculate metric score with 3-sigma rule.
//
// What's the 3-sigma rule?
//
//	states that nearly all values (99.7%) lie within the 3 standard deviations
//	of the mean in a normal distribution.
//
// Also like z-score, defined as
//
//	(val - mean) / stddev
//
// And we name the below as metric score, yet 1/3 of z-score
//
//	(val - mean) / (3 * stddev)
//
// The score has
//
//	score > 0   => values is trending up
//	score < 0   => values is trending down
//	score > 1   => values is anomalously trending up
//	score < -1  => values is anomalously trending down
//
// The following function will set the metric score and also the average.
//
func test3Sigma(vals []float64) float64 {
	if len(vals) == 0 {
		return 0
	}

	// Values average and standard deviation.
	avg := average(vals)
	std := stddev(vals, avg)
	last := average(vals[len(vals)-3:])

	if std == 0 {
		switch {
		case last == avg:
			return 0
		case last > avg:
			return 1
		case last < avg:
			return -1
		}
		return 0
	}
	// 3-sigma
	return math.Abs(last-avg) / (3 * std)
}

func getDataPoints(host, metric string, from, to time.Time) (DataPoints, error) {
	path := url.Values{}
	path.Set("start", from.Format("2006/01/02-15:04:05"))
	path.Set("end", to.Format("2006/01/02-15:04:05"))
	path.Set("m", metric)

	response, err := http.Get(fmt.Sprintf("%s/api/query/?%s", host, path.Encode()))
	if err != nil {
		return nil, fmt.Errorf("failed to get data from TSDB: %v", err)
	}

	var data []tsdbResponse
	if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
		return nil, fmt.Errorf("failed to decode TSDB response: %v", err)
	}

	dps := make(DataPoints, 0)
	for ts, val := range data[0].DataPoints {
		timestamp, err := strconv.Atoi(ts)
		if err != nil {
			continue
		}
		dp := DataPoint{
			Time:  time.Unix(int64(timestamp), 0),
			Value: val,
		}
		dps = append(dps, dp)
	}
	sort.Sort(dps)

	return dps, nil
}
<|endoftext|>


<commit_before>// Author: Yoshiyuki Koyanagi <moutend@gmail.com>
// License: mIT

// Package main implements mediumctl.
package main

import (
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"github.com/ericaro/frontmatter"
	medium "github.com/moutend/go-medium"
	"github.com/skratchdot/open-golang/open"
)

type token struct {
	ApplicationID     string
	ApplicationSecret string
	AccessToken       string
	ExpiresAt         int
	RefreshToken      string
}

var (
	version       = "v0.3.0"
	revision      = "dev"
	tokenFilePath string
)

const tokenFileName = ".mediumctl"

func showPostedArticleInfo(p *medium.PostedArticle) {
	fmt.Printf("Your article was successfully posted.\n\n")
	fmt.Printf("title: %s\n", p.Title)
	if p.PublishStatus == "" {
		fmt.Println("publishStatus: public")
	} else {
		fmt.Printf("publishStatus: %s\n", p.PublishStatus)
	}
	if len(p.Tags) > 0 {
		fmt.Printf("tags: %s\n", strings.Join(p.Tags, " "))
	}
	fmt.Printf("URL: %s\n", p.URL)
	if p.CanonicalURL != "" {
		fmt.Printf("canonicalURL: %s\n", p.CanonicalURL)
	}
	return
}

func parseArticle(filename string) (article medium.Article, publicationNumber int, err error) {
	type FrontmatterOption struct {
		Title             string   `fm:"title"`
		ContentFormat     string   `fm:"contentFormat"`
		Content           string   `fm:"content"`
		CanonicalURL      string   `fm:"canonicalUrl"`
		Tags              []string `fm:"tags"`
		PublishStatus     string   `fm:"publishStatus"`
		PublishedAt       string   `fm:"publishedAt"`
		PublicationNumber int      `fm:"publicationNumber"`
		License           string   `fm:"license"`
		NotifyFollowers   bool     `fm:"notifyFollowers"`
	}

	var file []byte
	var fo *FrontmatterOption = &FrontmatterOption{}

	if file, err = ioutil.ReadFile(filename); err != nil {
		return
	}
	if err = frontmatter.Unmarshal(file, fo); err != nil {
		return
	}
	if strings.HasSuffix(filename, "html") || strings.HasSuffix(filename, "htm") {
		fo.ContentFormat = "html"
	} else {
		fo.ContentFormat = "markdown"
	}

	article = medium.Article{
		Title:           fo.Title,
		ContentFormat:   fo.ContentFormat,
		Content:         fo.Content,
		CanonicalURL:    fo.CanonicalURL,
		Tags:            fo.Tags,
		PublishStatus:   fo.PublishStatus,
		PublishedAt:     fo.PublishedAt,
		License:         fo.License,
		NotifyFollowers: false,
	}
	publicationNumber = fo.PublicationNumber

	return
}
func getCode(clientID string, redirectURL *url.URL) (code string, err error) {
	listener, err := net.Listen("tcp", redirectURL.Hostname()+":"+redirectURL.Port())
	if err != nil {
		return
	}
	defer listener.Close()

	responseChann := make(chan string)

	go http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.Write([]byte(`<script>window.open("about:blank","_self").close()</script>`))
		w.(http.Flusher).Flush()

		e := req.FormValue("error")
		if e != "" {
			err = fmt.Errorf(e)
		}
		responseChann <- req.FormValue("code")
	}))

	stateBytes := make([]byte, 88)
	_, err = rand.Read(stateBytes)
	if err != nil {
		return
	}
	state := fmt.Sprintf("%x", stateBytes)
	scope := "basicProfile,listPublications,publishPost"
	query := fmt.Sprintf("client_id=%s&scope=%s&state=%s&response_type=code&redirect_uri=%s", clientID, scope, state, redirectURL)
	uri := "https://medium.com/m/oauth/authorize?" + query
	if err = open.Start(uri); err != nil {
		return
	}
	select {
	case code = <-responseChann:
		break
	case <-time.After(60 * time.Second):
		err = fmt.Errorf("timeout")
		break
	}
	return
}

func saveToken(clientID, clientSecret string, t *medium.Token) (err error) {
	b, err := json.Marshal(token{
		ApplicationID:     clientID,
		ApplicationSecret: clientSecret,
		AccessToken:       t.AccessToken,
		ExpiresAt:         t.ExpiresAt,
		RefreshToken:      t.RefreshToken,
	})
	if err != nil {
		return
	}
	err = ioutil.WriteFile(tokenFilePath, b, 0644)
	return
}

func loadToken() (*token, error) {
	b, err := ioutil.ReadFile(tokenFilePath)
	if err != nil {
		return nil, fmt.Errorf("API token is not set. Please run 'auth' at first")
	}
	var t token
	err = json.Unmarshal(b, &t)
	return &t, err
}

func main() {
	err := run(os.Args)

	if err != nil {
		log.New(os.Stderr, "error: ", 0).Fatal(err)
		os.Exit(1)
	}

	os.Exit(0)
}
func run(args []string) (err error) {
	if len(args) < 2 {
		return helpCommand(args)
	}
	u, err := user.Current()
	if err != nil {
		return
	}
	tokenFilePath = filepath.Join(u.HomeDir, tokenFileName)
	switch args[1] {
	case "o":
		err = authCommand(args)
	case "oauth":
		err = authCommand(args)
	case "r":
		err = refreshCommand(args)
	case "refresh":
		err = refreshCommand(args)
	case "i":
		err = infoCommand(args)
	case "info":
		err = infoCommand(args)
	case "p":
		err = postCommand(args, false)
	case "publication":
		err = postCommand(args, false)
	case "u":
		err = postCommand(args, true)
	case "user":
		err = postCommand(args, true)
	case "v":
		err = versionCommand(args)
	case "version":
		err = versionCommand(args)
	case "h":
		err = helpCommand(args)
	case "help":
		err = helpCommand(args)
	default:
		fmt.Fprintf(os.Stderr, "%s: '%s' is not a %s subcommand.\n", args[0], args[1], args[0])
		err = helpCommand(args)
	}
	return
}

func authCommand(args []string) (err error) {
	var (
		clientIDFlag     string
		clientSecretFlag string
		debugFlag        bool
		redirectURLFlag  string
		redirectURL      *url.URL
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.StringVar(&redirectURLFlag, "u", "", "Redirect URL for OAuth application.")
	f.StringVar(&clientIDFlag, "i", "", "Client ID of OAuth application.")
	f.StringVar(&clientSecretFlag, "s", "", "Client secret of OAuth application.")
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])
	if redirectURLFlag == "" {
		return fmt.Errorf("please specify redirect URI")
	}
	if clientIDFlag == "" {
		return fmt.Errorf("please specify client ID")
	}
	if clientSecretFlag == "" {
		return fmt.Errorf("please specify client secret")
	}
	if redirectURL, err = url.Parse(redirectURLFlag); err != nil {
		return
	}

	code, err := getCode(clientIDFlag, redirectURL)
	if err != nil {
		return
	}
	c := medium.NewClient(clientIDFlag, clientSecretFlag, "")
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	token, err := c.Token(code, redirectURLFlag)
	if err != nil {
		return
	}
	if err = saveToken(clientIDFlag, clientSecretFlag, token); err != nil {
		return
	}
	fmt.Printf("Your API token was successfully saved in '%s'.\n", tokenFilePath)
	fmt.Println("Note: This file should be treated as the password and please do NOT expose it.")
	return
}

func refreshCommand(args []string) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, "")
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	refreshedToken, err := c.RefreshToken(t.RefreshToken)
	if err != nil {
		return
	}
	if err = saveToken(t.ApplicationID, t.ApplicationSecret, refreshedToken); err != nil {
		return
	}
	fmt.Println("Your API token was successfully refreshed.")
	return
}
func infoCommand(args []string) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, t.AccessToken)
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	u, err := c.User()
	if err != nil {
		return
	}
	fmt.Printf("You are logged in as:\n\n")
	fmt.Printf("Name: %s\n", u.Name)
	fmt.Printf("Username: %s\n", u.Username)
	fmt.Printf("URL: %s", u.URL)
	fmt.Printf("\n")

	ps, err := u.Publications()
	if err != nil {
		return
	}
	if len(ps) == 0 {
		fmt.Println("You have no publications yet.")
		return
	}
	fmt.Printf("\nYou have publication(s) below:\n\n")
	for i, p := range ps {
		fmt.Printf("Number: %d\n", i)
		fmt.Printf("Name: %s\n", p.Name)
		fmt.Printf("Description: %s\n", p.Description)
		fmt.Printf("URL: %s\n\n", p.URL)
	}
	return
}

func postCommand(args []string, userFlag bool) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	article, publicationNumber, err := parseArticle(f.Args()[0])
	if err != nil {
		return
	}
	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, t.AccessToken)
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	u, err := c.User()
	if err != nil {
		return
	}
	if userFlag {
		p, err := u.Post(article)
		if err != nil {
			return err
		}
		showPostedArticleInfo(p)
		return nil
	}
	ps, err := u.Publications()
	if err != nil {
		return
	}
	if len(ps) == 0 {
		return fmt.Errorf("you have no publications yet")
	}
	if publicationNumber < 0 || publicationNumber > len(ps)-1 {
		err = fmt.Errorf("publication number '%d' is invalid", publicationNumber)
		return
	}
	p, err := ps[publicationNumber].Post(article)
	if err != nil {
		return
	}
	showPostedArticleInfo(p)
	return
}

func versionCommand(args []string) (err error) {
	fmt.Printf("%s-%s\n", version, revision)
	return
}

func helpCommand(args []string) (err error) {
	fmt.Println(`usage: mediumctl <command> [options]

Commands:
  oauth, o
    Setting up API token with OAuth.
  refresh, r
    Refresh existing API token.
  info, i
    Show the information about current user and its publications.
  user, u
    Post HTML or Markdown file to current user profile.
  publication, p
    Post HTML or Markdown file to current user's publication.
  version, v
  Show version and revision information.
  help, h
    Show this message.

For more information, please see https://github.com/moutend/mediumctl.`)
	return
}
<commit_msg>Refactor<commit_after>// Author: Yoshiyuki Koyanagi <moutend@gmail.com>
// License: mIT

// Package main implements mediumctl.
package main

import (
	"crypto/rand"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"github.com/ericaro/frontmatter"
	medium "github.com/moutend/go-medium"
	"github.com/skratchdot/open-golang/open"
)

type token struct {
	ApplicationID     string
	ApplicationSecret string
	AccessToken       string
	ExpiresAt         int
	RefreshToken      string
}

var (
	version       = "v0.3.0"
	revision      = "dev"
	tokenFilePath string
)

const tokenFileName = ".mediumctl"

func showPostedArticleInfo(p *medium.PostedArticle) {
	fmt.Printf("Your article was successfully posted.\n\n")
	fmt.Printf("title: %s\n", p.Title)
	if p.PublishStatus == "" {
		fmt.Println("publishStatus: public")
	} else {
		fmt.Printf("publishStatus: %s\n", p.PublishStatus)
	}
	if len(p.Tags) > 0 {
		fmt.Printf("tags: %s\n", strings.Join(p.Tags, " "))
	}
	fmt.Printf("URL: %s\n", p.URL)
	if p.CanonicalURL != "" {
		fmt.Printf("canonicalURL: %s\n", p.CanonicalURL)
	}
	return
}

func parseArticle(filename string) (article medium.Article, publicationNumber int, err error) {
	type FrontmatterOption struct {
		Title             string   `fm:"title"`
		ContentFormat     string   `fm:"contentFormat"`
		Content           string   `fm:"content"`
		CanonicalURL      string   `fm:"canonicalUrl"`
		Tags              []string `fm:"tags"`
		PublishStatus     string   `fm:"publishStatus"`
		PublishedAt       string   `fm:"publishedAt"`
		PublicationNumber int      `fm:"publicationNumber"`
		License           string   `fm:"license"`
		NotifyFollowers   bool     `fm:"notifyFollowers"`
	}

	var file []byte
	var fo *FrontmatterOption = &FrontmatterOption{}

	if file, err = ioutil.ReadFile(filename); err != nil {
		return
	}
	if err = frontmatter.Unmarshal(file, fo); err != nil {
		return
	}
	if strings.HasSuffix(filename, "html") || strings.HasSuffix(filename, "htm") {
		fo.ContentFormat = "html"
	} else {
		fo.ContentFormat = "markdown"
	}

	article = medium.Article{
		Title:           fo.Title,
		ContentFormat:   fo.ContentFormat,
		Content:         fo.Content,
		CanonicalURL:    fo.CanonicalURL,
		Tags:            fo.Tags,
		PublishStatus:   fo.PublishStatus,
		PublishedAt:     fo.PublishedAt,
		License:         fo.License,
		NotifyFollowers: false,
	}
	publicationNumber = fo.PublicationNumber

	return
}
func getCode(clientID string, redirectURL *url.URL) (code string, err error) {
	listener, err := net.Listen("tcp", redirectURL.Hostname()+":"+redirectURL.Port())
	if err != nil {
		return
	}
	defer listener.Close()

	responseChann := make(chan string)

	go http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.Write([]byte(`<script>window.open("about:blank","_self").close()</script>`))
		w.(http.Flusher).Flush()

		e := req.FormValue("error")
		if e != "" {
			err = fmt.Errorf(e)
		}
		responseChann <- req.FormValue("code")
	}))

	stateBytes := make([]byte, 88)
	_, err = rand.Read(stateBytes)
	if err != nil {
		return
	}
	state := fmt.Sprintf("%x", stateBytes)
	scope := "basicProfile,listPublications,publishPost"
	query := fmt.Sprintf("client_id=%s&scope=%s&state=%s&response_type=code&redirect_uri=%s", clientID, scope, state, redirectURL)
	uri := "https://medium.com/m/oauth/authorize?" + query
	if err = open.Start(uri); err != nil {
		return
	}
	select {
	case code = <-responseChann:
		break
	case <-time.After(60 * time.Second):
		err = fmt.Errorf("timeout")
		break
	}
	return
}

func saveToken(clientID, clientSecret string, t *medium.Token) (err error) {
	b, err := json.Marshal(token{
		ApplicationID:     clientID,
		ApplicationSecret: clientSecret,
		AccessToken:       t.AccessToken,
		ExpiresAt:         t.ExpiresAt,
		RefreshToken:      t.RefreshToken,
	})
	if err != nil {
		return
	}
	err = ioutil.WriteFile(tokenFilePath, b, 0644)
	return
}

func loadToken() (*token, error) {
	b, err := ioutil.ReadFile(tokenFilePath)
	if err != nil {
		return nil, fmt.Errorf("API token is not set. Please run 'auth' at first")
	}
	var t token
	err = json.Unmarshal(b, &t)
	return &t, err
}

func main() {
	err := run(os.Args)

	if err != nil {
		log.New(os.Stderr, "error: ", 0).Fatal(err)
		os.Exit(1)
	}

	os.Exit(0)
}
func run(args []string) (err error) {
	if len(args) < 2 {
		return helpCommand(args)
	}
	u, err := user.Current()
	if err != nil {
		return
	}
	tokenFilePath = filepath.Join(u.HomeDir, tokenFileName)
	switch args[1] {
	case "o":
		err = authCommand(args)
	case "oauth":
		err = authCommand(args)
	case "r":
		err = refreshCommand(args)
	case "refresh":
		err = refreshCommand(args)
	case "i":
		err = infoCommand(args)
	case "info":
		err = infoCommand(args)
	case "p":
		err = postCommand(args, false)
	case "publication":
		err = postCommand(args, false)
	case "u":
		err = postCommand(args, true)
	case "user":
		err = postCommand(args, true)
	case "v":
		err = versionCommand(args)
	case "version":
		err = versionCommand(args)
	case "h":
		err = helpCommand(args)
	case "help":
		err = helpCommand(args)
	default:
		fmt.Fprintf(os.Stderr, "%s: '%s' is not a %s subcommand.\n", args[0], args[1], args[0])
		err = helpCommand(args)
	}
	return
}

func authCommand(args []string) (err error) {
	var clientIDFlag string
	var clientSecretFlag string
	var debugFlag bool
	var redirectURLFlag string
	var redirectURL *url.URL
	var code string
	var token *medium.Token

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.StringVar(&redirectURLFlag, "u", "", "Redirect URL for OAuth application.")
	f.StringVar(&clientIDFlag, "i", "", "Client ID of OAuth application.")
	f.StringVar(&clientSecretFlag, "s", "", "Client secret of OAuth application.")
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	if redirectURLFlag == "" {
		return fmt.Errorf("please specify redirect URL")
	}
	if clientIDFlag == "" {
		return fmt.Errorf("please specify client ID")
	}
	if clientSecretFlag == "" {
		return fmt.Errorf("please specify client secret")
	}
	if redirectURL, err = url.Parse(redirectURLFlag); err != nil {
		return
	}
	if code, err = getCode(clientIDFlag, redirectURL); err != nil {
		return
	}

	c := medium.NewClient(clientIDFlag, clientSecretFlag, "")
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	if token, err = c.Token(code, redirectURLFlag); err != nil {
		return
	}
	if err = saveToken(clientIDFlag, clientSecretFlag, token); err != nil {
		return
	}

	fmt.Printf("Your API token was successfully saved in '%s'.\n", tokenFilePath)
	fmt.Println("Note: This file should be treated as the password and please do NOT expose it.")

	return
}

func refreshCommand(args []string) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, "")
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	refreshedToken, err := c.RefreshToken(t.RefreshToken)
	if err != nil {
		return
	}
	if err = saveToken(t.ApplicationID, t.ApplicationSecret, refreshedToken); err != nil {
		return
	}
	fmt.Println("Your API token was successfully refreshed.")
	return
}
func infoCommand(args []string) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, t.AccessToken)
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	u, err := c.User()
	if err != nil {
		return
	}
	fmt.Printf("You are logged in as:\n\n")
	fmt.Printf("Name: %s\n", u.Name)
	fmt.Printf("Username: %s\n", u.Username)
	fmt.Printf("URL: %s", u.URL)
	fmt.Printf("\n")

	ps, err := u.Publications()
	if err != nil {
		return
	}
	if len(ps) == 0 {
		fmt.Println("You have no publications yet.")
		return
	}
	fmt.Printf("\nYou have publication(s) below:\n\n")
	for i, p := range ps {
		fmt.Printf("Number: %d\n", i)
		fmt.Printf("Name: %s\n", p.Name)
		fmt.Printf("Description: %s\n", p.Description)
		fmt.Printf("URL: %s\n\n", p.URL)
	}
	return
}

func postCommand(args []string, userFlag bool) (err error) {
	var (
		debugFlag bool
	)

	f := flag.NewFlagSet(fmt.Sprintf("%s %s", args[0], args[1]), flag.ExitOnError)
	f.BoolVar(&debugFlag, "debug", false, "Enable debug output.")
	f.Parse(args[2:])

	article, publicationNumber, err := parseArticle(f.Args()[0])
	if err != nil {
		return
	}
	t, err := loadToken()
	if err != nil {
		return
	}
	c := medium.NewClient(t.ApplicationID, t.ApplicationSecret, t.AccessToken)
	if debugFlag {
		c.SetLogger(log.New(os.Stdout, "debug: ", 0))
	}
	u, err := c.User()
	if err != nil {
		return
	}
	if userFlag {
		p, err := u.Post(article)
		if err != nil {
			return err
		}
		showPostedArticleInfo(p)
		return nil
	}
	ps, err := u.Publications()
	if err != nil {
		return
	}
	if len(ps) == 0 {
		return fmt.Errorf("you have no publications yet")
	}
	if publicationNumber < 0 || publicationNumber > len(ps)-1 {
		err = fmt.Errorf("publication number '%d' is invalid", publicationNumber)
		return
	}
	p, err := ps[publicationNumber].Post(article)
	if err != nil {
		return
	}
	showPostedArticleInfo(p)
	return
}

func versionCommand(args []string) (err error) {
	fmt.Printf("%s-%s\n", version, revision)
	return
}

func helpCommand(args []string) (err error) {
	fmt.Println(`usage: mediumctl <command> [options]

Commands:
  oauth, o
    Setting up API token with OAuth.
  refresh, r
    Refresh existing API token.
  info, i
    Show the information about current user and its publications.
  user, u
    Post HTML or Markdown file to current user profile.
  publication, p
    Post HTML or Markdown file to current user's publication.
  version, v
  Show version and revision information.
  help, h
    Show this message.

For more information, please see https://github.com/moutend/mediumctl.`)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/bigroom/vision/tunnel"
	"github.com/gorilla/websocket"
	"github.com/nickvanw/ircx"
)

var (
	messages chan tunnel.MessageArgs
	clients  map[string][]*conn
)

func main() {
	var (
		host = "localhost"
		port = "8080"
	)

	messages = make(chan tunnel.MessageArgs)
	clients = make(map[string][]*conn)

	go tunnel.NewRPCServer(messages, host, port)
	go messageLoop()

	http.HandleFunc("/", dispatchHandler)

	log.Println(http.ListenAndServe("localhost:6060", nil))
}

func messageLoop() {
	for {
		log.Println("Waiting on message...")
		m := <-messages
		log.Println("Got message")
		log.Printf("Sending to channel with key: %v", m.Key())

		for _, u := range clients[m.Key()] {
			fmt.Println("Writing message")
			err := u.c.WriteJSON(m)
			if err != nil {
				fmt.Println("error (sending message):", err)
			}
		}
	}
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func dispatchHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "PUT")
	w.Header().Add("Access-Control-Allow-Headers", "Content-Type")

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("couldn't upgrade:", err)
		return
	}

	defer c.Close()

	user := &conn{
		c: c,
	}

	for {
		var a action
		err := user.c.ReadJSON(&a)
		if err != nil {
			log.Println("error reading:", err)
			return
		}

		if a.Name == "SET" {
			log.Println("User joined channel", a.Message)
			clients[a.Message] = append(clients[a.Message], user)
		} else if a.Name == "SEND" {
			log.Printf("Sending message '%s' to channel '%s'", a.Message, a.Channel)
		}

	}
}

type conn struct {
	c   *websocket.Conn
	irc *ircx.Bot
}

type action struct {
	Name    string `json:"name"`
	Message string `json:"message"`
	Channel string `json:"channel"`
}
<commit_msg>Add zombie IRC bots sending input<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"

	"github.com/bigroom/vision/tunnel"
	"github.com/gorilla/websocket"
	"github.com/nickvanw/ircx"
	"github.com/sorcix/irc"
)

var (
	messages chan tunnel.MessageArgs
	clients  map[string][]*conn
)

func main() {
	var (
		host = "localhost"
		port = "8080"
	)

	messages = make(chan tunnel.MessageArgs)
	clients = make(map[string][]*conn)

	go tunnel.NewRPCServer(messages, host, port)
	go messageLoop()

	http.HandleFunc("/", dispatchHandler)

	log.Println(http.ListenAndServe("localhost:6060", nil))
}

func messageLoop() {
	for {
		log.Println("Waiting on message...")
		m := <-messages
		log.Println("Got message")
		log.Printf("Sending to channel with key: %v", m.Key())

		for _, u := range clients[m.Key()] {
			fmt.Println("Writing message")
			err := u.c.WriteJSON(m)
			if err != nil {
				fmt.Println("error (sending message):", err)
			}
		}
	}
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func dispatchHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "PUT")
	w.Header().Add("Access-Control-Allow-Headers", "Content-Type")

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("couldn't upgrade:", err)
		return
	}

	defer c.Close()

	zombie := ircx.Classic("chat.freenode.net:6667", fmt.Sprintf("roombot%v", rand.Intn(999)))
	if err != nil {
		log.Println("Could not connect to IRC")
	}

	if err := zombie.Connect(); err != nil {
		log.Println("Unable to connect to IRC", err)
	}

	user := &conn{
		c:        c,
		irc:      zombie,
		messages: make(chan string),
	}

	zombie.HandleFunc(irc.PING, user.pingHandler)
	zombie.HandleFunc(irc.RPL_WELCOME, user.registerHandler)
	zombie.HandleFunc(irc.JOIN, user.messageHandler)

	go zombie.HandleLoop()

	for {
		var a action
		err := user.c.ReadJSON(&a)
		if err != nil {
			log.Println("error reading:", err)
			return
		}

		if a.Name == "SET" {
			log.Println("User joined channel", a.Message)
			clients[a.Message] = append(clients[a.Message], user)
		} else if a.Name == "SEND" {
			log.Printf("Sending message '%s' to channel '%s'", a.Message, a.Channel)
			user.messages <- a.Message
		}

	}
}

type conn struct {
	c        *websocket.Conn
	irc      *ircx.Bot
	messages chan string
}

type action struct {
	Name    string `json:"name"`
	Message string `json:"message"`
	Channel string `json:"channel"`
}

func (c *conn) messageHandler(s ircx.Sender, m *irc.Message) {
	go func() {
		for {
			log.Println("Waiting for message")
			msg := <-c.messages
			log.Println("Got message", msg)
			s.Send(&irc.Message{
				Command:  irc.PRIVMSG,
				Params:   []string{"#roomtest"},
				Trailing: msg,
			})
			log.Println("Message sent")
		}
	}()
}

func (c *conn) registerHandler(s ircx.Sender, m *irc.Message) {
	log.Println("Registering")
	s.Send(&irc.Message{
		Command: irc.JOIN,
		Params:  []string{"#roomtest"},
	})
}

func (c *conn) pingHandler(s ircx.Sender, m *irc.Message) {
	s.Send(&irc.Message{
		Command:  irc.PONG,
		Params:   m.Params,
		Trailing: m.Trailing,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/smtp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/jasonlvhit/gocron"
)

func ExampleScrape() {
	doc, err := goquery.NewDocument("http://www.stm.info/fr/infos/etat-du-service/metro")

	if err != nil {
		log.Fatal(err)
	}

	doc.Find("div.content-block section.item").Each(func(i int, s *goquery.Selection) {
		couleur := s.Find("h2").Text()
		couleur = strings.Trim(couleur, " ")
		couleur = strings.Replace(couleur, "\n", "", -1)
		etat := s.Find("p").Text()
		etat = strings.Trim(etat, " ")
		etat = strings.Replace(etat, "\n", "", -1)
<<<<<<< HEAD
		fmt.Printf("%s - %s\n", couleur, etat)

=======
		if strings.Contains(etat, "Interruption de service") {
			color.Red("%s - %s\n", couleur, etat)
			sendmailInteruption(couleur+etat, couleur)
		} else {
			fmt.Printf("%s - %s\n", couleur, etat)
		}
>>>>>>> 7c248485228c3915ad571ee0bb1fb45d07d132dc
	})
}

func sendmailInteruption(body string, couleur string) {
	from := "stminfogo@gmail.com"
	pass := "allostminfo"
	to := "fprieur@gmail.com"

	msg := "From: " + from + "\n" +
		"To: " + to + "\n" +
		"Subject: Interruption sur la ligne " + couleur + "\n\n" +
		body

	err := smtp.SendMail("smtp.gmail.com:587",
		smtp.PlainAuth("", from, pass, "smtp.gmail.com"),
		from, []string{to}, []byte(msg))

	if err != nil {
		log.Printf("smtp error: %s", err)
		return
	}

	log.Print("mail sent")
}

func main() {

	ExampleScrape()

	gocron.Every(1).Minute().Do(ExampleScrape)
	<-gocron.Start()
}
<commit_msg>testing sendmail on interruption<commit_after>package main

import (
	"fmt"
	"log"
	"net/smtp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/jasonlvhit/gocron"
)

func ExampleScrape() {
	doc, err := goquery.NewDocument("http://www.stm.info/fr/infos/etat-du-service/metro")

	if err != nil {
		log.Fatal(err)
	}

	doc.Find("div.content-block section.item").Each(func(i int, s *goquery.Selection) {
		couleur := s.Find("h2").Text()
		couleur = strings.Trim(couleur, " ")
		couleur = strings.Replace(couleur, "\n", "", -1)
		etat := s.Find("p").Text()
		etat = strings.Trim(etat, " ")
		etat = strings.Replace(etat, "\n", "", -1)
		if strings.Contains(etat, "Interruption de service") {
			color.Red("%s - %s\n", couleur, etat)
			sendmailInteruption(couleur+etat, couleur)
		} else {
			fmt.Printf("%s - %s\n", couleur, etat)
		}
	})
}

func sendmailInteruption(body string, couleur string) {
	from := "stminfogo@gmail.com"
	pass := "allostminfo"
	to := "fprieur@gmail.com"

	msg := "From: " + from + "\n" +
		"To: " + to + "\n" +
		"Subject: Interruption sur la ligne " + couleur + "\n\n" +
		body

	err := smtp.SendMail("smtp.gmail.com:587",
		smtp.PlainAuth("", from, pass, "smtp.gmail.com"),
		from, []string{to}, []byte(msg))

	if err != nil {
		log.Printf("smtp error: %s", err)
		return
	}

	log.Print("mail sent")
}

func main() {

	ExampleScrape()

	gocron.Every(1).Minute().Do(ExampleScrape)
	<-gocron.Start()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

const (
	HeartbeatEvent = "H"
	Query          = "Q"
	Verdict        = "V"

	HeaderLines = 5
)

var (
	mgcDataPath string
	node        string
	eventType   string
)

type Tuple struct {
	Node      string
	EventType string
	EventTime string
	SeqNo     int
	Delay     string
	Verdict   string
}

type MgcTuples []Tuple

func (slice MgcTuples) Len() int {
	return len(slice)
}

func (slice MgcTuples) Less(i, j int) bool {
	l, erl := strconv.Atoi(slice[i].EventTime)
	check(erl)
	r, err := strconv.Atoi(slice[j].EventTime)
	check(err)
	return l < r
}

func (slice MgcTuples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {
	flag.Parse()

	mgcRaw, err := ioutil.ReadFile(mgcDataPath)
	check(err)

	mgcLines := strings.Split(string(mgcRaw), "\n")
	mgcTuples := make(MgcTuples, len(mgcLines)-HeaderLines-1)
	idx := 0

	for i, l := range mgcLines {
		if i < HeaderLines || len(l) == 0 {
			continue
		}

		vals := strings.Fields(l)
		mgcTuples[idx].Node = strings.Trim(vals[0], "<>:")

		f := func(c rune) bool {
			return !unicode.IsLetter(c) && !unicode.IsNumber(c)
		}
		items := strings.FieldsFunc(vals[4], f)
		mgcTuples[idx].EventType = items[0]
		mgcTuples[idx].EventTime = items[1]
		mgcTuples[idx].SeqNo, err = strconv.Atoi(items[2])
		check(err)
		if mgcTuples[idx].EventType == Query {
			mgcTuples[idx].Delay = items[3]
		} else if mgcTuples[idx].EventType == HeartbeatEvent {
			mgcTuples[idx].Delay = items[3]
		} else if mgcTuples[idx].EventType == Verdict {
			mgcTuples[idx].Verdict = items[3]
		}

		idx++
	}

	sort.Sort(mgcTuples)
	output(mgcTuples)
}

func output(mgcTuples []Tuple) {
	for _, t := range mgcTuples {
		if t.Node == node {
			if eventType == HeartbeatEvent && t.EventType == HeartbeatEvent {
				fmt.Printf("%s,%d,%s\n", t.EventTime, t.SeqNo, t.Delay)
			} else if eventType == Query && t.EventType == Query {
				fmt.Printf("%s,%s\n", t.EventTime, t.Delay)
			} else if t.EventType == Verdict {
				fmt.Printf("%s,%s\n", t.EventTime, t.Verdict)
			}
		}
	}
}

func init() {
	flag.StringVar(&mgcDataPath, "file", "",
		"mgc data file")

	flag.StringVar(&node, "node", "",
		"node id")

	flag.StringVar(&eventType, "type", "",
		"event type [(H)eartbeat, (Q)uery, (V)erdict]")
}
<commit_msg>Modify output<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
	"unicode"
)

const (
	HeartbeatEvent = "H"
	Query          = "Q"
	Verdict        = "V"

	HeaderLines = 5
)

var (
	mgcDataPath string
	node        string
	eventType   string
)

type Tuple struct {
	Node      string
	EventType string
	EventTime string
	SeqNo     int
	Delay     string
	Verdict   string
}

type MgcTuples []Tuple

func (slice MgcTuples) Len() int {
	return len(slice)
}

func (slice MgcTuples) Less(i, j int) bool {
	l, erl := strconv.Atoi(slice[i].EventTime)
	check(erl)
	r, err := strconv.Atoi(slice[j].EventTime)
	check(err)
	return l < r
}

func (slice MgcTuples) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {
	flag.Parse()

	mgcRaw, err := ioutil.ReadFile(mgcDataPath)
	check(err)

	mgcLines := strings.Split(string(mgcRaw), "\n")
	mgcTuples := make(MgcTuples, len(mgcLines)-HeaderLines-1)
	idx := 0

	for i, l := range mgcLines {
		if i < HeaderLines || len(l) == 0 {
			continue
		}

		vals := strings.Fields(l)
		mgcTuples[idx].Node = strings.Trim(vals[0], "<>:")

		f := func(c rune) bool {
			return !unicode.IsLetter(c) && !unicode.IsNumber(c)
		}
		items := strings.FieldsFunc(vals[4], f)
		mgcTuples[idx].EventType = items[0]
		mgcTuples[idx].EventTime = items[1]
		mgcTuples[idx].SeqNo, err = strconv.Atoi(items[2])
		check(err)
		if mgcTuples[idx].EventType == Query {
			mgcTuples[idx].Delay = items[3]
		} else if mgcTuples[idx].EventType == HeartbeatEvent {
			mgcTuples[idx].Delay = items[3]
		} else if mgcTuples[idx].EventType == Verdict {
			mgcTuples[idx].Verdict = items[3]
		}

		idx++
	}

	sort.Sort(mgcTuples)
	output(mgcTuples)
}

func output(mgcTuples []Tuple) {
	for _, t := range mgcTuples {
		if t.Node == node {
			if eventType == HeartbeatEvent && t.EventType == HeartbeatEvent {
				fmt.Printf("%s,%s\n", t.EventTime, t.Delay)
			} else if eventType == Query && t.EventType == Query {
				fmt.Printf("%s,%s\n", t.EventTime, t.Delay)
			} else if eventType == Verdict && t.EventType == Verdict {
				fmt.Printf("%s,%s\n", t.EventTime, t.Verdict)
			}
		}
	}
}

func init() {
	flag.StringVar(&mgcDataPath, "file", "",
		"mgc data file")

	flag.StringVar(&node, "node", "",
		"node id")

	flag.StringVar(&eventType, "type", "",
		"event type [(H)eartbeat, (Q)uery, (V)erdict]")
}
<|endoftext|>