<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"os"
	"runtime"
	"sync"
)

const (
	// Timeout for DNS queries
	timeout = 3 * 1e9

	// maximum number of attempts for a query
	maxAttempts = 3
)

var (
	pending         = make(chan *job, 100)
	finished        = make(chan *job, 100)
	done            sync.WaitGroup
	workersCount    = 1
	referenceServer = "8.8.8.8"
	connection      string
	domainArg       string
)

func main() {
	databaseArg := flag.String("database", "database.yml", "Path to file containing the database configuration")
	flag.StringVar(&domainArg, "domains", "domains.txt", "Path to file containing the domain list")
	flag.StringVar(&geoDbPath, "geodb", "GeoLite2-City.mmdb", "Path to GeoDB database")
	flag.StringVar(&referenceServer, "reference", referenceServer, "The nameserver that every other is compared with")
	workersPerCore := flag.Int("workers-per-core", 8, "Number of worker routines per CPU core")
	flag.Parse()

	dnsClient.ReadTimeout = timeout

	environment := os.Getenv("RAILS_ENV")
	if environment == "" {
		environment = "development"
	}

	// read domain list
	if err := readDomains(domainArg); err != nil {
		fmt.Println("unable to read domain list")
		panic(err)
	}

	// load database configuration
	connection = databasePath(*databaseArg, environment)

	// check the GeoDB
	location(referenceServer)

	// Use all cores
	cpuCount := runtime.NumCPU()
	runtime.GOMAXPROCS(cpuCount)

	workersCount = *workersPerCore * cpuCount
	fmt.Println("Starting", workersCount, "workers")
	done.Add(workersCount)

	// Get results from the reference nameserver
	res, err := resolveDomains(referenceServer)
	if err != nil {
		panic(err)
	}
	expectedResults = res

	// Start result writer
	go resultWriter()

	// Start workers
	for i := 0; i < workersCount; i++ {
		go worker()
	}

	createJobs()

	// wait for workers to finish
	done.Wait()

	close(finished)
}

func createJobs() {
	currentId := 0
	batchSize := 1000
	found := batchSize

	// Open SQL connection
	db, err := sql.Open("mysql", connection)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	for batchSize == found {
		// Read the next batch
		rows, err := db.Query("SELECT id, ip FROM nameservers WHERE id > ? LIMIT ?", currentId, batchSize)
		if err != nil {
			panic(err)
		}

		found = 0
		for rows.Next() {
			j := new(job)

			// get RawBytes from data
			err = rows.Scan(&j.id, &j.address)
			if err != nil {
				panic(err)
			}
			pending <- j
			currentId = j.id
			found += 1
		}
		rows.Close()
	}
	close(pending)
}

func worker() {
	for job := range pending {
		executeJob(job)
		finished <- job
	}
	done.Done()
}

func resultWriter() {
	// Open SQL connection
	db, err := sql.Open("mysql", connection)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	stm, err := db.Prepare(
		"UPDATE nameservers SET name=?, state=?, error=?, version=?, checked_at=NOW(), country_id=?, city=?," +
			"state_changed_at = (CASE WHEN ? != state THEN NOW() ELSE state_changed_at END )" +
			"WHERE id=?")
	defer stm.Close()

	for res := range finished {
		log.Println(res)
		stm.Exec(res.name, res.state, res.err, res.version, res.country, res.city, res.state, res.id)
	}
}

// consumes a job and writes the result in the given job
func executeJob(job *job) {
	// log.Println("received job", job.id)

	// GeoDB lookup
	job.country, job.city = location(job.address)

	// Run the check
	err := check(job)
	job.name = ptrName(job.address)

	// query the bind version
	if err == nil || err.Error() != "i/o timeout" {
		job.version = version(job.address)
	}

	if err == nil {
		job.state = "valid"
		job.err = ""
	} else {
		job.state = "invalid"
		job.err = err.Error()
	}
}
<commit_msg>Simplfy workers setting<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"os"
	"sync"
)

const (
	// Timeout for DNS queries
	timeout = 3 * 1e9

	// maximum number of attempts for a query
	maxAttempts = 3
)

var (
	pending         = make(chan *job, 100)
	finished        = make(chan *job, 100)
	done            sync.WaitGroup
	workersCount    = 32
	referenceServer = "8.8.8.8"
	connection      string
	domainArg       string
)

func main() {
	databaseArg := flag.String("database", "database.yml", "Path to file containing the database configuration")
	flag.StringVar(&domainArg, "domains", "domains.txt", "Path to file containing the domain list")
	flag.StringVar(&geoDbPath, "geodb", "GeoLite2-City.mmdb", "Path to GeoDB database")
	flag.StringVar(&referenceServer, "reference", referenceServer, "The nameserver that every other is compared with")
	flag.IntVar(&workersCount, "workers", workersCount, "Number of worker routines")
	flag.Parse()

	dnsClient.ReadTimeout = timeout

	environment := os.Getenv("RAILS_ENV")
	if environment == "" {
		environment = "development"
	}

	// read domain list
	if err := readDomains(domainArg); err != nil {
		fmt.Println("unable to read domain list")
		panic(err)
	}

	// load database configuration
	connection = databasePath(*databaseArg, environment)

	// check the GeoDB
	location(referenceServer)

	// Use all cores

	// Get results from the reference nameserver
	res, err := resolveDomains(referenceServer)
	if err != nil {
		panic(err)
	}
	expectedResults = res

	// Start result writer
	go resultWriter()

	// Start workers
	done.Add(workersCount)
	for i := 0; i < workersCount; i++ {
		go worker()
	}

	createJobs()

	// wait for workers to finish
	done.Wait()

	close(finished)
}

func createJobs() {
	currentId := 0
	batchSize := 1000
	found := batchSize

	// Open SQL connection
	db, err := sql.Open("mysql", connection)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	for batchSize == found {
		// Read the next batch
		rows, err := db.Query("SELECT id, ip FROM nameservers WHERE id > ? LIMIT ?", currentId, batchSize)
		if err != nil {
			panic(err)
		}

		found = 0
		for rows.Next() {
			j := new(job)

			// get RawBytes from data
			err = rows.Scan(&j.id, &j.address)
			if err != nil {
				panic(err)
			}
			pending <- j
			currentId = j.id
			found += 1
		}
		rows.Close()
	}
	close(pending)
}

func worker() {
	for job := range pending {
		executeJob(job)
		finished <- job
	}
	done.Done()
}

func resultWriter() {
	// Open SQL connection
	db, err := sql.Open("mysql", connection)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	stm, err := db.Prepare(
		"UPDATE nameservers SET name=?, state=?, error=?, version=?, checked_at=NOW(), country_id=?, city=?," +
			"state_changed_at = (CASE WHEN ? != state THEN NOW() ELSE state_changed_at END )" +
			"WHERE id=?")
	defer stm.Close()

	for res := range finished {
		log.Println(res)
		stm.Exec(res.name, res.state, res.err, res.version, res.country, res.city, res.state, res.id)
	}
}

// consumes a job and writes the result in the given job
func executeJob(job *job) {
	// log.Println("received job", job.id)

	// GeoDB lookup
	job.country, job.city = location(job.address)

	// Run the check
	err := check(job)
	job.name = ptrName(job.address)

	// query the bind version
	if err == nil || err.Error() != "i/o timeout" {
		job.version = version(job.address)
	}

	if err == nil {
		job.state = "valid"
		job.err = ""
	} else {
		job.state = "invalid"
		job.err = err.Error()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"regexp"
	"syscall"
)

type Options struct {
	InputFile  string `long:"input" short:"i" description:"input Markdown"`
	OutputFile string `long:"output" short:"o" description:"output HTML"`
	EmbedImage bool   `long:"embed" short:"e" description:"embed image by base64 encoding"`
}

const (
	template = `<!DOCTYPE html>

<meta charset="UTF-8">
<title>%s</title>
%s

%s

`

	extensions = blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS
)

func main() {
	var opts Options
	inputs, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if len(opts.InputFile) > 0 {
		inputs = []string{opts.InputFile}
	}

	if len(inputs) <= 0 {
		fmt.Fprintln(os.Stderr, "Please specify input Markdown")
		os.Exit(1)
	}

	for _, input := range inputs {
		files, err := filepath.Glob(input)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		if len(files) <= 0 {
			fmt.Fprintln(os.Stderr, "File is not found")
			os.Exit(1)
		}

		if len(opts.OutputFile) > 0 {
			if err := writeHtmlConcat(files, opts.OutputFile, opts.EmbedImage); err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		} else {
			for _, file := range files {
				if err := writeHtml(file, file+".html", opts.EmbedImage); err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}
		}
	}
}

func writeHtml(input, output string, embed bool) error {
	fi, err := os.Open(input)
	if err != nil {
		return err
	}
	defer fi.Close()

	md, err := ioutil.ReadAll(fi)
	if err != nil {
		return err
	}

	js := string(js_bytes[:len(js_bytes)])
	css := string(css_bytes[:len(css_bytes)])
	html := string(blackfriday.MarkdownCommon(md))

	if embed {
		html, err = embedImage(html, filepath.Dir(input))
		if err != nil {
			return err
		}
	}

	fo, err := os.Create(output)
	if err != nil {
		return err
	}
	defer fo.Close()

	fmt.Fprintf(fo, template, input, js+"\n"+css, html)
	return nil
}

func writeHtmlConcat(inputs []string, output string, embed bool) error {
	js := string(js_bytes[:len(js_bytes)])
	css := string(css_bytes[:len(css_bytes)])
	html := ""

	for _, input := range inputs {
		fi, err := os.Open(input)
		if err != nil {
			return err
		}
		defer fi.Close()

		md, err := ioutil.ReadAll(fi)
		if err != nil {
			return err
		}

		h := string(blackfriday.MarkdownCommon(md))

		if embed {
			h, err = embedImage(h, filepath.Dir(input))
			if err != nil {
				return err
			}
		}

		html += h
	}

	fo, err := os.Create(output)
	if err != nil {
		return err
	}
	defer fo.Close()

	re := regexp.MustCompile(filepath.Ext(output) + "$")
	title := filepath.Base(re.ReplaceAllString(output, ""))
	fmt.Fprintf(fo, template, title, js+"\n"+css, html)
	return nil
}

func embedImage(src, parent string) (string, error) {
	re_find, err := regexp.Compile(`(<img[\S\s]+?src=")([\S\s]+?)("[\S\s]+?/>)`)
	if err != nil {
		return src, err
	}
	img_tags := re_find.FindAllString(src, -1)

	dest := src
	for _, t := range img_tags {
		img_src := re_find.ReplaceAllString(t, "$2")
		img_path := img_src
		if !filepath.IsAbs(img_src) {
			img_path = filepath.Join(parent, img_src)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue
			}
		}

		f, err := os.Open(img_path)
		if err != nil {
			pathErr := err.(*os.PathError)
			errno := pathErr.Err.(syscall.Errno)
			if errno != 0x7B { // suppress ERROR_INVALID_NAME
				fmt.Fprintln(os.Stderr, err)
			}
			continue
		}
		defer f.Close()

		d, err := ioutil.ReadAll(f)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		b64img := base64.StdEncoding.EncodeToString(d)
		re_replace, err := regexp.Compile(`(<img[\S\s]+?src=")` + regexp.QuoteMeta(img_src) + `("[\S\s]+?/>)`)
		if err != nil {
			return src, err
		}

		ext := filepath.Ext(img_src)
		mime_type := mime.TypeByExtension(ext)
		if len(mime_type) <= 0 {
			mime_type = "image"
		}
		dest = re_replace.ReplaceAllString(dest, "${1}data:"+mime_type+";base64,"+b64img+"${2}")
	}
	return dest, nil
}
<commit_msg>Use forked blackfriday<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/nocd5/blackfriday"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"regexp"
	"syscall"
)

type Options struct {
	InputFile  string `long:"input" short:"i" description:"input Markdown"`
	OutputFile string `long:"output" short:"o" description:"output HTML"`
	EmbedImage bool   `long:"embed" short:"e" description:"embed image by base64 encoding"`
}

const (
	template = `<!DOCTYPE html>

<meta charset="UTF-8">
<title>%s</title>
%s

%s

`

	extensions = blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES |
		blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK |
		blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS
)

func main() {
	var opts Options
	inputs, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if len(opts.InputFile) > 0 {
		inputs = []string{opts.InputFile}
	}

	if len(inputs) <= 0 {
		fmt.Fprintln(os.Stderr, "Please specify input Markdown")
		os.Exit(1)
	}

	for _, input := range inputs {
		files, err := filepath.Glob(input)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		if len(files) <= 0 {
			fmt.Fprintln(os.Stderr, "File is not found")
			os.Exit(1)
		}

		if len(opts.OutputFile) > 0 {
			if err := writeHtmlConcat(files, opts.OutputFile, opts.EmbedImage); err != nil {
				fmt.Fprintln(os.Stderr, err)
			}
		} else {
			for _, file := range files {
				if err := writeHtml(file, file+".html", opts.EmbedImage); err != nil {
					fmt.Fprintln(os.Stderr, err)
				}
			}
		}
	}
}

func writeHtml(input, output string, embed bool) error {
	fi, err := os.Open(input)
	if err != nil {
		return err
	}
	defer fi.Close()

	md, err := ioutil.ReadAll(fi)
	if err != nil {
		return err
	}

	js := string(js_bytes[:len(js_bytes)])
	css := string(css_bytes[:len(css_bytes)])
	html := string(blackfriday.MarkdownCommon(md))

	if embed {
		html, err = embedImage(html, filepath.Dir(input))
		if err != nil {
			return err
		}
	}

	fo, err := os.Create(output)
	if err != nil {
		return err
	}
	defer fo.Close()

	fmt.Fprintf(fo, template, input, js+"\n"+css, html)
	return nil
}

func writeHtmlConcat(inputs []string, output string, embed bool) error {
	js := string(js_bytes[:len(js_bytes)])
	css := string(css_bytes[:len(css_bytes)])
	html := ""

	for _, input := range inputs {
		fi, err := os.Open(input)
		if err != nil {
			return err
		}
		defer fi.Close()

		md, err := ioutil.ReadAll(fi)
		if err != nil {
			return err
		}

		h := string(blackfriday.MarkdownCommon(md))

		if embed {
			h, err = embedImage(h, filepath.Dir(input))
			if err != nil {
				return err
			}
		}

		html += h
	}

	fo, err := os.Create(output)
	if err != nil {
		return err
	}
	defer fo.Close()

	re := regexp.MustCompile(filepath.Ext(output) + "$")
	title := filepath.Base(re.ReplaceAllString(output, ""))
	fmt.Fprintf(fo, template, title, js+"\n"+css, html)
	return nil
}

func embedImage(src, parent string) (string, error) {
	re_find, err := regexp.Compile(`(<img[\S\s]+?src=")([\S\s]+?)("[\S\s]+?/>)`)
	if err != nil {
		return src, err
	}
	img_tags := re_find.FindAllString(src, -1)

	dest := src
	for _, t := range img_tags {
		img_src := re_find.ReplaceAllString(t, "$2")
		img_path := img_src
		if !filepath.IsAbs(img_src) {
			img_path = filepath.Join(parent, img_src)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				continue
			}
		}

		f, err := os.Open(img_path)
		if err != nil {
			pathErr := err.(*os.PathError)
			errno := pathErr.Err.(syscall.Errno)
			if errno != 0x7B { // suppress ERROR_INVALID_NAME
				fmt.Fprintln(os.Stderr, err)
			}
			continue
		}
		defer f.Close()

		d, err := ioutil.ReadAll(f)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}

		b64img := base64.StdEncoding.EncodeToString(d)
		re_replace, err := regexp.Compile(`(<img[\S\s]+?src=")` + regexp.QuoteMeta(img_src) + `("[\S\s]+?/>)`)
		if err != nil {
			return src, err
		}

		ext := filepath.Ext(img_src)
		mime_type := mime.TypeByExtension(ext)
		if len(mime_type) <= 0 {
			mime_type = "image"
		}
		dest = re_replace.ReplaceAllString(dest, "${1}data:"+mime_type+";base64,"+b64img+"${2}")
	}
	return dest, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/arjantop/imageoptimizer/optimizer"
)

type ImageDescription struct {
	Optimizer string
	Path      string
	MimeType  string
	Size      int64
}

func reportError(w http.ResponseWriter, msg string, err error) {
	w.WriteHeader(http.StatusInternalServerError)
	log.Printf("%s err=%s", msg, err)
}

func parseAcceptedTypes(acceptHeader string) []string {
	acceptedTypes := make([]string, 0, 1)
	for _, part := range strings.Split(acceptHeader, ",") {
		acceptedType := part
		if strings.Contains(part, ";") {
			acceptedType = strings.SplitN(part, ";", 2)[0]
		}
		acceptedTypes = append(acceptedTypes, acceptedType)
	}
	return acceptedTypes
}

var baseUrl = flag.String("baseurl", "", "Base url to which proxied requests are appended")

func main() {
	flag.Parse()

	if _, err := url.Parse(*baseUrl); *baseUrl == "" || err != nil {
		log.Fatalf("Invalid base url: %s", *baseUrl)
	}

	optimizers := []optimizer.ImageOptimizer{
		&optimizer.WebpLosslessOptimizer{
			Args: []string{"-z", "9"},
		},
		optimizer.NewWebpLossyPngOptimizer(0.998),
		optimizer.NewWebpLossyJpegOptimizer(0.995),
		&optimizer.OptipngOptimizer{
			Args: []string{"-strip", "all"},
		},
		&optimizer.MozjpegOptimizer{
			Args: []string{"-copy", "none", "-optimize"},
		},
		optimizer.NewMozjpegPngLossyOptimizer(0.997),
		optimizer.NewMozjpegLossyOptimizer(0.994),
	}

	client := &http.Client{}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		acceptedTypes := parseAcceptedTypes(r.Header.Get("Accept"))

		requestUrl, err := url.ParseRequestURI(r.RequestURI)
		if err != nil {
			http.Error(w, "Invalid url", http.StatusBadRequest)
			return
		}

		hidpi := strings.Contains(requestUrl.Path, "@2x.")

		log.Printf("Proxying: %s (hidpi=%t)", requestUrl.Path, hidpi)
		resp, err := client.Get(*baseUrl + requestUrl.Path)
		if err != nil {
			reportError(w, "Call failed", err)
			return
		}
		defer resp.Body.Close()

		if !optimizer.CanOptimize(optimizers, resp.Header.Get("Content-Type"), acceptedTypes) {
			for key, vals := range resp.Header {
				for _, val := range vals {
					w.Header().Add(key, val)
				}
			}
			w.WriteHeader(resp.StatusCode)
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				reportError(w, "Could not copy data to client", err)
				return
			}
			return
		}

		tempFile, err := ioutil.TempFile(os.TempDir(), strings.Replace(r.RequestURI, "/", "", -1))
		if err != nil {
			reportError(w, "Could not create temp file", err)
			return
		}
		defer tempFile.Close()

		_, err = io.Copy(tempFile, resp.Body)
		if err != nil {
			reportError(w, "Could not copy data to temp file", err)
			return
		}

		optimizedImage, err := optimizer.Optimize(r.Context(), optimizers, optimizer.OptimizeParams{
			AcceptedTypes: acceptedTypes,
			SourcePath:    tempFile.Name(),
			Hidpi:         hidpi,
		})
		if err != nil {
			reportError(w, "Could not optimize the file", err)
			return
		}

		log.Printf("Chosen optimizer: %s", optimizedImage.Optimizer)

		file, err := os.Open(optimizedImage.Path)
		if err != nil {
			reportError(w, "opening file", err)
			return
		}
		w.Header().Set("Content-Type", optimizedImage.MimeType)
		w.Header().Set("Content-Length", strconv.FormatInt(optimizedImage.Size, 10))
		w.WriteHeader(http.StatusOK)

		_, err = io.Copy(w, file)
		if err != nil {
			reportError(w, "reading file", err)
			return
		}
	})

	log.Fatal(http.ListenAndServe(":8888", nil))
}
<commit_msg>Forward all headers.<commit_after>package main

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/arjantop/imageoptimizer/optimizer"
)

type ImageDescription struct {
	Optimizer string
	Path      string
	MimeType  string
	Size      int64
}

func reportError(w http.ResponseWriter, msg string, err error) {
	w.WriteHeader(http.StatusInternalServerError)
	log.Printf("%s err=%s", msg, err)
}

func parseAcceptedTypes(acceptHeader string) []string {
	acceptedTypes := make([]string, 0, 1)
	for _, part := range strings.Split(acceptHeader, ",") {
		acceptedType := part
		if strings.Contains(part, ";") {
			acceptedType = strings.SplitN(part, ";", 2)[0]
		}
		acceptedTypes = append(acceptedTypes, acceptedType)
	}
	return acceptedTypes
}

var baseUrl = flag.String("baseurl", "", "Base url to which proxied requests are appended")

func main() {
	flag.Parse()

	if _, err := url.Parse(*baseUrl); *baseUrl == "" || err != nil {
		log.Fatalf("Invalid base url: %s", *baseUrl)
	}

	optimizers := []optimizer.ImageOptimizer{
		&optimizer.WebpLosslessOptimizer{
			Args: []string{"-z", "9"},
		},
		optimizer.NewWebpLossyPngOptimizer(0.998),
		optimizer.NewWebpLossyJpegOptimizer(0.995),
		&optimizer.OptipngOptimizer{
			Args: []string{"-strip", "all"},
		},
		&optimizer.MozjpegOptimizer{
			Args: []string{"-copy", "none", "-optimize"},
		},
		optimizer.NewMozjpegPngLossyOptimizer(0.997),
		optimizer.NewMozjpegLossyOptimizer(0.994),
	}

	client := &http.Client{}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		acceptedTypes := parseAcceptedTypes(r.Header.Get("Accept"))

		requestUrl, err := url.ParseRequestURI(r.RequestURI)
		if err != nil {
			http.Error(w, "Invalid url", http.StatusBadRequest)
			return
		}

		hidpi := strings.Contains(requestUrl.Path, "@2x.")

		log.Printf("Proxying: %s (hidpi=%t)", requestUrl.Path+"?"+requestUrl.RawQuery, hidpi)

		req, err := http.NewRequest(http.MethodGet, *baseUrl+requestUrl.Path+"?"+requestUrl.RawQuery, nil)
		if err != nil {
			log.Fatal("Invalid request")
		}
		for key, vals := range r.Header {
			for _, val := range vals {
				req.Header.Set(key, val)
			}
		}

		resp, err := client.Do(req)
		if err != nil {
			reportError(w, "Call failed", err)
			return
		}
		defer resp.Body.Close()

		if !optimizer.CanOptimize(optimizers, resp.Header.Get("Content-Type"), acceptedTypes) {
			for key, vals := range resp.Header {
				for _, val := range vals {
					w.Header().Add(key, val)
				}
			}
			w.WriteHeader(resp.StatusCode)
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				reportError(w, "Could not copy data to client", err)
				return
			}
			return
		}

		tempFile, err := ioutil.TempFile(os.TempDir(), strings.Replace(r.RequestURI, "/", "", -1))
		if err != nil {
			reportError(w, "Could not create temp file", err)
			return
		}
		defer tempFile.Close()

		_, err = io.Copy(tempFile, resp.Body)
		if err != nil {
			reportError(w, "Could not copy data to temp file", err)
			return
		}

		optimizedImage, err := optimizer.Optimize(r.Context(), optimizers, optimizer.OptimizeParams{
			AcceptedTypes: acceptedTypes,
			SourcePath:    tempFile.Name(),
			Hidpi:         hidpi,
		})
		if err != nil {
			reportError(w, "Could not optimize the file", err)
			return
		}

		log.Printf("Chosen optimizer: %s", optimizedImage.Optimizer)

		file, err := os.Open(optimizedImage.Path)
		if err != nil {
			reportError(w, "opening file", err)
			return
		}
		w.Header().Set("Content-Type", optimizedImage.MimeType)
		w.Header().Set("Content-Length", strconv.FormatInt(optimizedImage.Size, 10))
		w.WriteHeader(http.StatusOK)

		_, err = io.Copy(w, file)
		if err != nil {
			reportError(w, "reading file", err)
			return
		}
	})

	log.Fatal(http.ListenAndServe(":8888", nil))
}
<|endoftext|>


<commit_before>package main

// The import section defines libraries that we are going to use in our program.
import (
	"fmt"
	"log"
	"net/http"

	"encoding/json"
	"github.com/rs/cors"
	"github.com/gorilla/mux"
	"github.com/ory-am/common/pkg"
	"github.com/ory-am/common/env"
)

// In a 12 factor app, we must obey the environment variables.
var envHost = env.Getenv("HOST", "")
var envPort = env.Getenv("PORT", "5678")

// Contact defines the structure of a contact which including name, department and company.
type Contact struct {
	ID string `json:"id"`

	// Name is the contact's full name.
	Name string `json:"name"`

	// Department is the contact's department in a company.
	Department string `json:"department"`

	// Company is the name of the company the contact works for.
	Company string `json:"company"`
}

// Contacts is a list of contact structs.
type Contacts map[string]Contact

// MyContacts is an exemplary list of contacts.
var MyContacts = Contacts{
	"john-bravo": Contact{
		Name:       "John Bravo",
		Department: "IT",
		Company:    "ACME Inc",
	},
	"cathrine-mueller": Contact{
		Name:       "Cathrine Müller",
		Department: "HR",
		Company:    "Grove AG",
	},
	"maximilian-schmidt": Contact{
		Name:       "Maximilian Schmidt",
		Department: "PR",
		Company:    "Titanpad AG",
	},
}

// The main routine is going the "entry" point.
func main() {
	// Create a new router.
	router := mux.NewRouter()

	// RESTful defines operations
	// * GET for fetching data
	// * POST for inserting data
	// * PUT for updating existing data
	// * DELETE for deleting data
	router.HandleFunc("/contacts/{id}", UpdateContact(MyContacts)).Methods("PUT")
	router.HandleFunc("/contacts/{id}", DeleteContact(MyContacts)).Methods("DELETE")
	router.HandleFunc("/contacts", ListContacts(MyContacts)).Methods("GET")
	router.HandleFunc("/contacts", AddContact(MyContacts)).Methods("POST")

	// Print some information.
	fmt.Printf("Listening on %s\n", "http://localhost:5678")

	// Cross origin resource requests
	c := cors.New(cors.Options{AllowedOrigins: []string{"*"}})

	// Start up the server and check for errors.
	listenOn := fmt.Sprintf("%s:%s", envHost, envPort)
	err := http.ListenAndServe(listenOn, c.Handler(router))
	if err != nil {
		log.Fatalf("Could not set up server because %s", err)
	}
}

// ListContacts takes a contact list and outputs it.
func ListContacts(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// Write contact list to output
		pkg.WriteIndentJSON(rw, contacts)
	}
}

// AddContact will add a contact to the list
func AddContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// We parse the request's information into contactToBeAdded
		contactToBeAdded, err := ReadContactData(rw, r)

		// Abort handling the request if an error occurs.
		if err != nil {
			return
		}

		// Save newContact to the list of contacts.
		contacts[contactToBeAdded.ID] = contactToBeAdded

		// Output our newly created contact
		pkg.WriteIndentJSON(rw, contactToBeAdded)
	}
}

// DeleteContact will delete a contact from the list
func DeleteContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		// Fetch the ID of the contact that is going to be deleted
		contactToBeDeleted := mux.Vars(r)["id"]

		// Check if the contact exists and return an error if not
		if _, found := contacts[contactToBeDeleted]; !found {
			http.Error(rw, "I do not know any contact by that ID.", http.StatusNotFound)
			return
		}

		// Delete the contact from the list
		delete(contacts, contactToBeDeleted)

		// Per specification, RESTful may return an empty response when a DELETE request was successful
		rw.WriteHeader(http.StatusNoContent)
	}
}

// UpdateContact will update a contact on the list
func UpdateContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		// Fetch the ID of the contact that is going to be updated
		contactToBeUpdated := mux.Vars(r)["id"]

		// Check if the contact exists
		if _, found := contacts[contactToBeUpdated]; !found {
			http.Error(rw, "I don't know any contact by that ID.", http.StatusNotFound)
			return
		}

		// We parse the request's information into newContactData.
		newContactData, err := ReadContactData(rw, r)

		// Abort handling the request if an error occurs.
		if err != nil {
			return
		}

		// Update the data in the contact list.
		delete(contacts, contactToBeUpdated)
		contacts[newContactData.ID] = newContactData

		// Set the new data
		pkg.WriteIndentJSON(rw, newContactData)
	}
}

// ReadContactData is a helper function for parsing a HTTP request body. It returns a contact on success and an
// error if something went wrong.
func ReadContactData(rw http.ResponseWriter, r *http.Request) (contact Contact, err error) {
	err = json.NewDecoder(r.Body).Decode(&contact)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Could not read input data because %s", err), http.StatusBadRequest)
		return contact, err
	}

	return contact, nil
}<commit_msg>fixed tests<commit_after>package main

// The import section defines libraries that we are going to use in our program.
import (
	"fmt"
	"log"
	"net/http"

	"encoding/json"
	"github.com/rs/cors"
	"github.com/gorilla/mux"
	"github.com/ory-am/common/pkg"
	"github.com/ory-am/common/env"
)

// In a 12 factor app, we must obey the environment variables.
var envHost = env.Getenv("HOST", "")
var envPort = env.Getenv("PORT", "5678")

// Contact defines the structure of a contact which including name, department and company.
type Contact struct {
	ID         string `json:"id,omitempty"`

	// Name is the contact's full name.
	Name       string `json:"name"`

	// Department is the contact's department in a company.
	Department string `json:"department"`

	// Company is the name of the company the contact works for.
	Company    string `json:"company"`
}

// Contacts is a list of contact structs.
type Contacts map[string]Contact

// MyContacts is an exemplary list of contacts.
var MyContacts = Contacts{
	"john-bravo": Contact{
		Name:       "John Bravo",
		Department: "IT",
		Company:    "ACME Inc",
	},
	"cathrine-mueller": Contact{
		Name:       "Cathrine Müller",
		Department: "HR",
		Company:    "Grove AG",
	},
	"maximilian-schmidt": Contact{
		Name:       "Maximilian Schmidt",
		Department: "PR",
		Company:    "Titanpad AG",
	},
}

// The main routine is going the "entry" point.
func main() {
	// Create a new router.
	router := mux.NewRouter()

	// RESTful defines operations
	// * GET for fetching data
	// * POST for inserting data
	// * PUT for updating existing data
	// * DELETE for deleting data
	router.HandleFunc("/contacts/{id}", UpdateContact(MyContacts)).Methods("PUT")
	router.HandleFunc("/contacts/{id}", DeleteContact(MyContacts)).Methods("DELETE")
	router.HandleFunc("/contacts", ListContacts(MyContacts)).Methods("GET")
	router.HandleFunc("/contacts", AddContact(MyContacts)).Methods("POST")

	// Print some information.
	fmt.Printf("Listening on %s\n", "http://localhost:5678")

	// Cross origin resource requests
	c := cors.New(cors.Options{
		AllowedOrigins: []string{"*"},
		AllowedMethods: []string{"GET", "POST", "DELETE", "PUT"}},
	)

	// Start up the server and check for errors.
	listenOn := fmt.Sprintf("%s:%s", envHost, envPort)
	err := http.ListenAndServe(listenOn, c.Handler(router))
	if err != nil {
		log.Fatalf("Could not set up server because %s", err)
	}
}

// ListContacts takes a contact list and outputs it.
func ListContacts(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// Write contact list to output
		pkg.WriteIndentJSON(rw, contacts)
	}
}

// AddContact will add a contact to the list
func AddContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {

		// We parse the request's information into contactToBeAdded
		contactToBeAdded, err := ReadContactData(rw, r)

		// Abort handling the request if an error occurs.
		if err != nil {
			return
		}

		// Save newContact to the list of contacts.
		contacts[contactToBeAdded.ID] = contactToBeAdded

		// Output our newly created contact
		pkg.WriteIndentJSON(rw, contactToBeAdded)
	}
}

// DeleteContact will delete a contact from the list
func DeleteContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		// Fetch the ID of the contact that is going to be deleted
		contactToBeDeleted := mux.Vars(r)["id"]

		// Check if the contact exists and return an error if not
		if _, found := contacts[contactToBeDeleted]; !found {
			http.Error(rw, "I do not know any contact by that ID.", http.StatusNotFound)
			return
		}

		// Delete the contact from the list
		delete(contacts, contactToBeDeleted)

		// Per specification, RESTful may return an empty response when a DELETE request was successful
		rw.WriteHeader(http.StatusNoContent)
	}
}

// UpdateContact will update a contact on the list
func UpdateContact(contacts Contacts) func(rw http.ResponseWriter, r *http.Request) {
	return func(rw http.ResponseWriter, r *http.Request) {
		// Fetch the ID of the contact that is going to be updated
		contactToBeUpdated := mux.Vars(r)["id"]

		// Check if the contact exists
		if _, found := contacts[contactToBeUpdated]; !found {
			http.Error(rw, "I don't know any contact by that ID.", http.StatusNotFound)
			return
		}

		// We parse the request's information into newContactData.
		newContactData, err := ReadContactData(rw, r)

		// Abort handling the request if an error occurs.
		if err != nil {
			return
		}

		// Update the data in the contact list.
		delete(contacts, contactToBeUpdated)
		contacts[newContactData.ID] = newContactData

		// Set the new data
		pkg.WriteIndentJSON(rw, newContactData)
	}
}

// ReadContactData is a helper function for parsing a HTTP request body. It returns a contact on success and an
// error if something went wrong.
func ReadContactData(rw http.ResponseWriter, r *http.Request) (contact Contact, err error) {
	err = json.NewDecoder(r.Body).Decode(&contact)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Could not read input data because %s", err), http.StatusBadRequest)
		return contact, err
	}

	return contact, nil
}<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/alexflint/go-arg"
	"github.com/brentp/gargs/process"
	"github.com/fatih/color"
	isatty "github.com/mattn/go-isatty"
	"github.com/valyala/fasttemplate"
)

// Version is the current version
const Version = "0.3.4-dev"

// ExitCode is the highest exit code seen in any command
var ExitCode = 0

// Params are the user-specified command-line arguments
type Params struct {
	Procs       int      `arg:"-p,help:number of processes to use."`
	Nlines      int      `arg:"-n,help:number of lines to consume for each command. -s and -n are mutually exclusive."`
	Retry       int      `arg:"-r,help:number of times to retry a command if it fails (default is 0)."`
	Sep         string   `arg:"-s,help:regular expression split line with to fill multiple template spots default is not to split. -s and -n are mutually exclusive."`
	Verbose     bool     `arg:"-v,help:print commands to stderr as they are executed."`
	StopOnError bool     `arg:"-s,--stop-on-error,help:stop execution on any error. default is to report errors and continue execution."`
	DryRun      bool     `arg:"-d,--dry-run,help:print (but do not run) the commands."`
	Log         string   `arg:"-l,--log,help:file to log commands. Successful commands are prefixed with '#'."`
	Command     string   `arg:"positional,required,help:command to execute."`
	log         *os.File `arg:"-"`
}

// isStdin checks if we are getting data from stdin.
func isStdin() bool {
	// http://stackoverflow.com/a/26567513
	stat, err := os.Stdin.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}

func main() {
	args := Params{Procs: 1, Nlines: 1}
	p := arg.MustParse(&args)
	if args.Sep != "" && args.Nlines > 1 {
		p.Fail("must specify either sep (-s) or n-lines (-n), not both")
	}
	// if neither is specified then we default to whitespace
	if args.Nlines == 1 && args.Sep == "" {
		args.Sep = "\\s+"
	}
	if !isStdin() {
		fmt.Fprintln(os.Stderr, color.RedString("ERROR: expecting input on STDIN"))
		os.Exit(255)
	}
	if args.Log != "" {
		var err error
		args.log, err = os.Create(args.Log)
		check(err)
	}
	runtime.GOMAXPROCS(args.Procs)
	run(args)
	os.Exit(ExitCode)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func handleCommand(args *Params, cmd string, ch chan string) {
	if args.DryRun {
		fmt.Fprintf(os.Stdout, "%s\n", cmd)
		return
	}
	ch <- cmd
}

func fillTmplMap(toks []string, line string) map[string]interface{} {
	m := make(map[string]interface{}, 5)
	if toks != nil {
		for i, t := range toks {
			m[strconv.FormatInt(int64(i), 10)] = t
		}
	}
	m["Line"] = line
	return m
}

func getScanner() *bufio.Scanner {
	scanner := bufio.NewScanner(os.Stdin)
	scanner.Buffer(make([]byte, 0, 16384), 5e9)
	//if rs := os.Getenv("RS"); rs != "" && rs != "\n" && rs != "\r\n" {
	if rs := os.Getenv("RS"); rs != "" {
		brs := []byte(rs)
		scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
			if atEOF && len(data) == 0 {
				return 0, nil, nil
			}
			if i := bytes.Index(data, brs); i >= 0 {
				// Note that by adding the length here, we include the delimiter.
				return i + 1, data[:i+len(brs)], nil
			}
			if atEOF {
				return len(data), data, nil
			}
			return 0, nil, nil
		})
	}
	return scanner
}

func genCommands(args *Params, tmpl *fasttemplate.Template) <-chan string {
	ch := make(chan string)
	var resep *regexp.Regexp
	if args.Sep != "" {
		resep = regexp.MustCompile(args.Sep)
	}

	scanner := getScanner()
	fs := os.Getenv("FS")
	if fs == "" {
		fs = " "
	}

	go func() {
		var lines []string
		if resep == nil {
			lines = make([]string, 0, args.Nlines)
		}
		var buf bytes.Buffer
		for scanner.Scan() {
			buf.Reset()
			line := scanner.Text()
			serr := scanner.Err()
			if serr == nil || (serr == io.EOF && len(line) > 0) {
				// TODO: make dropping bytes optional.
				if resep != nil {
					toks := resep.Split(line, -1)
					targs := fillTmplMap(toks, line)
					_, err := tmpl.Execute(&buf, targs)
					check(err)
					handleCommand(args, buf.String(), ch)
				} else {
					lines = append(lines, line)
				}
			} else {
				if serr == io.EOF {
					break
				}
				log.Fatal(serr)
			}
			if len(lines) >= args.Nlines {
				targs := fillTmplMap(lines, strings.Join(lines, fs))
				_, err := tmpl.Execute(&buf, targs)
				check(err)
				lines = lines[:0]
				handleCommand(args, buf.String(), ch)
			}
		}
		if len(lines) > 0 {
			targs := fillTmplMap(lines, strings.Join(lines, fs))
			_, err := tmpl.Execute(&buf, targs)
			check(err)
			handleCommand(args, buf.String(), ch)
		}
		close(ch)
	}()
	return ch
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func init() {
	color.NoColor = !isatty.IsTerminal(os.Stderr.Fd())
}

func run(args Params) {

	tmpl := makeCommandTmpl(args.Command)
	cmds := genCommands(&args, tmpl)

	stdout := bufio.NewWriter(os.Stdout)
	defer stdout.Flush()

	cancel := make(chan bool)
	defer close(cancel)
	fails := 0

	// flush stdout every 2 seconds.
	last := time.Now().Add(2 * time.Second)
	for p := range process.Runner(cmds, args.Retry, cancel) {
		if ex := p.ExitCode(); ex != 0 {
			c := color.New(color.BgRed).Add(color.Bold)
			fmt.Fprintf(os.Stderr, "%s\n", c.SprintFunc()(fmt.Sprintf("ERROR with command: %s", p)))
			ExitCode = max(ExitCode, ex)
			fails++
			if args.StopOnError {
				break
			}
		}
		if args.Verbose {
			fmt.Fprintf(os.Stderr, "%s\n", p)
		}
		_, err := io.Copy(stdout, p)
		check(err)

		p.Cleanup()
		if t := time.Now(); t.After(last) {
			stdout.Flush()
			last = t.Add(2 * time.Second)
		}
		if args.log != nil {
			// if no error prefix the command with '#'
			if p.ExitCode() == 0 {
				args.log.WriteString("# " + strings.Replace(p.CmdStr, "\n", "\n# ", -1) + "\n")
			} else {
				args.log.WriteString(p.CmdStr + "\n")
			}
			stdout.Flush()
		}
	}
	stdout.Flush()
	if ExitCode == 0 && args.log != nil {
		args.log.WriteString("# SUCCESS\n")
	} else if args.log != nil {
		fmt.Fprintf(args.log, "# FAILED %d commands\n", fails)
	}

}

func makeCommandTmpl(cmd string) *fasttemplate.Template {
	v := strings.Replace(cmd, "{}", "{Line}", -1)
	return fasttemplate.New(v, "{", "}")
}
<commit_msg>version<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/alexflint/go-arg"
	"github.com/brentp/gargs/process"
	"github.com/fatih/color"
	isatty "github.com/mattn/go-isatty"
	"github.com/valyala/fasttemplate"
)

// Version is the current version
const Version = "0.3.5"

// ExitCode is the highest exit code seen in any command
var ExitCode = 0

// Params are the user-specified command-line arguments
type Params struct {
	Procs       int      `arg:"-p,help:number of processes to use."`
	Nlines      int      `arg:"-n,help:number of lines to consume for each command. -s and -n are mutually exclusive."`
	Retry       int      `arg:"-r,help:number of times to retry a command if it fails (default is 0)."`
	Sep         string   `arg:"-s,help:regular expression split line with to fill multiple template spots default is not to split. -s and -n are mutually exclusive."`
	Verbose     bool     `arg:"-v,help:print commands to stderr as they are executed."`
	StopOnError bool     `arg:"-s,--stop-on-error,help:stop execution on any error. default is to report errors and continue execution."`
	DryRun      bool     `arg:"-d,--dry-run,help:print (but do not run) the commands."`
	Log         string   `arg:"-l,--log,help:file to log commands. Successful commands are prefixed with '#'."`
	Command     string   `arg:"positional,required,help:command to execute."`
	log         *os.File `arg:"-"`
}

// isStdin checks if we are getting data from stdin.
func isStdin() bool {
	// http://stackoverflow.com/a/26567513
	stat, err := os.Stdin.Stat()
	if err != nil {
		return false
	}
	return (stat.Mode() & os.ModeCharDevice) == 0
}

func main() {
	args := Params{Procs: 1, Nlines: 1}
	p := arg.MustParse(&args)
	if args.Sep != "" && args.Nlines > 1 {
		p.Fail("must specify either sep (-s) or n-lines (-n), not both")
	}
	// if neither is specified then we default to whitespace
	if args.Nlines == 1 && args.Sep == "" {
		args.Sep = "\\s+"
	}
	if !isStdin() {
		fmt.Fprintln(os.Stderr, color.RedString("ERROR: expecting input on STDIN"))
		os.Exit(255)
	}
	if args.Log != "" {
		var err error
		args.log, err = os.Create(args.Log)
		check(err)
	}
	runtime.GOMAXPROCS(args.Procs)
	run(args)
	os.Exit(ExitCode)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func handleCommand(args *Params, cmd string, ch chan string) {
	if args.DryRun {
		fmt.Fprintf(os.Stdout, "%s\n", cmd)
		return
	}
	ch <- cmd
}

func fillTmplMap(toks []string, line string) map[string]interface{} {
	m := make(map[string]interface{}, 5)
	if toks != nil {
		for i, t := range toks {
			m[strconv.FormatInt(int64(i), 10)] = t
		}
	}
	m["Line"] = line
	return m
}

func getScanner() *bufio.Scanner {
	scanner := bufio.NewScanner(os.Stdin)
	scanner.Buffer(make([]byte, 0, 16384), 5e9)
	//if rs := os.Getenv("RS"); rs != "" && rs != "\n" && rs != "\r\n" {
	if rs := os.Getenv("RS"); rs != "" {
		brs := []byte(rs)
		scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
			if atEOF && len(data) == 0 {
				return 0, nil, nil
			}
			if i := bytes.Index(data, brs); i >= 0 {
				// Note that by adding the length here, we include the delimiter.
				return i + 1, data[:i+len(brs)], nil
			}
			if atEOF {
				return len(data), data, nil
			}
			return 0, nil, nil
		})
	}
	return scanner
}

func genCommands(args *Params, tmpl *fasttemplate.Template) <-chan string {
	ch := make(chan string)
	var resep *regexp.Regexp
	if args.Sep != "" {
		resep = regexp.MustCompile(args.Sep)
	}

	scanner := getScanner()
	fs := os.Getenv("FS")
	if fs == "" {
		fs = " "
	}

	go func() {
		var lines []string
		if resep == nil {
			lines = make([]string, 0, args.Nlines)
		}
		var buf bytes.Buffer
		for scanner.Scan() {
			buf.Reset()
			line := scanner.Text()
			serr := scanner.Err()
			if serr == nil || (serr == io.EOF && len(line) > 0) {
				// TODO: make dropping bytes optional.
				if resep != nil {
					toks := resep.Split(line, -1)
					targs := fillTmplMap(toks, line)
					_, err := tmpl.Execute(&buf, targs)
					check(err)
					handleCommand(args, buf.String(), ch)
				} else {
					lines = append(lines, line)
				}
			} else {
				if serr == io.EOF {
					break
				}
				log.Fatal(serr)
			}
			if len(lines) >= args.Nlines {
				targs := fillTmplMap(lines, strings.Join(lines, fs))
				_, err := tmpl.Execute(&buf, targs)
				check(err)
				lines = lines[:0]
				handleCommand(args, buf.String(), ch)
			}
		}
		if len(lines) > 0 {
			targs := fillTmplMap(lines, strings.Join(lines, fs))
			_, err := tmpl.Execute(&buf, targs)
			check(err)
			handleCommand(args, buf.String(), ch)
		}
		close(ch)
	}()
	return ch
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func init() {
	color.NoColor = !isatty.IsTerminal(os.Stderr.Fd())
}

func run(args Params) {

	tmpl := makeCommandTmpl(args.Command)
	cmds := genCommands(&args, tmpl)

	stdout := bufio.NewWriter(os.Stdout)
	defer stdout.Flush()

	cancel := make(chan bool)
	defer close(cancel)
	fails := 0

	// flush stdout every 2 seconds.
	last := time.Now().Add(2 * time.Second)
	for p := range process.Runner(cmds, args.Retry, cancel) {
		if ex := p.ExitCode(); ex != 0 {
			c := color.New(color.BgRed).Add(color.Bold)
			fmt.Fprintf(os.Stderr, "%s\n", c.SprintFunc()(fmt.Sprintf("ERROR with command: %s", p)))
			ExitCode = max(ExitCode, ex)
			fails++
			if args.StopOnError {
				break
			}
		}
		if args.Verbose {
			fmt.Fprintf(os.Stderr, "%s\n", p)
		}
		_, err := io.Copy(stdout, p)
		check(err)

		p.Cleanup()
		if t := time.Now(); t.After(last) {
			stdout.Flush()
			last = t.Add(2 * time.Second)
		}
		if args.log != nil {
			// if no error prefix the command with '#'
			if p.ExitCode() == 0 {
				args.log.WriteString("# " + strings.Replace(p.CmdStr, "\n", "\n# ", -1) + "\n")
			} else {
				args.log.WriteString(p.CmdStr + "\n")
			}
			stdout.Flush()
		}
	}
	stdout.Flush()
	if ExitCode == 0 && args.log != nil {
		args.log.WriteString("# SUCCESS\n")
	} else if args.log != nil {
		fmt.Fprintf(args.log, "# FAILED %d commands\n", fails)
	}

}

func makeCommandTmpl(cmd string) *fasttemplate.Template {
	v := strings.Replace(cmd, "{}", "{Line}", -1)
	return fasttemplate.New(v, "{", "}")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/neurodrone/aws-sqs/sqs"
	"log"
	"os"
)

var (
	awsAccessKey = flag.String("accesskey", "", "AWS Access Key")
	awsSecret    = flag.String("secret", "", "AWS Secret Key")

	regionId  = flag.String("region", "", "AWS Region ID")
	uuid      = flag.String("uuid", "", "AWS Unique ID")
	queueName = flag.String("queue", "", "AWS Queue Name")
)

func main() {
	flag.Parse()

	e := validateInputs()
	if e.hasErrors() {
		e.printErrors(os.Stderr)
		log.Fatalf("Aborting.")
	}

	sqsReq := &sqs.SQSRequest{
		*regionId,
		*uuid,
		*queueName,
		*awsAccessKey,
		*awsSecret,
	}

	/*
		message := "Message"
		_, err := sqsReq.SendSQSMessage(message)
		if err != nil {
			log.Fatalf("Unable to enqueue message: %s", err)
		}

		log.Println("Message sent.")
	*/
	msgResp, err := sqsReq.ReceiveSQSMessage()
	if err != nil {
		log.Fatalf("Unable to receive message: %s", err)
	}

	log.Println(msgResp.MessageId, "received.")
	log.Println(msgResp.MessageBody)

	_, err = sqsReq.DeleteSQSMessage(msgResp.ReceiptHandle)
	if err != nil {
		log.Fatalf("Unable to delete message: %s", msgResp.MessageId)
	}

	log.Println("Successfully received and deleted.")
}

func validateInputs() Errors {
	errs := make(Errors, 0)

	flag.VisitAll(func(fl *flag.Flag) {
		if fl.Value.String() == fl.DefValue {
			errs = append(errs, fmt.Errorf("%s needs to be set.", fl.Usage))
		}
	})

	return errs
}
<commit_msg>Add gobs to the main stub to ensure the transcoding is handled fine for it<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"flag"
	"fmt"
	"github.com/neurodrone/aws-sqs/sqs"
	"log"
	"os"
)

var (
	awsAccessKey = flag.String("accesskey", "", "AWS Access Key")
	awsSecret    = flag.String("secret", "", "AWS Secret Key")

	regionId  = flag.String("region", "", "AWS Region ID")
	uuid      = flag.String("uuid", "", "AWS Unique ID")
	queueName = flag.String("queue", "", "AWS Queue Name")
)

type SampleMessageStruct struct {
	SomeStr string
	SomeInt int
}

func main() {
	flag.Parse()

	e := validateInputs()
	if e.hasErrors() {
		e.printErrors(os.Stderr)
		log.Fatalf("Aborting.")
	}

	sqsReq := &sqs.SQSRequest{
		*regionId,
		*uuid,
		*queueName,
		*awsAccessKey,
		*awsSecret,
	}

	var buf bytes.Buffer
	var message string
	var m *SampleMessageStruct

	m = &SampleMessageStruct{"strVal", 7}
	gob.NewEncoder(&buf).Encode(m)

	message = buf.String()
	_, err := sqsReq.SendSQSMessage(message)
	if err != nil {
		log.Fatalf("Unable to enqueue message: %s", err)
	}
	log.Println("Message sent.")

	msgResp, err := sqsReq.ReceiveSQSMessage()
	if err != nil {
		log.Fatalf("Unable to receive message: %s", err)
	}

	log.Println(msgResp.MessageId, "received.")
	message = msgResp.MessageBody

	m = new(SampleMessageStruct)
	gob.NewDecoder(bytes.NewBufferString(message)).Decode(m)
	log.Println(m)

	_, err = sqsReq.DeleteSQSMessage(msgResp.ReceiptHandle)
	if err != nil {
		log.Fatalf("Unable to delete message: %s", msgResp.MessageId)
	}

	log.Println("Successfully received and deleted.")
}

func validateInputs() Errors {
	errs := make(Errors, 0)

	flag.VisitAll(func(fl *flag.Flag) {
		if fl.Value.String() == fl.DefValue {
			errs = append(errs, fmt.Errorf("%s needs to be set.", fl.Usage))
		}
	})

	return errs
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Service struct {
	Name    string
	Address string
	Port    int
}

func (s Service) AddressAndPort() string {
	return fmt.Sprintf("%s:%d", s.Address, s.Port)
}

func main() {
	services := loadServicesFromEnv()
	log.Printf("Services: %#v", services)

	var wg sync.WaitGroup
	cancel := make(chan struct{})

	for _, service := range services {
		wg.Add(1)
		go func(service Service) {
			waitForTcpConn(service, cancel)
			wg.Done()
		}(service)
	}

	timer := time.AfterFunc(10*time.Second, func() {
		close(cancel)
	})

	wg.Wait()

	// There's a race here that might result in assuming that a timeout happend
	// although none happend. It appears when the timer fires after the connection
	// succeeded, but before the check via Stop() below.
	// That shouldn't happen very often and the service was pretty short of timing out
	// anyway, so I guess that's ok for now.
	if !timer.Stop() {
		log.Printf("Error: One or more services timed out")
		os.Exit(1)
	}
	log.Printf("All services are up!")
}

func loadServicesFromEnv() []Service {
	services := make([]Service, 0)
	for _, line := range os.Environ() {
		keyAndValue := strings.SplitN(line, "=", 2)
		addrKey := keyAndValue[0]
		if strings.HasSuffix(addrKey, "_TCP_ADDR") {
			addr := os.Getenv(addrKey)
			name := addrKey[:len(addrKey)-9] // cut off "_TCP_ADDR"

			portKey := name + "_TCP_PORT"
			portStr := os.Getenv(portKey)
			port, err := strconv.Atoi(portStr)
			if err != nil {
				log.Printf("Failed to convert '%v' to int, value: '%v' - skipping service '%v'",
					portKey, portStr, name)
				continue
			}
			services = append(services, Service{Name: name, Address: addr, Port: port})
		}
	}
	return services
}

func waitForTcpConn(service Service, cancel <-chan struct{}) {
	var cancelled int32 = 0
	go func() {
		<-cancel
		atomic.StoreInt32(&cancelled, 1)
	}()

	var conn net.Conn
	err := errors.New("init")
	for err != nil {
		conn, err = net.DialTimeout("tcp", service.AddressAndPort(), 1*time.Second)

		if cancelled == 1 && err != nil {
			log.Printf("Service %v (%v) timed out. Last error: %v",
				service.Name, service.AddressAndPort(), err)
			return
		}
	}
	conn.Close()
	log.Printf("Service %v (%v) is up", service.Name, service.AddressAndPort())
}
<commit_msg>Make timeout configurable<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Service struct {
	Name    string
	Address string
	Port    int
}

func (s Service) AddressAndPort() string {
	return fmt.Sprintf("%s:%d", s.Address, s.Port)
}

var timeout = flag.Int64("timeout", 60, "time to wait for all services to be up (seconds)")

func main() {
	flag.Parse()

	services := loadServicesFromEnv()
	log.Printf("Services: %#v", services)

	var wg sync.WaitGroup
	cancel := make(chan struct{})

	for _, service := range services {
		wg.Add(1)
		go func(service Service) {
			waitForTcpConn(service, cancel)
			wg.Done()
		}(service)
	}

	timer := time.AfterFunc(time.Duration(*timeout)*time.Second, func() {
		close(cancel)
	})

	wg.Wait()

	// There's a race here that might result in assuming that a timeout happend
	// although none happend. It appears when the timer fires after the connection
	// succeeded, but before the check via Stop() below.
	// That shouldn't happen very often and the service was pretty short of timing out
	// anyway, so I guess that's ok for now.
	if !timer.Stop() {
		log.Printf("Error: One or more services timed out after %d second(s)", *timeout)
		os.Exit(1)
	}
	log.Printf("All services are up!")
}

func loadServicesFromEnv() []Service {
	services := make([]Service, 0)
	for _, line := range os.Environ() {
		keyAndValue := strings.SplitN(line, "=", 2)
		addrKey := keyAndValue[0]
		if strings.HasSuffix(addrKey, "_TCP_ADDR") {
			addr := os.Getenv(addrKey)
			name := addrKey[:len(addrKey)-9] // cut off "_TCP_ADDR"

			portKey := name + "_TCP_PORT"
			portStr := os.Getenv(portKey)
			port, err := strconv.Atoi(portStr)
			if err != nil {
				log.Printf("Failed to convert '%v' to int, value: '%v' - skipping service '%v'",
					portKey, portStr, name)
				continue
			}
			services = append(services, Service{Name: name, Address: addr, Port: port})
		}
	}
	return services
}

func waitForTcpConn(service Service, cancel <-chan struct{}) {
	var cancelled int32 = 0
	go func() {
		<-cancel
		atomic.StoreInt32(&cancelled, 1)
	}()

	var conn net.Conn
	err := errors.New("init")
	for err != nil {
		conn, err = net.DialTimeout("tcp", service.AddressAndPort(), 1*time.Second)

		if cancelled == 1 && err != nil {
			log.Printf("Service %v (%v) timed out. Last error: %v",
				service.Name, service.AddressAndPort(), err)
			return
		}
	}
	conn.Close()
	log.Printf("Service %v (%v) is up", service.Name, service.AddressAndPort())
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"sort"
	"time"

	"github.com/gorilla/feeds"
	rss "github.com/jteeuwen/go-pkg-rss"
)

var podcastEpisodePrefix = regexp.MustCompile(`^\d+: `)

var sourceFeeds = []sourceFeed{
	{uri: "https://robots.thoughtbot.com/summaries.xml", name: "Giant Robots blog"},
	{uri: "http://simplecast.fm/podcasts/271/rss", name: "Giant Robots podcast"},
	{uri: "http://simplecast.fm/podcasts/272/rss", name: "Build Phase podcast"},
	{uri: "http://simplecast.fm/podcasts/282/rss", name: "The Bike Shed podcast"},
	{uri: "http://simplecast.fm/podcasts/1088/rss", name: "Tentative podcast"},
	{uri: "https://simplecast.com/podcasts/2025/rss", name: "The Laila and Brenda Show podcast"},
	{uri: "https://upcase.com/the-weekly-iteration.rss", name: "The Weekly Iteration videos"},
}

func main() {
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()
	http.Handle("/", rssHandler(sourceFeeds))
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}

func rssHandler(sourceFeeds []sourceFeed) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.Referer() == "http://rss.thoughtbot.com" {
			http.Redirect(
				w,
				req,
				"https://rss.thoughtbot.com/",
				http.StatusMovedPermanently,
			)
			return
		}

		master := &feeds.Feed{
			Title:       "thoughtbot",
			Link:        &feeds.Link{Href: "https://rss.thoughtbot.com"},
			Description: "All the thoughts fit to bot.",
			Author:      &feeds.Author{Name: "thoughtbot", Email: "hello@thoughtbot.com"},
			Created:     time.Now(),
		}

		for _, feed := range sourceFeeds {
			fetch(feed, master)
		}

		sort.Sort(byCreated(master.Items))

		result, err := master.ToAtom()
		if err != nil {
			log.Printf("error generating feed: %v", err)
			http.Error(w, "error generating feed", http.StatusInternalServerError)
			return
		}

		_, err = fmt.Fprintln(w, result)
		if err != nil {
			log.Printf("error printing feed: %v", err)
			http.Error(w, "error printing feed", http.StatusInternalServerError)
			return
		}
	})
}

func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	err := fetcher.FetchClient(feed.uri, client, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	// no need to do anything...
}

func makeHandler(master *feeds.Feed, sourceName string) rss.ItemHandlerFunc {
	return func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for i := 0; i < len(items); i++ {
			published, err := items[i].ParsedPubDate()
			if err != nil {
				log.Printf("error parsing publication date: %v", err)
				continue
			}

			weekAgo := time.Now().AddDate(0, 0, -7)

			if published.After(weekAgo) {
				item := &feeds.Item{
					Title:       stripPodcastEpisodePrefix(items[i].Title),
					Link:        &feeds.Link{Href: items[i].Links[0].Href},
					Description: items[i].Description,
					Author:      &feeds.Author{Name: sourceName},
					Created:     published,
				}
				master.Add(item)
			}
		}
	}
}

type byCreated []*feeds.Item

func (s byCreated) Len() int {
	return len(s)
}

func (s byCreated) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byCreated) Less(i, j int) bool {
	return s[j].Created.Before(s[i].Created)
}

func stripPodcastEpisodePrefix(s string) string {
	return podcastEpisodePrefix.ReplaceAllString(s, "")
}

type sourceFeed struct {
	uri  string
	name string
}
<commit_msg>Use more generic technique to force SSL redirect<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"sort"
	"time"

	"github.com/gorilla/feeds"
	rss "github.com/jteeuwen/go-pkg-rss"
)

var podcastEpisodePrefix = regexp.MustCompile(`^\d+: `)

var sourceFeeds = []sourceFeed{
	{uri: "https://robots.thoughtbot.com/summaries.xml", name: "Giant Robots blog"},
	{uri: "http://simplecast.fm/podcasts/271/rss", name: "Giant Robots podcast"},
	{uri: "http://simplecast.fm/podcasts/272/rss", name: "Build Phase podcast"},
	{uri: "http://simplecast.fm/podcasts/282/rss", name: "The Bike Shed podcast"},
	{uri: "http://simplecast.fm/podcasts/1088/rss", name: "Tentative podcast"},
	{uri: "https://simplecast.com/podcasts/2025/rss", name: "The Laila and Brenda Show podcast"},
	{uri: "https://upcase.com/the-weekly-iteration.rss", name: "The Weekly Iteration videos"},
}

func main() {
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()
	http.Handle("/", rssHandler(sourceFeeds))
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}

func rssHandler(sourceFeeds []sourceFeed) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.Header.Get("X-Forwarded-Proto") == "http" {
			destination := *req.URL
			destination.Host = req.Host
			destination.Scheme = "https"
			http.Redirect(w, req, destination.String(), http.StatusFound)
			return
		}

		master := &feeds.Feed{
			Title:       "thoughtbot",
			Link:        &feeds.Link{Href: "https://rss.thoughtbot.com"},
			Description: "All the thoughts fit to bot.",
			Author:      &feeds.Author{Name: "thoughtbot", Email: "hello@thoughtbot.com"},
			Created:     time.Now(),
		}

		for _, feed := range sourceFeeds {
			fetch(feed, master)
		}

		sort.Sort(byCreated(master.Items))

		result, err := master.ToAtom()
		if err != nil {
			log.Printf("error generating feed: %v", err)
			http.Error(w, "error generating feed", http.StatusInternalServerError)
			return
		}

		_, err = fmt.Fprintln(w, result)
		if err != nil {
			log.Printf("error printing feed: %v", err)
			http.Error(w, "error printing feed", http.StatusInternalServerError)
			return
		}
	})
}

func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	err := fetcher.FetchClient(feed.uri, client, nil)
	if err != nil {
		log.Fatal(err)
	}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	// no need to do anything...
}

func makeHandler(master *feeds.Feed, sourceName string) rss.ItemHandlerFunc {
	return func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for i := 0; i < len(items); i++ {
			published, err := items[i].ParsedPubDate()
			if err != nil {
				log.Printf("error parsing publication date: %v", err)
				continue
			}

			weekAgo := time.Now().AddDate(0, 0, -7)

			if published.After(weekAgo) {
				item := &feeds.Item{
					Title:       stripPodcastEpisodePrefix(items[i].Title),
					Link:        &feeds.Link{Href: items[i].Links[0].Href},
					Description: items[i].Description,
					Author:      &feeds.Author{Name: sourceName},
					Created:     published,
				}
				master.Add(item)
			}
		}
	}
}

type byCreated []*feeds.Item

func (s byCreated) Len() int {
	return len(s)
}

func (s byCreated) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byCreated) Less(i, j int) bool {
	return s[j].Created.Before(s[i].Created)
}

func stripPodcastEpisodePrefix(s string) string {
	return podcastEpisodePrefix.ReplaceAllString(s, "")
}

type sourceFeed struct {
	uri  string
	name string
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

/*
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
var bot *linebot.Client
var eggyoID = "ufa92a3a52f197e19bfddeb5ca0595e93"
var logNof = "open"

type GeoContent struct {
	LatLong string `json:"latLon"`
	Utm     string `json:"utm"`
	Mgrs    string `json:"mgrs"`
}

type ResultGeoLoc struct {
	Results GeoContent `json:"result"`
}

func getGeoLoc(body []byte) (*ResultGeoLoc, error) {
	var s = new(ResultGeoLoc)
	err := json.Unmarshal(body, &s)
	if err != nil {
		fmt.Println("whoops:", err)
	}
	return s, err
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
}*/

func main() {

	bot, err := linebot.New(
		os.Getenv("ChannelSecret"),
		os.Getenv("MID"),
	)
	if err != nil {
		log.Fatal(err)
	}
	// Setup HTTP Server for receiving requests from LINE platform
	http.HandleFunc("/callback", func(w http.ResponseWriter, req *http.Request) {
		events, err := bot.ParseRequest(req)
		if err != nil {
			if err == linebot.ErrInvalidSignature {
				w.WriteHeader(400)
			} else {
				w.WriteHeader(500)
			}
			return
		}
		for _, event := range events {
			if event.Type == linebot.EventTypeMessage {
				switch message := event.Message.(type) {
				case *linebot.TextMessage:
					if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.Text)).Do(); err != nil {
						log.Print(err)
					}

				case *linebot.ImageMessage:

					content, err := bot.GetMessageContent(message.ID).Do()
					if err != nil {
						return err
					}
					defer content.Content.Close()
					log.Printf("Got file: %s", content.ContentType)

				}

			}

		}
	})
	// This is just sample code.
	// For actual use, you must support HTTPS by using `ListenAndServeTLS`, a reverse proxy or something else.
	if err := http.ListenAndServe(":"+os.Getenv("PORT"), nil); err != nil {
		log.Fatal(err)
	}
}

/*
<<<<<<< HEAD
<<<<<<< HEAD
=======
}

func main() {
	getAllUser()
	// line bot
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}

	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
	//test

}
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
func callbackHandler(w http.ResponseWriter, r *http.Request) {

	received, err := bot.ParseRequest(r)
	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
	for _, result := range received.Results {
		content := result.Content()
		log.Println("-->", content)

		//Log detail receive content
		if content != nil {
			log.Println("RECEIVE Msg:", content.IsMessage, " OP:", content.IsOperation, " type:", content.ContentType, " from:", content.From, "to:", content.To, " ID:", content.ID)
		}
		// user add friend
		if content != nil && content.IsOperation && content.OpType == linebot.OpTypeAddedAsFriend {
			out := fmt.Sprintf("Bot แปลงพิกัด Eggyo\nวิธีใช้\nเพียงแค่กดแชร์ Location ที่ต้องการ ระบบจะทำการแปลง Location เป็นพิกัดระบบต่างๆ และหาความสูงจากระดับน้ำทะเลให้\n\nหรือจะพูดคุยกับ bot ก็ได้\nกด #help เพื่อดูวิธีใช้อื่นๆ \nติดต่อผู้พัฒนา LINE ID : eggyo")
			//result.RawContent.Params[0] is who send your bot friend added operation, otherwise you cannot get in content or operation content.
			_, err = bot.SendText([]string{content.From}, out)
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot has a new friend :"+content.From)
			}

			addNewUser(content.From)

			if err != nil {
				log.Println(err)
			}
		}

		if content != nil && content.IsMessage && content.ContentType == linebot.ContentTypeText {

			text, err := content.TextContent()
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot get msg:"+text.Text+"\nfrom :"+content.From)
			}
			// reply message
			var processedText = messageCheck(text.Text)
			_, err = bot.SendText([]string{content.From}, processedText)

			if err != nil {
				log.Println(err)
			}
		}
		if content != nil && content.ContentType == linebot.ContentTypeLocation {
			_, err = bot.SendText([]string{content.From}, "ระบบกำลังประมวลผล...")

			loc, err := content.LocationContent()

<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
=======
			// add eggyo geo test
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "/" + FloatToString(loc.Longitude))
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
			if err != nil {
				println(err.Error())
				return
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			log.Println(string(body))

			var elev = callGoogleElev(loc.Latitude, loc.Longitude)
			geo, err := getGeoLoc([]byte(body))
			_, err = bot.SendText([]string{content.From}, "LatLong :"+geo.Results.LatLong)
			_, err = bot.SendText([]string{content.From}, "Utm :"+geo.Results.Utm+"\n\nMgrs :"+geo.Results.Mgrs+"\n\nAltitude :"+elev)
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot get loc:"+geo.Results.Mgrs+"\nfrom :"+content.From)
			}
			if err != nil {
				log.Println(err)
			}
		}
	}
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
*/
<commit_msg>v.5.1.2<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

/*
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
var bot *linebot.Client
var eggyoID = "ufa92a3a52f197e19bfddeb5ca0595e93"
var logNof = "open"

type GeoContent struct {
	LatLong string `json:"latLon"`
	Utm     string `json:"utm"`
	Mgrs    string `json:"mgrs"`
}

type ResultGeoLoc struct {
	Results GeoContent `json:"result"`
}

func getGeoLoc(body []byte) (*ResultGeoLoc, error) {
	var s = new(ResultGeoLoc)
	err := json.Unmarshal(body, &s)
	if err != nil {
		fmt.Println("whoops:", err)
	}
	return s, err
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
}*/

func main() {

	bot, err := linebot.New(
		os.Getenv("ChannelSecret"),
		os.Getenv("MID"),
	)
	if err != nil {
		log.Fatal(err)
	}
	// Setup HTTP Server for receiving requests from LINE platform
	http.HandleFunc("/callback", func(w http.ResponseWriter, req *http.Request) {
		events, err := bot.ParseRequest(req)
		if err != nil {
			if err == linebot.ErrInvalidSignature {
				w.WriteHeader(400)
			} else {
				w.WriteHeader(500)
			}
			return
		}
		for _, event := range events {
			if event.Type == linebot.EventTypeMessage {
				switch message := event.Message.(type) {
				case *linebot.TextMessage:
					if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(message.Text)).Do(); err != nil {
						log.Print(err)
					}

				case *linebot.ImageMessage:

					content, err := bot.GetMessageContent(message.ID).Do()
					if err != nil {
						log.Print(err)
					}
					defer content.Content.Close()
					log.Printf("Got file: %s", content.ContentType)

				}

			}

		}
	})
	// This is just sample code.
	// For actual use, you must support HTTPS by using `ListenAndServeTLS`, a reverse proxy or something else.
	if err := http.ListenAndServe(":"+os.Getenv("PORT"), nil); err != nil {
		log.Fatal(err)
	}
}

/*
<<<<<<< HEAD
<<<<<<< HEAD
=======
}

func main() {
	getAllUser()
	// line bot
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}

	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
	//test

}
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
func callbackHandler(w http.ResponseWriter, r *http.Request) {

	received, err := bot.ParseRequest(r)
	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
	for _, result := range received.Results {
		content := result.Content()
		log.Println("-->", content)

		//Log detail receive content
		if content != nil {
			log.Println("RECEIVE Msg:", content.IsMessage, " OP:", content.IsOperation, " type:", content.ContentType, " from:", content.From, "to:", content.To, " ID:", content.ID)
		}
		// user add friend
		if content != nil && content.IsOperation && content.OpType == linebot.OpTypeAddedAsFriend {
			out := fmt.Sprintf("Bot แปลงพิกัด Eggyo\nวิธีใช้\nเพียงแค่กดแชร์ Location ที่ต้องการ ระบบจะทำการแปลง Location เป็นพิกัดระบบต่างๆ และหาความสูงจากระดับน้ำทะเลให้\n\nหรือจะพูดคุยกับ bot ก็ได้\nกด #help เพื่อดูวิธีใช้อื่นๆ \nติดต่อผู้พัฒนา LINE ID : eggyo")
			//result.RawContent.Params[0] is who send your bot friend added operation, otherwise you cannot get in content or operation content.
			_, err = bot.SendText([]string{content.From}, out)
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot has a new friend :"+content.From)
			}

			addNewUser(content.From)

			if err != nil {
				log.Println(err)
			}
		}

		if content != nil && content.IsMessage && content.ContentType == linebot.ContentTypeText {

			text, err := content.TextContent()
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot get msg:"+text.Text+"\nfrom :"+content.From)
			}
			// reply message
			var processedText = messageCheck(text.Text)
			_, err = bot.SendText([]string{content.From}, processedText)

			if err != nil {
				log.Println(err)
			}
		}
		if content != nil && content.ContentType == linebot.ContentTypeLocation {
			_, err = bot.SendText([]string{content.From}, "ระบบกำลังประมวลผล...")

			loc, err := content.LocationContent()

<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
=======
			// add eggyo geo test
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "/" + FloatToString(loc.Longitude))
>>>>>>> 09a61a620751c49e8b67d3244f5280b4b309e1ae
=======
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
			// add eggyo geo test//
			resp, err := http.Get("http://eggyo-geo-node.herokuapp.com/geo/" + FloatToString(loc.Latitude) + "," + FloatToString(loc.Longitude))
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
			if err != nil {
				println(err.Error())
				return
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)
			log.Println(string(body))

			var elev = callGoogleElev(loc.Latitude, loc.Longitude)
			geo, err := getGeoLoc([]byte(body))
			_, err = bot.SendText([]string{content.From}, "LatLong :"+geo.Results.LatLong)
			_, err = bot.SendText([]string{content.From}, "Utm :"+geo.Results.Utm+"\n\nMgrs :"+geo.Results.Mgrs+"\n\nAltitude :"+elev)
			if logNof == "open" {
				bot.SendText([]string{eggyoID}, "bot get loc:"+geo.Results.Mgrs+"\nfrom :"+content.From)
			}
			if err != nil {
				log.Println(err)
			}
		}
	}
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
=======
>>>>>>> 66cf34abf6b2ed7caa5c018b73c50380be01e401
*/
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/zerok/statusd/Godeps/_workspace/src/gopkg.in/v2/yaml"
)

const (
	DEFAULT_TIMEOUT = 30
	DEFAULT_DELAY   = 30
	STATUS_OFFLINE  = "offline"
	STATUS_ONLINE   = "online"
)

type ServerConfiguration struct {
	IsAliveUrl string `yaml:"isAliveUrl"`
	Timeout    int    `yaml:"timeout"`
	Delay      int    `yaml:"delay"`
}

type HttpConfiguration struct {
	HostAddr string `yaml:"addr"`
}

type SlackConfiguration struct {
	Token            string              `yaml:"token"`
	Team             string              `yaml:"team"`
	NotifiedChannels map[string][]string `yaml:"channels"`
}

type Configuration struct {
	Servers map[string]ServerConfiguration `yaml:"servers"`
	Slack   SlackConfiguration             `yaml:"slack"`
	Http    HttpConfiguration              `yaml:"http"`
}

var statusRegistryManager StatusRegistryManager = NewStatusRegistryManager()

// NewConfiguration parses YAML data provided through a Reader
// into our configuration object. If any error occurs, no
// Configuration will be returned and an error is generated.
func NewConfiguration(r io.Reader) (*Configuration, error) {
	result := &Configuration{}
	rawData, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal(rawData, result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// NewConfigurationFromFile creates a new Configuration struct from
// the file behind the given path.
func NewConfigurationFromFile(filepath string) (*Configuration, error) {
	file, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return NewConfiguration(file)
}

// The StatusHandler updates the global server status mapping and triggers notifications
// if a server's status has changed.
func StatusHandler(config Configuration, statusUpdateChannel <-chan StatusUpdate, exitChannel chan struct{}, doneGroup *sync.WaitGroup) {
	notificationChannel := make(chan StatusUpdate, 10)
	notificationDoneGroup := sync.WaitGroup{}
	var notifySlack bool = (len(config.Slack.NotifiedChannels) != 0)
	if notifySlack {
		notificationDoneGroup.Add(1)
		go SlackNotifier(config, notificationChannel, &notificationDoneGroup)
	}
loop:
	for {
		select {
		case status := <-statusUpdateChannel:
			previousStatus := statusRegistryManager.GetStatus(status.ServerName)
			statusRegistryManager.SetStatus(status)
			log.Println(status)
			if notifySlack {
				// If this was the first time the server got a status, don't send out a notification to avoid
				// noise during restarts.
				if previousStatus != "" {
					notificationChannel <- status
				} else {
					log.Println("Skipping first status from entering the notification chain")
				}
			}
			break
		case <-exitChannel:
			break loop
		}
	}
	close(notificationChannel)
	log.Println("Waiting for notification handlers to shut down.")
	notificationDoneGroup.Wait()
	doneGroup.Done()
}

// ServerHandler is responsible for checking a single server periodically and
// reporting any status changes through the statusUpdateChannel.
func ServerHandler(serverName string, serverConfig ServerConfiguration, statusUpdateChannel chan<- StatusUpdate, exitChannel chan struct{}, doneGroup *sync.WaitGroup) {
	client := http.Client{}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return fmt.Errorf("Received a redirection as response")
	}
	previousStatus := ""
	newStatus := ""
	timeout := serverConfig.Timeout
	delay := serverConfig.Delay
	if timeout == 0 {
		timeout = DEFAULT_TIMEOUT
	}
	if delay == 0 {
		delay = DEFAULT_DELAY
	}
	finalTimeout, err := time.ParseDuration(fmt.Sprintf("%ds", timeout))
	if err != nil {
		finalTimeout = DEFAULT_TIMEOUT * time.Second
	}
	finalDelay, err := time.ParseDuration(fmt.Sprintf("%ds", delay))
	if err != nil {
		finalDelay = DEFAULT_DELAY * time.Second
	}
	client.Timeout = finalTimeout
	log.Printf("Processing server %v with a timeout of %vs\n", serverName, finalTimeout.Seconds())
	var nextPlannedCheck time.Time
loop:
	for {
		select {
		case <-exitChannel:
			log.Printf("Handler for %s received exit signal\n", serverName)
			break loop
		default:
		}

		if time.Now().Before(nextPlannedCheck) {
			time.Sleep(time.Second)
			continue
		}

		log.Printf("Checking %s\n", serverName)
		startTime := time.Now()
		resp, err := client.Get(serverConfig.IsAliveUrl)
		duration := time.Now().Sub(startTime)
		if err != nil {
			newStatus = STATUS_OFFLINE
		} else {
			if resp.StatusCode != 200 {
				newStatus = STATUS_OFFLINE

			} else {
				newStatus = STATUS_ONLINE
			}
			resp.Body.Close()
		}
		if newStatus != previousStatus {
			statusUpdateChannel <- StatusUpdate{ServerName: serverName, Status: newStatus, Duration: duration}
		}
		previousStatus = newStatus

		// Check the server periodically
		if newStatus == STATUS_OFFLINE {
			nextPlannedCheck = time.Now().Add(finalDelay * 2)
		} else {
			nextPlannedCheck = time.Now().Add(finalDelay)
		}
	}
	log.Printf("Shutting down %s worker\n", serverName)
	doneGroup.Done()
}

func main() {
	var configPath string
	// First we have to determine what servers should be checked and how. For that we
	// parse our configuration file.
	flag.StringVar(&configPath, "config", "", "Path to a configuration file")
	flag.Parse()
	if configPath == "" {
		log.Fatalln("Please specify a configuration file using the -config flag")
	}
	config, err := NewConfigurationFromFile(configPath)
	if err != nil {
		log.Fatalln(err)
	}

	if config.Servers == nil || len(config.Servers) == 0 {
		log.Fatalln("No servers configured")
	}

	doneGroup := sync.WaitGroup{}
	exitChannel := make(chan struct{}, len(config.Servers))
	statusUpdateChannel := make(chan StatusUpdate, len(config.Servers))
	signalChannel := make(chan os.Signal)

	doneGroup.Add(1)
	go StatusHandler(*config, statusUpdateChannel, exitChannel, &doneGroup)

	// For every server we create a seperate go-routine that checks the server periodically
	for serverName, serverConfig := range config.Servers {
		doneGroup.Add(1)
		go ServerHandler(serverName, serverConfig, statusUpdateChannel, exitChannel, &doneGroup)
	}

	if config.Http.HostAddr != "" {
		// Can't add a waitgroup handler for the HTTP server just yet. Perhaps in Go 1.4 ;)
		go HttpHandler(config.Http.HostAddr, &doneGroup)
	} else {
		log.Println("No HTTP configuration present. Not starting HTTP server.")
	}

	go func() {
		for {
			sign := <-signalChannel
			statusRegistryManager.ShowDown()
			log.Printf("Received %v. Shutting down workers", sign)
			for _ = range config.Servers {
				exitChannel <- struct{}{}
			}
			// An additional notification for the status handler and the web interface
			exitChannel <- struct{}{}
			exitChannel <- struct{}{}
		}
	}()

	signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)

	log.Println("Waiting for threads to exit.")
	doneGroup.Wait()
	close(exitChannel)
	close(statusUpdateChannel)
}
<commit_msg>Make things work for broken TLS certs<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/zerok/statusd/Godeps/_workspace/src/gopkg.in/v2/yaml"
)

const (
	DEFAULT_TIMEOUT = 30
	DEFAULT_DELAY   = 30
	STATUS_OFFLINE  = "offline"
	STATUS_ONLINE   = "online"
)

type ServerConfiguration struct {
	IsAliveUrl string `yaml:"isAliveUrl"`
	Timeout    int    `yaml:"timeout"`
	Delay      int    `yaml:"delay"`
}

type HttpConfiguration struct {
	HostAddr string `yaml:"addr"`
}

type SlackConfiguration struct {
	Token            string              `yaml:"token"`
	Team             string              `yaml:"team"`
	NotifiedChannels map[string][]string `yaml:"channels"`
}

type Configuration struct {
	Servers map[string]ServerConfiguration `yaml:"servers"`
	Slack   SlackConfiguration             `yaml:"slack"`
	Http    HttpConfiguration              `yaml:"http"`
}

var statusRegistryManager StatusRegistryManager = NewStatusRegistryManager()

// NewConfiguration parses YAML data provided through a Reader
// into our configuration object. If any error occurs, no
// Configuration will be returned and an error is generated.
func NewConfiguration(r io.Reader) (*Configuration, error) {
	result := &Configuration{}
	rawData, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal(rawData, result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// NewConfigurationFromFile creates a new Configuration struct from
// the file behind the given path.
func NewConfigurationFromFile(filepath string) (*Configuration, error) {
	file, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return NewConfiguration(file)
}

// The StatusHandler updates the global server status mapping and triggers notifications
// if a server's status has changed.
func StatusHandler(config Configuration, statusUpdateChannel <-chan StatusUpdate, exitChannel chan struct{}, doneGroup *sync.WaitGroup) {
	notificationChannel := make(chan StatusUpdate, 10)
	notificationDoneGroup := sync.WaitGroup{}
	var notifySlack bool = (len(config.Slack.NotifiedChannels) != 0)
	if notifySlack {
		notificationDoneGroup.Add(1)
		go SlackNotifier(config, notificationChannel, &notificationDoneGroup)
	}
loop:
	for {
		select {
		case status := <-statusUpdateChannel:
			previousStatus := statusRegistryManager.GetStatus(status.ServerName)
			statusRegistryManager.SetStatus(status)
			log.Println(status)
			if notifySlack {
				// If this was the first time the server got a status, don't send out a notification to avoid
				// noise during restarts.
				if previousStatus != "" {
					notificationChannel <- status
				} else {
					log.Println("Skipping first status from entering the notification chain")
				}
			}
			break
		case <-exitChannel:
			break loop
		}
	}
	close(notificationChannel)
	log.Println("Waiting for notification handlers to shut down.")
	notificationDoneGroup.Wait()
	doneGroup.Done()
}

// ServerHandler is responsible for checking a single server periodically and
// reporting any status changes through the statusUpdateChannel.
func ServerHandler(serverName string, serverConfig ServerConfiguration, statusUpdateChannel chan<- StatusUpdate, exitChannel chan struct{}, doneGroup *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := http.Client{Transport: tr}
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return fmt.Errorf("Received a redirection as response")
	}
	previousStatus := ""
	newStatus := ""
	timeout := serverConfig.Timeout
	delay := serverConfig.Delay
	if timeout == 0 {
		timeout = DEFAULT_TIMEOUT
	}
	if delay == 0 {
		delay = DEFAULT_DELAY
	}
	finalTimeout, err := time.ParseDuration(fmt.Sprintf("%ds", timeout))
	if err != nil {
		finalTimeout = DEFAULT_TIMEOUT * time.Second
	}
	finalDelay, err := time.ParseDuration(fmt.Sprintf("%ds", delay))
	if err != nil {
		finalDelay = DEFAULT_DELAY * time.Second
	}
	client.Timeout = finalTimeout
	log.Printf("Processing server %v with a timeout of %vs\n", serverName, finalTimeout.Seconds())
	var nextPlannedCheck time.Time
loop:
	for {
		select {
		case <-exitChannel:
			log.Printf("Handler for %s received exit signal\n", serverName)
			break loop
		default:
		}

		if time.Now().Before(nextPlannedCheck) {
			time.Sleep(time.Second)
			continue
		}

		startTime := time.Now()
		resp, err := client.Get(serverConfig.IsAliveUrl)
		duration := time.Now().Sub(startTime)
		if err != nil {
			log.Printf(err.Error())
			newStatus = STATUS_OFFLINE
		} else {
			if resp.StatusCode != 200 {
				log.Printf("Returned status %v", resp.StatusCode)
				newStatus = STATUS_OFFLINE

			} else {
				newStatus = STATUS_ONLINE
			}
			resp.Body.Close()
		}
		if newStatus == STATUS_ONLINE {
			log.Printf("%s is online\n", serverName)
		} else {
			log.Printf("%s is offline\n", serverName)
		}
		if newStatus != previousStatus {
			statusUpdateChannel <- StatusUpdate{ServerName: serverName, Status: newStatus, Duration: duration}
		}
		previousStatus = newStatus

		// Check the server periodically
		if newStatus == STATUS_OFFLINE {
			nextPlannedCheck = time.Now().Add(finalDelay * 2)
		} else {
			nextPlannedCheck = time.Now().Add(finalDelay)
		}
	}
	log.Printf("Shutting down %s worker\n", serverName)
	doneGroup.Done()
}

func main() {
	var configPath string
	// First we have to determine what servers should be checked and how. For that we
	// parse our configuration file.
	flag.StringVar(&configPath, "config", "", "Path to a configuration file")
	flag.Parse()
	if configPath == "" {
		log.Fatalln("Please specify a configuration file using the -config flag")
	}
	config, err := NewConfigurationFromFile(configPath)
	if err != nil {
		log.Fatalln(err)
	}

	if config.Servers == nil || len(config.Servers) == 0 {
		log.Fatalln("No servers configured")
	}

	doneGroup := sync.WaitGroup{}
	exitChannel := make(chan struct{}, len(config.Servers))
	statusUpdateChannel := make(chan StatusUpdate, len(config.Servers))
	signalChannel := make(chan os.Signal)

	doneGroup.Add(1)
	go StatusHandler(*config, statusUpdateChannel, exitChannel, &doneGroup)

	// For every server we create a seperate go-routine that checks the server periodically
	for serverName, serverConfig := range config.Servers {
		doneGroup.Add(1)
		go ServerHandler(serverName, serverConfig, statusUpdateChannel, exitChannel, &doneGroup)
	}

	if config.Http.HostAddr != "" {
		// Can't add a waitgroup handler for the HTTP server just yet. Perhaps in Go 1.4 ;)
		go HttpHandler(config.Http.HostAddr, &doneGroup)
	} else {
		log.Println("No HTTP configuration present. Not starting HTTP server.")
	}

	go func() {
		for {
			sign := <-signalChannel
			statusRegistryManager.ShowDown()
			log.Printf("Received %v. Shutting down workers", sign)
			for _ = range config.Servers {
				exitChannel <- struct{}{}
			}
			// An additional notification for the status handler and the web interface
			exitChannel <- struct{}{}
			exitChannel <- struct{}{}
		}
	}()

	signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)

	log.Println("Waiting for threads to exit.")
	doneGroup.Wait()
	close(exitChannel)
	close(statusUpdateChannel)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"

	"github.com/liyinan926/spark-operator/pkg/initializer"

	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	// Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters).
	// _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

func main() {
	kubeconfig := flag.String("kubeconfig", "", "Path to a kube config. Only required if out-of-cluster.")
	initializerThreads := flag.Int("initializer-threads", 10, "Number of worker threads in the initializer controller.")
	flag.Parse()

	glog.Info("Starting the Spark operator...")

	// Create the client config. Use kubeconfig if given, otherwise assume in-cluster.
	config, err := buildConfig(*kubeconfig)
	if err != nil {
		panic(err)
	}

	kubeClient, err := clientset.NewForConfig(config)
	if err != nil {
		panic(err)
	}

	stopCh := make(chan struct{})
	errCh := make(chan error)

	initializerController := initializer.NewController(kubeClient)
	go initializerController.Run(*initializerThreads, stopCh, errCh)

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
	<-signalCh

	glog.Info("Shutting down the Spark operator...")
	// This causes the custom controller and initializer to stop.
	close(stopCh)

	err = <-errCh
	if err != nil {
		glog.Errorf("Spark operator failed with error: %v", err)
	}
}

func buildConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}
<commit_msg>Enabled the client-go gcp auth plugin<commit_after>package main

import (
	"flag"
	"os"
	"os/signal"
	"syscall"

	"github.com/golang/glog"

	"github.com/liyinan926/spark-operator/pkg/initializer"

	clientset "k8s.io/client-go/kubernetes"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

func main() {
	kubeconfig := flag.String("kubeconfig", "", "Path to a kube config. Only required if out-of-cluster.")
	initializerThreads := flag.Int("initializer-threads", 10, "Number of worker threads in the initializer controller.")
	flag.Parse()

	glog.Info("Starting the Spark operator...")

	// Create the client config. Use kubeconfig if given, otherwise assume in-cluster.
	config, err := buildConfig(*kubeconfig)
	if err != nil {
		panic(err)
	}

	kubeClient, err := clientset.NewForConfig(config)
	if err != nil {
		panic(err)
	}

	stopCh := make(chan struct{})
	errCh := make(chan error)

	initializerController := initializer.NewController(kubeClient)
	go initializerController.Run(*initializerThreads, stopCh, errCh)

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
	<-signalCh

	glog.Info("Shutting down the Spark operator...")
	// This causes the custom controller and initializer to stop.
	close(stopCh)

	err = <-errCh
	if err != nil {
		glog.Errorf("Spark operator failed with error: %v", err)
	}
}

func buildConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ContainerLabs/terraform-provider-marathon/marathon"
	"github.com/hashicorp/terraform/plugin"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: marathon.Provider,
	})
}
<commit_msg>fix main.go deps<commit_after>package main

import (
	"github.com/hashicorp/terraform/plugin"
	"github.com/nicgrayson/terraform-provider-marathon/marathon"
)

func main() {
	plugin.Serve(&plugin.ServeOpts{
		ProviderFunc: marathon.Provider,
	})
}
<|endoftext|>