<commit_before>package main

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
	"os"

	"github.com/ederoyd46/osm/fileformat"
	"github.com/ederoyd46/osm/osmformat"

	"github.com/golang/protobuf/proto"
)

func main() {
	startImport("./download/hertfordshire-latest.osm.pbf")
}

func startImport(fileName string) {
	file, err := os.Open(fileName)
	LogError(err)

	getBlock(4, file)
	file.Close()
}

func getBlock(size int64, file *os.File) {

	headerSizeData := make([]byte, size)
	_, err := file.Read(headerSizeData)
	if err != nil {
		//Probably end of file
		return
	}

	var headerSize uint32
	err = binary.Read(bytes.NewBuffer(headerSizeData), binary.BigEndian, &headerSize)
	LogError(err)

	headerData := make([]byte, headerSize)
	file.Read(headerData)

	var header fileformat.BlockHeader
	proto.Unmarshal(headerData, &header)

	fmt.Println(header.GetType())

	blobData := make([]byte, header.GetDatasize())
	file.Read(blobData)

	var blob fileformat.Blob
	proto.Unmarshal(blobData, &blob)

	zr, err := zlib.NewReader(bytes.NewBuffer(blob.GetZlibData()))
	LogError(err)

	fmt.Println("Compressed Size", len(blob.GetZlibData()))
	fmt.Println("Raw Size:", blob.GetRawSize())

	var blobUncompressed = make([]byte, blob.GetRawSize())
	io.ReadFull(zr, blobUncompressed)
	LogError(err)
	zr.Close()

	if "OSMHeader" == header.GetType() {
		osmHeader(blobUncompressed)
		// getBlock(4, file)
	}

	if "OSMData" == header.GetType() {
		osmData(blobUncompressed)
	}

	getBlock(4, file)

}

func osmHeader(blobUncompressed []byte) {
	var headerBlock osmformat.HeaderBlock
	proto.Unmarshal(blobUncompressed, &headerBlock)

	minlat := float64(headerBlock.GetBbox().GetBottom()) / nano
	minlon := float64(headerBlock.GetBbox().GetLeft()) / nano
	maxlat := float64(headerBlock.GetBbox().GetTop()) / nano
	maxlon := float64(headerBlock.GetBbox().GetRight()) / nano

	fmt.Println(" Min Lat:", minlat)
	fmt.Println(" Min Lon:", minlon)
	fmt.Println(" Max Lat:", maxlat)
	fmt.Println(" Max Lon:", maxlon)

}

func osmData(blobUncompressed []byte) {
	var primitiveBlock osmformat.PrimitiveBlock
	proto.Unmarshal(blobUncompressed, &primitiveBlock)

	fmt.Println(" DateGranularity:", primitiveBlock.GetDateGranularity())
	fmt.Println(" LatOffset:", primitiveBlock.GetLatOffset())
	fmt.Println(" LonOffset:", primitiveBlock.GetLonOffset())
	fmt.Println(" Granularity:", primitiveBlock.GetGranularity())
	fmt.Println(" PrimitiveGroups:", len(primitiveBlock.GetPrimitivegroup()))

	var stringTable = ConvertStringTable(primitiveBlock.GetStringtable().GetS())
	fmt.Println(" StringTable:", len(stringTable))

	var primitiveGroup = primitiveBlock.GetPrimitivegroup()

	for _, group := range primitiveGroup {
		handlePrimitiveGroupData(group,
			stringTable,
			float64(primitiveBlock.GetGranularity()),
			int64(primitiveBlock.GetDateGranularity()),
		)
	}

}

func buildKeyVals(mixedKeyVals []int32, stringTable []string) []map[string]string {
	var keyvals []map[string]string
	keyvalEntry := make(map[string]string)

	count := len(mixedKeyVals)
	for i := 0; i < count; {
		if mixedKeyVals[i] == 0 {
			keyvals = append(keyvals, keyvalEntry)
			keyvalEntry = map[string]string{}
			i = i + 1
		} else {
			key := stringTable[mixedKeyVals[i]]
			val := stringTable[mixedKeyVals[i+1]]
			keyvalEntry[key] = val
			i = i + 2
		}
	}
	return keyvals
}

func handlePrimitiveGroupData(group *osmformat.PrimitiveGroup, stringTable []string, granularity float64, dateGranularity int64) {
	denseNodes := group.GetDense()
	size := len(denseNodes.GetId())
	uids := DeltaDecodeInt32(0, denseNodes.GetDenseinfo().GetUid())
	sids := DeltaDecodeInt32(0, denseNodes.GetDenseinfo().GetUserSid())
	timestamps := DeltaDecodeInt64(0, denseNodes.GetDenseinfo().GetTimestamp())
	changesets := DeltaDecodeInt64(0, denseNodes.GetDenseinfo().GetChangeset())
	latitudes := DeltaDecodeInt64ToFloat(0, denseNodes.GetLat())
	longitudes := DeltaDecodeInt64ToFloat(0, denseNodes.GetLon())
	keyvals := buildKeyVals(denseNodes.GetKeysVals(), stringTable)

	var nodes []Node
	for i := 0; i < size; i++ {
		node := Node{
			ID:        denseNodes.GetId()[i],
			Latitude:  CalculateDegrees(latitudes[i], granularity),
			Longitude: CalculateDegrees(longitudes[i], granularity),
			Timestamp: CalculateTime(timestamps[i], dateGranularity),
			Changeset: changesets[i],
			UID:       uids[i],
			SID:       stringTable[sids[i]],
			Tags:      keyvals[i]}
		nodes = append(nodes, node)
	}
	SaveNodes(nodes)
}
<commit_msg>updated imports<commit_after>package main

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
	"os"

	"github.com/ederoyd46/osm/fileformat"
	"github.com/ederoyd46/osm/osmformat"

	"github.com/golang/protobuf/proto"
)

func main() {
	startImport("./download/hertfordshire-latest.osm.pbf")
}

func startImport(fileName string) {
	file, err := os.Open(fileName)
	LogError(err)

	getBlock(4, file)
	file.Close()
}

func getBlock(size int64, file *os.File) {

	headerSizeData := make([]byte, size)
	_, err := file.Read(headerSizeData)
	if err != nil {
		//Probably end of file
		return
	}

	var headerSize uint32
	err = binary.Read(bytes.NewBuffer(headerSizeData), binary.BigEndian, &headerSize)
	LogError(err)

	headerData := make([]byte, headerSize)
	file.Read(headerData)

	var header fileformat.BlockHeader
	proto.Unmarshal(headerData, &header)

	fmt.Println(header.GetType())

	blobData := make([]byte, header.GetDatasize())
	file.Read(blobData)

	var blob fileformat.Blob
	proto.Unmarshal(blobData, &blob)

	zr, err := zlib.NewReader(bytes.NewBuffer(blob.GetZlibData()))
	LogError(err)

	fmt.Println("Compressed Size", len(blob.GetZlibData()))
	fmt.Println("Raw Size:", blob.GetRawSize())

	var blobUncompressed = make([]byte, blob.GetRawSize())
	io.ReadFull(zr, blobUncompressed)
	LogError(err)
	zr.Close()

	if "OSMHeader" == header.GetType() {
		osmHeader(blobUncompressed)
		// getBlock(4, file)
	}

	if "OSMData" == header.GetType() {
		osmData(blobUncompressed)
	}

	getBlock(4, file)

}

func osmHeader(blobUncompressed []byte) {
	var headerBlock osmformat.HeaderBlock

	proto.Unmarshal(blobUncompressed, &headerBlock)

	minlat := float64(headerBlock.GetBbox().GetBottom()) / nano
	minlon := float64(headerBlock.GetBbox().GetLeft()) / nano
	maxlat := float64(headerBlock.GetBbox().GetTop()) / nano
	maxlon := float64(headerBlock.GetBbox().GetRight()) / nano

	fmt.Println(" Min Lat:", minlat)
	fmt.Println(" Min Lon:", minlon)
	fmt.Println(" Max Lat:", maxlat)
	fmt.Println(" Max Lon:", maxlon)

}

func osmData(blobUncompressed []byte) {
	var primitiveBlock osmformat.PrimitiveBlock
	proto.Unmarshal(blobUncompressed, &primitiveBlock)

	fmt.Println(" DateGranularity:", primitiveBlock.GetDateGranularity())
	fmt.Println(" LatOffset:", primitiveBlock.GetLatOffset())
	fmt.Println(" LonOffset:", primitiveBlock.GetLonOffset())
	fmt.Println(" Granularity:", primitiveBlock.GetGranularity())
	fmt.Println(" PrimitiveGroups:", len(primitiveBlock.GetPrimitivegroup()))

	var stringTable = ConvertStringTable(primitiveBlock.GetStringtable().GetS())
	fmt.Println(" StringTable:", len(stringTable))

	var primitiveGroup = primitiveBlock.GetPrimitivegroup()

	for _, group := range primitiveGroup {
		handlePrimitiveGroupData(group,
			stringTable,
			float64(primitiveBlock.GetGranularity()),
			int64(primitiveBlock.GetDateGranularity()),
		)
	}

}

func buildKeyVals(mixedKeyVals []int32, stringTable []string) []map[string]string {
	var keyvals []map[string]string
	keyvalEntry := make(map[string]string)

	count := len(mixedKeyVals)
	for i := 0; i < count; {
		if mixedKeyVals[i] == 0 {
			keyvals = append(keyvals, keyvalEntry)
			keyvalEntry = map[string]string{}
			i = i + 1
		} else {
			key := stringTable[mixedKeyVals[i]]
			val := stringTable[mixedKeyVals[i+1]]
			keyvalEntry[key] = val
			i = i + 2
		}
	}
	return keyvals
}

func handlePrimitiveGroupData(group *osmformat.PrimitiveGroup, stringTable []string, granularity float64, dateGranularity int64) {
	denseNodes := group.GetDense()
	size := len(denseNodes.GetId())
	uids := DeltaDecodeInt32(0, denseNodes.GetDenseinfo().GetUid())
	sids := DeltaDecodeInt32(0, denseNodes.GetDenseinfo().GetUserSid())
	timestamps := DeltaDecodeInt64(0, denseNodes.GetDenseinfo().GetTimestamp())
	changesets := DeltaDecodeInt64(0, denseNodes.GetDenseinfo().GetChangeset())
	latitudes := DeltaDecodeInt64ToFloat(0, denseNodes.GetLat())
	longitudes := DeltaDecodeInt64ToFloat(0, denseNodes.GetLon())
	keyvals := buildKeyVals(denseNodes.GetKeysVals(), stringTable)

	var nodes []Node
	for i := 0; i < size; i++ {
		node := Node{
			ID:        denseNodes.GetId()[i],
			Latitude:  CalculateDegrees(latitudes[i], granularity),
			Longitude: CalculateDegrees(longitudes[i], granularity),
			Timestamp: CalculateTime(timestamps[i], dateGranularity),
			Changeset: changesets[i],
			UID:       uids[i],
			SID:       stringTable[sids[i]],
			Tags:      keyvals[i]}
		nodes = append(nodes, node)
	}
	SaveNodes(nodes)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
)

func Cwd() string {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return pwd
}

// make into command line arguments
var (
	in        = filepath.Join(Cwd(), "fixtures", "cms.json")
	out       = filepath.Join(Cwd(), "fixtures", "cms.downloaded.json")
	imagesDir = filepath.Join(Cwd(), "src", "images")
	rel       = "/images"
	needle    = "https://drive.google.com/file/d/"
)

func Run() error {
	var l Lister

	l.Assets = make(map[string]Asset)
	l.Needle = needle
	l.InputPath = in
	l.Data = struct{}{}

	// l := Lister{
	// 	make(map[string]Asset),
	// 	JsonLooper{needle, in, struct{}{}, Lister.SearchCell},
	// 	// needle, in, struct{}{},
	// }

	d := Downloader{
		imagesDir,
		rel,
		[]Res{},
	}

	// r := Renamer{
	// 	in,
	// 	out,
	// 	rel,
	// 	needle,
	// }

	if err := l.Run(); err != nil {
		return fmt.Errorf("%v", err)
	}

	if err := d.Run(l.Assets); err != nil {
		return fmt.Errorf("%q", err)
	}

	// if err := r.Run(d.Results, io.Writer{}); err != nil {
	// 	return fmt.Errorf("%q", err)
	// }

	fmt.Printf("%q", l.Assets)

	return nil
}
<commit_msg>use NewLister() factory to create new lister instance in main<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
)

func Cwd() string {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return pwd
}

// make into command line arguments
var (
	in        = filepath.Join(Cwd(), "fixtures", "cms.json")
	out       = filepath.Join(Cwd(), "fixtures", "cms.downloaded.json")
	imagesDir = filepath.Join(Cwd(), "src", "images")
	rel       = "/images"
	needle    = "https://drive.google.com/file/d/"
)

func Run() error {

	l := NewLister(needle, in)

	d := Downloader{
		imagesDir,
		rel,
		[]Res{},
	}

	// r := Renamer{
	// 	in,
	// 	out,
	// 	rel,
	// 	needle,
	// }

	if err := l.Run(); err != nil {
		return fmt.Errorf("%v", err)
	}

	if err := d.Run(l.Assets); err != nil {
		return fmt.Errorf("%q", err)
	}

	// if err := r.Run(d.Results, io.Writer{}); err != nil {
	// 	return fmt.Errorf("%q", err)
	// }

	fmt.Printf("%q", l.Assets)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

var (
	flagOut    = flag.String("o", "", "Output file, else stdout.")
	flagPkg    = flag.String("pkg", "main", "Package.")
	flagPrefix = flag.String("prefix", "", "Prefix to strip from filesnames.")
)

type file struct {
	data  []byte
	local string
}

func main() {
	flag.Parse()
	var err error
	var fnames []string
	content := make(map[string]file)
	prefix := filepath.ToSlash(*flagPrefix)
	for _, base := range flag.Args() {
		files := []string{base}
		for len(files) > 0 {
			fname := files[0]
			files = files[1:]
			f, err := os.Open(fname)
			if err != nil {
				log.Fatal(err)
			}
			fi, err := f.Stat()
			if err != nil {
				log.Fatal(err)
			}
			if fi.IsDir() {
				fis, err := f.Readdir(0)
				if err != nil {
					log.Fatal(err)
				}
				for _, fi := range fis {
					files = append(files, filepath.Join(fname, fi.Name()))
				}
			} else {
				b, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal(err)
				}
				fpath := filepath.ToSlash(fname)
				n := strings.TrimPrefix(fpath, prefix)
				n = path.Join("/", n)
				content[n] = file{data: b, local: fpath}
				fnames = append(fnames, n)
			}
			f.Close()
		}
	}
	sort.Strings(fnames)
	w := os.Stdout
	if *flagOut != "" {
		if w, err = os.Create(*flagOut); err != nil {
			log.Fatal(err)
		}
		defer w.Close()
	}
	fmt.Fprintf(w, header, *flagPkg)
	for _, fname := range fnames {
		f := content[fname]
		var buf bytes.Buffer
		gw := gzip.NewWriter(&buf)
		if _, err := gw.Write(f.data); err != nil {
			log.Fatal(err)
		}
		if err := gw.Close(); err != nil {
			log.Fatal(err)
		}
		fmt.Fprintf(w, "\n\t%q: {compressed: %q, size: %v, local: %q},\n", fname, buf.String(), len(f.data), f.local)
	}
	fmt.Fprint(w, footer)
}

const (
	header = `package %s

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type localFS struct{}

var local localFS

type staticFS struct{}

var static staticFS

type file struct {
	compressed string
	size       int64
	local      string

	data []byte
	once sync.Once
	name string
}

func (_ localFS) Open(name string) (http.File, error) {
	f, present := data[name]
	if !present {
		return nil, os.ErrNotExist
	}
	return os.Open(f.local)
}

func (_ staticFS) Open(name string) (http.File, error) {
	f, present := data[name]
	if !present {
		return nil, os.ErrNotExist
	}
	var err error
	f.once.Do(func() {
		var gr *gzip.Reader
		gr, err = gzip.NewReader(bytes.NewBufferString(f.compressed))
		if err != nil {
			return
		}
		f.data, err = ioutil.ReadAll(gr)
		f.name = path.Base(name)
	})
	return f.File()
}

func (f *file) File() (http.File, error) {
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		file:   f,
	}, nil
}

type httpFile struct {
	*bytes.Reader
	*file
}

func (f *file) Close() error {
	return nil
}

func (f *file) Readdir(count int) ([]os.FileInfo, error) {
	return nil, nil
}

func (f *file) Stat() (os.FileInfo, error) {
	return f, nil
}

func (f *file) Name() string {
	return f.name
}

func (f *file) Size() int64 {
	return f.size
}

func (f *file) Mode() os.FileMode {
	return 0
}

func (f *file) ModTime() time.Time {
	return time.Time{}
}
func (f *file) IsDir() bool {
	return false
}
func (f *file) Sys() interface{} {
	return f
}

// FS returns a http.Filesystem for the embedded assets. If useLocal is true,
// the filesystem's contents are instead used.
func FS(useLocal bool) http.FileSystem {
	if useLocal {
		return local
	}
	return static
}

var data = map[string]*file{
`
	footer = `}
`
)
<commit_msg>Complain if there was an error<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

var (
	flagOut    = flag.String("o", "", "Output file, else stdout.")
	flagPkg    = flag.String("pkg", "main", "Package.")
	flagPrefix = flag.String("prefix", "", "Prefix to strip from filesnames.")
)

type file struct {
	data  []byte
	local string
}

func main() {
	flag.Parse()
	var err error
	var fnames []string
	content := make(map[string]file)
	prefix := filepath.ToSlash(*flagPrefix)
	for _, base := range flag.Args() {
		files := []string{base}
		for len(files) > 0 {
			fname := files[0]
			files = files[1:]
			f, err := os.Open(fname)
			if err != nil {
				log.Fatal(err)
			}
			fi, err := f.Stat()
			if err != nil {
				log.Fatal(err)
			}
			if fi.IsDir() {
				fis, err := f.Readdir(0)
				if err != nil {
					log.Fatal(err)
				}
				for _, fi := range fis {
					files = append(files, filepath.Join(fname, fi.Name()))
				}
			} else {
				b, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal(err)
				}
				fpath := filepath.ToSlash(fname)
				n := strings.TrimPrefix(fpath, prefix)
				n = path.Join("/", n)
				content[n] = file{data: b, local: fpath}
				fnames = append(fnames, n)
			}
			f.Close()
		}
	}
	sort.Strings(fnames)
	w := os.Stdout
	if *flagOut != "" {
		if w, err = os.Create(*flagOut); err != nil {
			log.Fatal(err)
		}
		defer w.Close()
	}
	fmt.Fprintf(w, header, *flagPkg)
	for _, fname := range fnames {
		f := content[fname]
		var buf bytes.Buffer
		gw := gzip.NewWriter(&buf)
		if _, err := gw.Write(f.data); err != nil {
			log.Fatal(err)
		}
		if err := gw.Close(); err != nil {
			log.Fatal(err)
		}
		fmt.Fprintf(w, "\n\t%q: {compressed: %q, size: %v, local: %q},\n", fname, buf.String(), len(f.data), f.local)
	}
	fmt.Fprint(w, footer)
}

const (
	header = `package %s

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type localFS struct{}

var local localFS

type staticFS struct{}

var static staticFS

type file struct {
	compressed string
	size       int64
	local      string

	data []byte
	once sync.Once
	name string
}

func (_ localFS) Open(name string) (http.File, error) {
	f, present := data[name]
	if !present {
		return nil, os.ErrNotExist
	}
	return os.Open(f.local)
}

func (_ staticFS) Open(name string) (http.File, error) {
	f, present := data[name]
	if !present {
		return nil, os.ErrNotExist
	}
	var err error
	f.once.Do(func() {
		var gr *gzip.Reader
		gr, err = gzip.NewReader(bytes.NewBufferString(f.compressed))
		if err != nil {
			return
		}
		f.data, err = ioutil.ReadAll(gr)
		f.name = path.Base(name)
	})
	if err != nil {
		return nil, err
	}
	return f.File()
}

func (f *file) File() (http.File, error) {
	return &httpFile{
		Reader: bytes.NewReader(f.data),
		file:   f,
	}, nil
}

type httpFile struct {
	*bytes.Reader
	*file
}

func (f *file) Close() error {
	return nil
}

func (f *file) Readdir(count int) ([]os.FileInfo, error) {
	return nil, nil
}

func (f *file) Stat() (os.FileInfo, error) {
	return f, nil
}

func (f *file) Name() string {
	return f.name
}

func (f *file) Size() int64 {
	return f.size
}

func (f *file) Mode() os.FileMode {
	return 0
}

func (f *file) ModTime() time.Time {
	return time.Time{}
}
func (f *file) IsDir() bool {
	return false
}
func (f *file) Sys() interface{} {
	return f
}

// FS returns a http.Filesystem for the embedded assets. If useLocal is true,
// the filesystem's contents are instead used.
func FS(useLocal bool) http.FileSystem {
	if useLocal {
		return local
	}
	return static
}

var data = map[string]*file{
`
	footer = `}
`
)
<|endoftext|>


<commit_before>package main

import (
	// "flag"
	"os"
	"os/signal"
	"runtime"
	// "runtime/pprof"
	"sync"
	"syscall"
)

// var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var quit chan struct{}

func sigHandler() {
	// TODO On Windows, these signals will not be triggered on closing cmd
	// window. How to detect this?
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	for sig := range sigChan {
		// May handle other signals in the future.
		info.Printf("%v caught, exit\n", sig)
		storeSiteStat(siteStatExit)
		break
	}
	/*
		if *cpuprofile != "" {
			pprof.StopCPUProfile()
		}
	*/
	close(quit)
}

func main() {
	quit = make(chan struct{})
	// Parse flags after load config to allow override options in config
	cmdLineConfig := parseCmdLineConfig()
	if cmdLineConfig.PrintVer {
		printVersion()
		os.Exit(0)
	}

	parseConfig(cmdLineConfig.RcFile, cmdLineConfig)

	initSelfListenAddr()
	initLog()
	initAuth()
	initSiteStat()
	initPAC() // initPAC uses siteStat, so must init after site stat

	initStat()

	initParentPool()

	/*
		if *cpuprofile != "" {
			f, err := os.Create(*cpuprofile)
			if err != nil {
				Fatal(err)
			}
			pprof.StartCPUProfile(f)
		}
	*/

	if config.Core > 0 {
		runtime.GOMAXPROCS(config.Core)
	}

	go sigHandler()
	go runSSH()
	if config.EstimateTimeout {
		go runEstimateTimeout()
	} else {
		info.Println("timeout estimation disabled")
	}

	var wg sync.WaitGroup
	wg.Add(len(listenProxy))
	for _, proxy := range listenProxy {
		go proxy.Serve(&wg, quit)
	}
	wg.Wait()

	debug.Println("listners are done, exiting...")
}
<commit_msg>support relaunching cow on SIGUSR1 fixes #322<commit_after>package main

import (
	// "flag"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	// "runtime/pprof"
	"sync"
	"syscall"
)

// var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var (
	quit     chan struct{}
	relaunch bool
)

func sigHandler() {
	// TODO On Windows, these signals will not be triggered on closing cmd
	// window. How to detect this?
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)

	for sig := range sigChan {
		// May handle other signals in the future.
		info.Printf("%v caught, exit\n", sig)
		storeSiteStat(siteStatExit)
		if sig == syscall.SIGUSR1 {
			relaunch = true
		}
		close(quit)
		break
	}
	/*
		if *cpuprofile != "" {
			pprof.StopCPUProfile()
		}
	*/
}

// This code is from goagain
func lookPath() (argv0 string, err error) {
	argv0, err = exec.LookPath(os.Args[0])
	if nil != err {
		return
	}
	if _, err = os.Stat(argv0); nil != err {
		return
	}
	return
}

func main() {
	quit = make(chan struct{})
	// Parse flags after load config to allow override options in config
	cmdLineConfig := parseCmdLineConfig()
	if cmdLineConfig.PrintVer {
		printVersion()
		os.Exit(0)
	}

	parseConfig(cmdLineConfig.RcFile, cmdLineConfig)

	initSelfListenAddr()
	initLog()
	initAuth()
	initSiteStat()
	initPAC() // initPAC uses siteStat, so must init after site stat

	initStat()

	initParentPool()

	/*
		if *cpuprofile != "" {
			f, err := os.Create(*cpuprofile)
			if err != nil {
				Fatal(err)
			}
			pprof.StartCPUProfile(f)
		}
	*/

	if config.Core > 0 {
		runtime.GOMAXPROCS(config.Core)
	}

	go sigHandler()
	go runSSH()
	if config.EstimateTimeout {
		go runEstimateTimeout()
	} else {
		info.Println("timeout estimation disabled")
	}

	var wg sync.WaitGroup
	wg.Add(len(listenProxy))
	for _, proxy := range listenProxy {
		go proxy.Serve(&wg, quit)
	}

	wg.Wait()

	if relaunch {
		info.Println("Relunching cow...")
		// Need to fork me.
		argv0, err := lookPath()
		if nil != err {
			errl.Println(err)
			return
		}

		err = syscall.Exec(argv0, os.Args, os.Environ())
		if err != nil {
			errl.Println(err)
		}
	}
	debug.Println("the main process is , exiting...")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/tools/go/types"

	_ "golang.org/x/tools/go/gcimporter"
)

var (
	objectType = flag.String("type", "", "type to wrap in a reactive struct, required")
	name       = flag.String("name", "", "name of reactive type, required")
	output     = flag.String("output", "", "output file name; default srcdir/<type>_rx.go")
)

func main() {

	reg := regexp.MustCompile("[^A-Za-z0-9_]+")

	flag.Parse()

	if *objectType == "" {
		flag.Usage()
		os.Exit(2)
	}

	var (
		dir string
		g   Generator
	)

	args := flag.Args()
	if len(args) == 1 && isDirectory(args[0]) {
		dir = args[0]
		g.parsePackageDir(args[0])
	} else {
		dir = filepath.Dir(args[0])
		g.parsePackageFiles(args)
	}

	g.generateRx(*name, *objectType)

	src := g.format()

	outputName := *output
	if outputName == "" {
		// remove non alpha-numeric in filename
		o := reg.ReplaceAllString(*objectType, "")
		baseName := fmt.Sprintf("%s_rx.go", o)
		outputName = filepath.Join(dir, strings.ToLower(baseName))
	}
	err := ioutil.WriteFile(outputName, src, 0644)
	if err != nil {
		log.Fatalf("writing output: %s", err)
	}
}

func isDirectory(name string) bool {
	info, err := os.Stat(name)
	if err != nil {
		log.Fatal(err)
	}

	return info.IsDir()
}

type Generator struct {
	buf bytes.Buffer
	pkg *Package

	Params  *ast.FieldList
	Results *ast.FieldList
}

type File struct {
	pkg  *Package
	file *ast.File
}

type Package struct {
	dir      string
	name     string
	defs     map[*ast.Ident]types.Object
	files    []*File
	typesPkg *types.Package
}

// parsePackageDir parses the package residing in the directory.
func (g *Generator) parsePackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		log.Fatalf("cannot process directory %s: %s", directory, err)
	}
	var names []string
	names = append(names, pkg.GoFiles...)
	names = append(names, pkg.CgoFiles...)
	// TODO: Need to think about constants in test files. Maybe write type_string_test.go
	// in a separate pass? For later.
	// names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package.
	names = append(names, pkg.SFiles...)
	names = prefixDirectory(directory, names)
	g.parsePackage(directory, names, nil)
}

// parsePackageFiles parses the package occupying the named files.
func (g *Generator) parsePackageFiles(names []string) {
	g.parsePackage(".", names, nil)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) []string {
	if directory == "." {
		return names
	}
	ret := make([]string, len(names))
	for i, name := range names {
		ret[i] = filepath.Join(directory, name)
	}
	return ret
}

// parsePackage analyzes the single package constructed from the named files.
// If text is non-nil, it is a string to be used instead of the content of the file,
// to be used for testing. parsePackage exits if there is an error.
func (g *Generator) parsePackage(directory string, names []string, text interface{}) {
	var files []*File
	var astFiles []*ast.File
	g.pkg = new(Package)
	fs := token.NewFileSet()
	for _, name := range names {
		if !strings.HasSuffix(name, ".go") {
			continue
		}
		parsedFile, err := parser.ParseFile(fs, name, text, 0)
		if err != nil {
			log.Fatalf("parsing package: %s: %s", name, err)
		}
		astFiles = append(astFiles, parsedFile)
		files = append(files, &File{
			file: parsedFile,
			pkg:  g.pkg,
		})
	}
	if len(astFiles) == 0 {
		log.Fatalf("%s: no buildable Go files", directory)
	}
	g.pkg.name = astFiles[0].Name.Name
	g.pkg.files = files
	g.pkg.dir = directory

	// Type check the package.
	g.pkg.check(fs, astFiles)
}

// check type-checks the package. The package must be OK to proceed.
func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) {
	pkg.defs = make(map[*ast.Ident]types.Object)
	config := types.Config{FakeImportC: true}
	info := &types.Info{
		Defs: pkg.defs,
	}
	typesPkg, err := config.Check(pkg.dir, fs, astFiles, info)
	if err != nil {
		log.Fatalf("checking package: %s", err)
	}
	pkg.typesPkg = typesPkg
}

// format returns the gofmt-ed contents of the Generator's buffer.
func (g *Generator) format() []byte {
	src, err := format.Source(g.buf.Bytes())
	if err != nil {
		// Should never happen, but can arise when developing this code.
		// The user can compile the output to see the error.
		log.Printf("warning: internal error: invalid Go generated: %s", err)
		log.Printf("warning: compile the package to analyze the error")
		return g.buf.Bytes()
	}
	return src
}
<commit_msg>temporary fix for go 1.6<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	objectType = flag.String("type", "", "type to wrap in a reactive struct, required")
	name       = flag.String("name", "", "name of reactive type, required")
	output     = flag.String("output", "", "output file name; default srcdir/<type>_rx.go")
)

func main() {

	reg := regexp.MustCompile("[^A-Za-z0-9_]+")

	flag.Parse()

	if *objectType == "" {
		flag.Usage()
		os.Exit(2)
	}

	var (
		dir string
		g   Generator
	)

	args := flag.Args()
	if len(args) == 1 && isDirectory(args[0]) {
		dir = args[0]
		g.parsePackageDir(args[0])
	} else {
		dir = filepath.Dir(args[0])
		g.parsePackageFiles(args)
	}

	g.generateRx(*name, *objectType)

	src := g.format()

	outputName := *output
	if outputName == "" {
		// remove non alpha-numeric in filename
		o := reg.ReplaceAllString(*objectType, "")
		baseName := fmt.Sprintf("%s_rx.go", o)
		outputName = filepath.Join(dir, strings.ToLower(baseName))
	}
	err := ioutil.WriteFile(outputName, src, 0644)
	if err != nil {
		log.Fatalf("writing output: %s", err)
	}
}

func isDirectory(name string) bool {
	info, err := os.Stat(name)
	if err != nil {
		log.Fatal(err)
	}

	return info.IsDir()
}

type Generator struct {
	buf bytes.Buffer
	pkg *Package

	Params  *ast.FieldList
	Results *ast.FieldList
}

type File struct {
	pkg  *Package
	file *ast.File
}

type Package struct {
	dir      string
	name     string
	defs     map[*ast.Ident]types.Object
	files    []*File
	typesPkg *types.Package
}

// parsePackageDir parses the package residing in the directory.
func (g *Generator) parsePackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		log.Fatalf("cannot process directory %s: %s", directory, err)
	}
	var names []string
	names = append(names, pkg.GoFiles...)
	names = append(names, pkg.CgoFiles...)
	// TODO: Need to think about constants in test files. Maybe write type_string_test.go
	// in a separate pass? For later.
	// names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package.
	names = append(names, pkg.SFiles...)
	names = prefixDirectory(directory, names)
	g.parsePackage(directory, names, nil)
}

// parsePackageFiles parses the package occupying the named files.
func (g *Generator) parsePackageFiles(names []string) {
	g.parsePackage(".", names, nil)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) []string {
	if directory == "." {
		return names
	}
	ret := make([]string, len(names))
	for i, name := range names {
		ret[i] = filepath.Join(directory, name)
	}
	return ret
}

// parsePackage analyzes the single package constructed from the named files.
// If text is non-nil, it is a string to be used instead of the content of the file,
// to be used for testing. parsePackage exits if there is an error.
func (g *Generator) parsePackage(directory string, names []string, text interface{}) {
	var files []*File
	var astFiles []*ast.File
	g.pkg = new(Package)
	fs := token.NewFileSet()
	for _, name := range names {
		if !strings.HasSuffix(name, ".go") {
			continue
		}
		parsedFile, err := parser.ParseFile(fs, name, text, 0)
		if err != nil {
			log.Fatalf("parsing package: %s: %s", name, err)
		}
		astFiles = append(astFiles, parsedFile)
		files = append(files, &File{
			file: parsedFile,
			pkg:  g.pkg,
		})
	}
	if len(astFiles) == 0 {
		log.Fatalf("%s: no buildable Go files", directory)
	}
	g.pkg.name = astFiles[0].Name.Name
	g.pkg.files = files
	g.pkg.dir = directory

	// Type check the package.
	g.pkg.check(fs, astFiles)
}

// check type-checks the package. The package must be OK to proceed.
func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) {
	pkg.defs = make(map[*ast.Ident]types.Object)
	config := types.Config{FakeImportC: true}
	info := &types.Info{
		Defs: pkg.defs,
	}
	typesPkg, err := config.Check(pkg.dir, fs, astFiles, info)
	if err != nil {
		log.Fatalf("checking package: %s", err)
	}
	pkg.typesPkg = typesPkg
}

// format returns the gofmt-ed contents of the Generator's buffer.
func (g *Generator) format() []byte {
	src, err := format.Source(g.buf.Bytes())
	if err != nil {
		// Should never happen, but can arise when developing this code.
		// The user can compile the output to see the error.
		log.Printf("warning: internal error: invalid Go generated: %s", err)
		log.Printf("warning: compile the package to analyze the error")
		return g.buf.Bytes()
	}
	return src
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/op/go-logging"
	"github.com/spf13/cobra"

	"github.com/pulcy/quark/providers"
	"github.com/pulcy/quark/providers/cloudflare"
	"github.com/pulcy/quark/providers/digitalocean"
	"github.com/pulcy/quark/providers/vagrant"
	"github.com/pulcy/quark/providers/vultr"
)

var (
	projectVersion = "dev"
	projectBuild   = "dev"
	domain         = "pulcy.com"
)

var (
	cmdMain = &cobra.Command{
		Use:              "quark",
		Run:              showUsage,
		PersistentPreRun: loadDefaults,
	}

	provider          string
	digitalOceanToken string
	cloudflareApiKey  string
	cloudflareEmail   string
	vagrantFolder     string
	vultrApiKey       string

	log = logging.MustGetLogger(cmdMain.Use)
)

func init() {
	logging.SetFormatter(logging.MustStringFormatter("[%{level:-5s}] %{message}"))
	cmdMain.PersistentFlags().StringVarP(&provider, "provider", "p", "", "Provider used for creating clusters [digitalocean|vagrant|vultr]")
	cmdMain.PersistentFlags().StringVarP(&digitalOceanToken, "digitalocean-token", "t", "", "Digital Ocean token")
	cmdMain.PersistentFlags().StringVarP(&cloudflareApiKey, "cloudflare-apikey", "k", "", "Cloudflare API key")
	cmdMain.PersistentFlags().StringVarP(&cloudflareEmail, "cloudflare-email", "e", "", "Cloudflare email address")
	cmdMain.PersistentFlags().StringVarP(&vagrantFolder, "vagrant-folder", "f", defaultVagrantFolder(), "Directory containing vagrant files")
	cmdMain.PersistentFlags().StringVarP(&vultrApiKey, "vultr-apikey", "", "", "Vultr API key")
}

func main() {
	cmdMain.Execute()
}

func showUsage(cmd *cobra.Command, args []string) {
	cmd.Usage()
}

func loadDefaults(cmd *cobra.Command, args []string) {
	if digitalOceanToken == "" {
		digitalOceanToken = os.Getenv("DIGITALOCEAN_TOKEN")
	}
	if cloudflareApiKey == "" {
		cloudflareApiKey = os.Getenv("CLOUDFLARE_APIKEY")
	}
	if cloudflareEmail == "" {
		cloudflareEmail = os.Getenv("CLOUDFLARE_EMAIL")
	}
	if vultrApiKey == "" {
		vultrApiKey = os.Getenv("VULTR_APIKEY")
	}
}

func newProvider() providers.CloudProvider {
	switch provider {
	case "digitalocean":
		if digitalOceanToken == "" {
			Exitf("Please specify a token\n")
		}
		return digitalocean.NewProvider(log, digitalOceanToken)
	case "vagrant":
		if vagrantFolder == "" {
			Exitf("Please specify a vagrant-folder\n")
		}
		return vagrant.NewProvider(log, vagrantFolder)
	case "vultr":
		if vultrApiKey == "" {
			Exitf("Please specify a vultr-apikey\n")
		}
		return vultr.NewProvider(log, vultrApiKey)
	default:
		Exitf("Unknown provider '%s'\n", provider)
		return nil
	}
}

func newDnsProvider() providers.DnsProvider {
	if cloudflareApiKey == "" {
		Exitf("Please specify a cloudflare-apikey\n")
	}
	if cloudflareEmail == "" {
		Exitf("Please specify a cloudflare-email\n")
	}
	return cloudflare.NewProvider(log, cloudflareApiKey, cloudflareEmail)
}

func confirm(question string) error {
	for {
		fmt.Printf("%s [yes|no]", question)
		bufStdin := bufio.NewReader(os.Stdin)
		line, _, err := bufStdin.ReadLine()
		if err != nil {
			return err
		}

		if string(line) == "yes" || string(line) == "y" {
			return nil
		}
		fmt.Println("Please enter 'yes' to confirm.")
	}
}

func Exitf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	os.Exit(1)
}

func Infof(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}

// clusterInfoFromArgs fills the given cluster info from a command line argument
func clusterInfoFromArgs(info *providers.ClusterInfo, args []string) {
	if len(args) == 1 && info.Name == "" {
		parts := strings.SplitN(args[0], ".", 2)
		info.Name = parts[0]
		info.Domain = parts[1]
	}
}

// clusterInstanceInfoFromArgs fills the given cluster info from a command line argument
func clusterInstanceInfoFromArgs(info *providers.ClusterInstanceInfo, args []string) {
	if len(args) == 1 && info.Prefix == "" && info.Name == "" {
		parts := strings.SplitN(args[0], ".", 3)
		info.Prefix = parts[0]
		info.Name = parts[1]
		info.Domain = parts[2]
	}
}
<commit_msg>Configurable loglevel<commit_after>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/op/go-logging"
	"github.com/spf13/cobra"

	"github.com/pulcy/quark/providers"
	"github.com/pulcy/quark/providers/cloudflare"
	"github.com/pulcy/quark/providers/digitalocean"
	"github.com/pulcy/quark/providers/vagrant"
	"github.com/pulcy/quark/providers/vultr"
)

var (
	projectVersion = "dev"
	projectBuild   = "dev"
	domain         = "pulcy.com"
)

const (
	projectName     = "quark"
	defaultLogLevel = "info"
)

var (
	cmdMain = &cobra.Command{
		Use:              projectName,
		Run:              showUsage,
		PersistentPreRun: loadDefaults,
	}

	provider          string
	digitalOceanToken string
	cloudflareApiKey  string
	cloudflareEmail   string
	vagrantFolder     string
	vultrApiKey       string
	logLevel          string

	log = logging.MustGetLogger(projectName)
)

func init() {
	logging.SetFormatter(logging.MustStringFormatter("[%{level:-5s}] %{message}"))
	cmdMain.PersistentFlags().StringVar(&logLevel, "log-level", defaultLogLevel, "Log level (debug|info|warning|error)")
	cmdMain.PersistentFlags().StringVarP(&provider, "provider", "p", "", "Provider used for creating clusters [digitalocean|vagrant|vultr]")
	cmdMain.PersistentFlags().StringVarP(&digitalOceanToken, "digitalocean-token", "t", "", "Digital Ocean token")
	cmdMain.PersistentFlags().StringVarP(&cloudflareApiKey, "cloudflare-apikey", "k", "", "Cloudflare API key")
	cmdMain.PersistentFlags().StringVarP(&cloudflareEmail, "cloudflare-email", "e", "", "Cloudflare email address")
	cmdMain.PersistentFlags().StringVarP(&vagrantFolder, "vagrant-folder", "f", defaultVagrantFolder(), "Directory containing vagrant files")
	cmdMain.PersistentFlags().StringVarP(&vultrApiKey, "vultr-apikey", "", "", "Vultr API key")
}

func main() {
	cmdMain.Execute()
}

func showUsage(cmd *cobra.Command, args []string) {
	cmd.Usage()
}

func loadDefaults(cmd *cobra.Command, args []string) {
	if digitalOceanToken == "" {
		digitalOceanToken = os.Getenv("DIGITALOCEAN_TOKEN")
	}
	if cloudflareApiKey == "" {
		cloudflareApiKey = os.Getenv("CLOUDFLARE_APIKEY")
	}
	if cloudflareEmail == "" {
		cloudflareEmail = os.Getenv("CLOUDFLARE_EMAIL")
	}
	if vultrApiKey == "" {
		vultrApiKey = os.Getenv("VULTR_APIKEY")
	}

	// Set loglevel
	level, err := logging.LogLevel(logLevel)
	if err != nil {
		Exitf("Invalid log-level '%s': %#v", logLevel, err)
	}
	logging.SetLevel(level, projectName)
}

func newProvider() providers.CloudProvider {
	switch provider {
	case "digitalocean":
		if digitalOceanToken == "" {
			Exitf("Please specify a token\n")
		}
		return digitalocean.NewProvider(log, digitalOceanToken)
	case "vagrant":
		if vagrantFolder == "" {
			Exitf("Please specify a vagrant-folder\n")
		}
		return vagrant.NewProvider(log, vagrantFolder)
	case "vultr":
		if vultrApiKey == "" {
			Exitf("Please specify a vultr-apikey\n")
		}
		return vultr.NewProvider(log, vultrApiKey)
	default:
		Exitf("Unknown provider '%s'\n", provider)
		return nil
	}
}

func newDnsProvider() providers.DnsProvider {
	if cloudflareApiKey == "" {
		Exitf("Please specify a cloudflare-apikey\n")
	}
	if cloudflareEmail == "" {
		Exitf("Please specify a cloudflare-email\n")
	}
	return cloudflare.NewProvider(log, cloudflareApiKey, cloudflareEmail)
}

func confirm(question string) error {
	for {
		fmt.Printf("%s [yes|no]", question)
		bufStdin := bufio.NewReader(os.Stdin)
		line, _, err := bufStdin.ReadLine()
		if err != nil {
			return err
		}

		if string(line) == "yes" || string(line) == "y" {
			return nil
		}
		fmt.Println("Please enter 'yes' to confirm.")
	}
}

func Exitf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	os.Exit(1)
}

func Infof(format string, args ...interface{}) {
	fmt.Printf(format, args...)
}

// clusterInfoFromArgs fills the given cluster info from a command line argument
func clusterInfoFromArgs(info *providers.ClusterInfo, args []string) {
	if len(args) == 1 && info.Name == "" {
		parts := strings.SplitN(args[0], ".", 2)
		info.Name = parts[0]
		info.Domain = parts[1]
	}
}

// clusterInstanceInfoFromArgs fills the given cluster info from a command line argument
func clusterInstanceInfoFromArgs(info *providers.ClusterInstanceInfo, args []string) {
	if len(args) == 1 && info.Prefix == "" && info.Name == "" {
		parts := strings.SplitN(args[0], ".", 3)
		info.Prefix = parts[0]
		info.Name = parts[1]
		info.Domain = parts[2]
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"runtime"
	"time"

	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
	"github.com/VOID001/D-judge/config"
	"github.com/VOID001/D-judge/judge-controller"
	"github.com/VOID001/D-judge/request"

	"github.com/pkg/errors"
)

// Log level constant
const (
	INFO    = 0    // INFO Level
	WARN    = 1    // WARN Level
	DEBUG   = 2    // DEBUG Level
	DirPerm = 0744 // Dir Permission
)

// Error constants
const (
	ErrNoDir  = "no such file or directory"
	ErrNoFile = "no such file or directory"
)

var path string
var debuglv int64

// GlobalConfig Config Object contain the global system config
var GlobalConfig config.SystemConfig

func init() {
	var logfile string
	flag.StringVar(&path, "c", "config.toml", "select configuration file")
	flag.Int64Var(&debuglv, "d", 0, "debug mode enabled")
	flag.StringVar(&logfile, "log", "/dev/stdout", "log file")
	flag.Parse()
	_, err := toml.DecodeFile(path, &GlobalConfig)
	if err != nil {
		err = errors.Wrap(err, "Processing config file error")
		log.Fatal(err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		err = errors.Wrap(err, "Get current directory error")
	}
	if !filepath.IsAbs(GlobalConfig.JudgeRoot) {
		GlobalConfig.JudgeRoot = filepath.Join(cwd, GlobalConfig.JudgeRoot)
	}
	if !filepath.IsAbs(GlobalConfig.CacheRoot) {
		GlobalConfig.CacheRoot = filepath.Join(cwd, GlobalConfig.CacheRoot)
	}
	if debuglv == INFO {
		log.SetLevel(log.InfoLevel)
	}
	if debuglv == WARN {
		log.SetLevel(log.WarnLevel)
	}
	if debuglv == DEBUG {
		log.SetLevel(log.DebugLevel)
	}
	f, _ := os.Create(logfile)
	log.SetOutput(f)
	config.GlobalConfig = GlobalConfig
}

func main() {
	log.Debugf("Settings %+v", GlobalConfig)
	// Perform Sanity Check
	log.Infof("sanity check start")
	err := sanityCheckDir(GlobalConfig.JudgeRoot)
	if err != nil {
		err = errors.Wrap(err, "sanity check dir judgeroot error")
		log.Fatal(err)
	}
	err = sanityCheckDir(GlobalConfig.CacheRoot)
	if err != nil {
		err = errors.Wrap(err, "sanity check dir cacheroot error")
		log.Fatal(err)
	}
	err = sanityCheckConnection(GlobalConfig.EndpointURL)
	if err != nil {
		err = errors.Wrap(err, "sanity check connection error")
		log.Fatal(err)
	}
	err = sanityCheckDocker()
	if err != nil {
		err = errors.Wrap(err, "sanity check docker error")
		log.Fatal(err)
	}

	// Error When Requesting Judgehost
	err = request.Do(context.Background(), http.MethodPost, "/judgehosts", url.Values{"hostname": {config.GlobalConfig.HostName}}, request.TypeForm, nil)
	if err != nil {
		err = errors.Wrap(err, "main loop error")
		log.Fatal(err)
	}
	log.Infof("sanity check success")

	// PerformRequest Lifcycle
	daemon := controller.Daemon{}
	daemon.MaxWorker = runtime.NumCPU()
	go daemon.Run(context.Background())
	for {
		jinfo := config.JudgeInfo{}
		// Request For Judge
		err = request.Do(context.Background(), http.MethodPost, fmt.Sprintf("/judgings?judgehost=%s", config.GlobalConfig.HostName), nil, "", &jinfo)
		if err != nil {
			log.Warn(err)
		}
		log.Debugf("Judge Info %+v", jinfo)
		if jinfo.SubmitID != 0 {
			log.Infof("Fetched Submission ID #%d", jinfo.SubmitID)
			workDir := fmt.Sprintf("%s/c%d-s%d-j%d", config.GlobalConfig.JudgeRoot, jinfo.ContestID, jinfo.SubmitID, jinfo.JudgingID)
			if _, err := os.Stat(workDir); err == nil {
				oldWorkDir := fmt.Sprintf("%s-old-%d", workDir, time.Now().Unix())
				err := os.Rename(workDir, oldWorkDir)
				if err != nil {
					err = errors.Wrap(err, "main loop error")
					log.Fatal(err)
				}
			}
			os.Mkdir(workDir, DirPerm)
			daemon.AddTask(context.Background(), jinfo, workDir, config.GlobalConfig.DockerImage)
		}
		//time.Sleep(time.Duration(rand.Intn(2500)) * time.Millisecond)
		time.Sleep(time.Duration(1000) * time.Millisecond)
	}
}

func sanityCheckDir(dir string) (err error) {
	_, err = ioutil.ReadDir(dir)
	if err != nil && os.IsNotExist(err) {
		err = os.Mkdir(dir, DirPerm)
		if err != nil {
			err = errors.Wrap(err, fmt.Sprintf("cannot make %s", dir))
			return
		}
		log.Infof("created dir %s with mode %04o", dir, DirPerm)
	}
	info, err := os.Stat(dir)
	log.Infof("dir %s mode bits %s", dir, info.Mode())
	return
}

func sanityCheckConnection(endpoint string) (err error) {
	req, err := http.NewRequest(http.MethodPost, endpoint, nil)
	req.SetBasicAuth(config.GlobalConfig.EndpointUser, config.GlobalConfig.EndpointPassword)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("cannot create request %s", endpoint))
		return
	}
	cli := http.Client{}
	resp, err := cli.Do(req)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("cannot connect to %s", endpoint))
		return
	}
	defer func() {
		err := resp.Body.Close()
		if err != nil {
			log.Errorf("body close error %s", err.Error())
		}
	}()
	return
}

func sanityCheckDocker() (err error) {
	err = controller.Ping(context.Background())
	if err != nil {
		err = errors.Wrap(err, "docker Ping error")
	}
	return
}
<commit_msg>Fix data race and make log JSON formatted<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"runtime"
	"time"

	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"

	"github.com/BurntSushi/toml"
	log "github.com/Sirupsen/logrus"
	"github.com/VOID001/D-judge/config"
	"github.com/VOID001/D-judge/judge-controller"
	"github.com/VOID001/D-judge/request"

	"github.com/pkg/errors"
)

// Log level constant
const (
	INFO    = 0    // INFO Level
	WARN    = 1    // WARN Level
	DEBUG   = 2    // DEBUG Level
	DirPerm = 0744 // Dir Permission
)

// Error constants
const (
	ErrNoDir  = "no such file or directory"
	ErrNoFile = "no such file or directory"
)

var path string
var debuglv int64

// GlobalConfig Config Object contain the global system config
var GlobalConfig config.SystemConfig

func init() {
	var logfile string
	flag.StringVar(&path, "c", "config.toml", "select configuration file")
	flag.Int64Var(&debuglv, "d", 0, "debug mode enabled")
	flag.StringVar(&logfile, "log", "/dev/stdout", "log file")
	flag.Parse()
	_, err := toml.DecodeFile(path, &GlobalConfig)
	if err != nil {
		err = errors.Wrap(err, "Processing config file error")
		log.Fatal(err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		err = errors.Wrap(err, "Get current directory error")
	}
	if !filepath.IsAbs(GlobalConfig.JudgeRoot) {
		GlobalConfig.JudgeRoot = filepath.Join(cwd, GlobalConfig.JudgeRoot)
	}
	if !filepath.IsAbs(GlobalConfig.CacheRoot) {
		GlobalConfig.CacheRoot = filepath.Join(cwd, GlobalConfig.CacheRoot)
	}
	if debuglv == INFO {
		log.SetLevel(log.InfoLevel)
	}
	if debuglv == WARN {
		log.SetLevel(log.WarnLevel)
	}
	if debuglv == DEBUG {
		log.SetLevel(log.DebugLevel)
	}
	f, _ := os.Create(logfile)
	log.SetOutput(f)
	log.SetFormatter(&log.JSONFormatter{})
	config.GlobalConfig = GlobalConfig
}

func main() {
	log.Debugf("Settings %+v", GlobalConfig)
	// Perform Sanity Check
	log.Infof("sanity check start")
	err := sanityCheckDir(GlobalConfig.JudgeRoot)
	if err != nil {
		err = errors.Wrap(err, "sanity check dir judgeroot error")
		log.Fatal(err)
	}
	err = sanityCheckDir(GlobalConfig.CacheRoot)
	if err != nil {
		err = errors.Wrap(err, "sanity check dir cacheroot error")
		log.Fatal(err)
	}
	err = sanityCheckConnection(GlobalConfig.EndpointURL)
	if err != nil {
		err = errors.Wrap(err, "sanity check connection error")
		log.Fatal(err)
	}
	err = sanityCheckDocker()
	if err != nil {
		err = errors.Wrap(err, "sanity check docker error")
		log.Fatal(err)
	}

	// Error When Requesting Judgehost
	err = request.Do(context.Background(), http.MethodPost, "/judgehosts", url.Values{"hostname": {config.GlobalConfig.HostName}}, request.TypeForm, nil)
	if err != nil {
		err = errors.Wrap(err, "main loop error")
		log.Fatal(err)
	}
	log.Infof("sanity check success")

	// PerformRequest Lifcycle
	daemon := controller.Daemon{}
	daemon.MaxWorker = runtime.NumCPU()
	daemon.Run(context.Background())
	for {
		jinfo := config.JudgeInfo{}
		// Request For Judge
		err = request.Do(context.Background(), http.MethodPost, fmt.Sprintf("/judgings?judgehost=%s", config.GlobalConfig.HostName), nil, "", &jinfo)
		if err != nil {
			log.Warn(err)
		}
		log.Debugf("Judge Info %+v", jinfo)
		if jinfo.SubmitID != 0 {
			log.Infof("Fetched Submission ID #%d", jinfo.SubmitID)
			workDir := fmt.Sprintf("%s/c%d-s%d-j%d", config.GlobalConfig.JudgeRoot, jinfo.ContestID, jinfo.SubmitID, jinfo.JudgingID)
			if _, err := os.Stat(workDir); err == nil {
				oldWorkDir := fmt.Sprintf("%s-old-%d", workDir, time.Now().Unix())
				log.Infof("Found stale working directory, rename to %s", oldWorkDir)
				err := os.Rename(workDir, oldWorkDir)
				if err != nil {
					err = errors.Wrap(err, "main loop error")
					log.Fatal(err)
				}
			}
			os.Mkdir(workDir, DirPerm)
			daemon.AddTask(context.Background(), jinfo, workDir, config.GlobalConfig.DockerImage)
		}
		time.Sleep(time.Duration(rand.Intn(2500)) * time.Millisecond)
	}
}

func sanityCheckDir(dir string) (err error) {
	_, err = ioutil.ReadDir(dir)
	if err != nil && os.IsNotExist(err) {
		err = os.Mkdir(dir, DirPerm)
		if err != nil {
			err = errors.Wrap(err, fmt.Sprintf("cannot make %s", dir))
			return
		}
		log.Infof("created dir %s with mode %04o", dir, DirPerm)
	}
	info, err := os.Stat(dir)
	log.Infof("dir %s mode bits %s", dir, info.Mode())
	return
}

func sanityCheckConnection(endpoint string) (err error) {
	req, err := http.NewRequest(http.MethodPost, endpoint, nil)
	req.SetBasicAuth(config.GlobalConfig.EndpointUser, config.GlobalConfig.EndpointPassword)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("cannot create request %s", endpoint))
		return
	}
	cli := http.Client{}
	resp, err := cli.Do(req)
	if err != nil {
		err = errors.Wrap(err, fmt.Sprintf("cannot connect to %s", endpoint))
		return
	}
	defer func() {
		err := resp.Body.Close()
		if err != nil {
			log.Errorf("body close error %s", err.Error())
		}
	}()
	return
}

func sanityCheckDocker() (err error) {
	err = controller.Ping(context.Background())
	if err != nil {
		err = errors.Wrap(err, "docker Ping error")
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/Cepave/open-falcon-backend/cmd"
	"github.com/chyeh/viper"
	"github.com/spf13/cobra"
)

var RootCmd = &cobra.Command{
	Use:   "open-falcon",
	Short: "A brief description of your application",
	Long: `A longer description that spans multiple lines and likely contains
examples and usage of using your application. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
	// Uncomment the following line if your bare application
	// has an action associated with it:
	//	Run: func(cmd *cobra.Command, args []string) { },
}

// Commands is the mapping of all the available Consul commands.
//var Commands map[string]cli.CommandFactory
//var startCommand cli.Command

func init() {
	cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports Persistent Flags, which, if defined here,
	// will be global for your application.

	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.newAppName.yaml)")
	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	//RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
	RootCmd.AddCommand(cmd.Start)
	RootCmd.AddCommand(cmd.Stop)
	RootCmd.AddCommand(cmd.Restart)
	RootCmd.AddCommand(cmd.Status)
	RootCmd.AddCommand(cmd.Tail)
	RootCmd.AddCommand(cmd.Reload)
	cmd.Start.Flags().BoolVar(&cmd.PreqOrderFlag, "preq-order", false, "start modules in the order of prerequisites")
	cmd.Start.Flags().BoolVar(&cmd.LogfileFlag, "logfile", false, "log modules' output to files")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	//if cfgFile != "" { // enable ability to specify config file via flag
	//	viper.SetConfigFile(cfgFile)
	//}

	viper.SetConfigName(".newAppName") // name of config file (without extension)
	viper.AddConfigPath("$HOME")       // adding home directory as first search path
	viper.AutomaticEnv()               // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}

func main() {
	log.SetOutput(ioutil.Discard)

	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	args := os.Args[1:]
	for _, arg := range args {
		if arg == "--" {
			break
		}
		if arg == "-v" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	//cli := &cli.CLI{
	//	Args:     args,
	//	Commands: Commands,
	//	//	HelpFunc: cli.BasicHelpFunc("consul"),
	//}
	//exitCode, err := cli.Run()
	//if err != nil {
	//	fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
	//	return 1
	//}

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>[OWL-914][backend] Comment some unused code<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/Cepave/open-falcon-backend/cmd"
	"github.com/chyeh/viper"
	"github.com/spf13/cobra"
)

var RootCmd = &cobra.Command{
	Use: "open-falcon",
	//Short: "A brief description of your application",
	//Long: `A longer description that spans multiple lines and likely contains
	//examples and usage of using your application. For example:
	//
	//Cobra is a CLI library for Go that empowers applications.
	//This application is a tool to generate the needed files
	//to quickly create a Cobra application.`,
	// Uncomment the following line if your bare application
	// has an action associated with it:
	//	Run: func(cmd *cobra.Command, args []string) { },
}

// Commands is the mapping of all the available Consul commands.
//var Commands map[string]cli.CommandFactory
//var startCommand cli.Command

func init() {
	//cobra.OnInitialize(initConfig)

	// Here you will define your flags and configuration settings.
	// Cobra supports Persistent Flags, which, if defined here,
	// will be global for your application.

	//RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.newAppName.yaml)")
	// Cobra also supports local flags, which will only run
	// when this action is called directly.
	//RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
	RootCmd.AddCommand(cmd.Start)
	RootCmd.AddCommand(cmd.Stop)
	RootCmd.AddCommand(cmd.Restart)
	RootCmd.AddCommand(cmd.Status)
	RootCmd.AddCommand(cmd.Tail)
	RootCmd.AddCommand(cmd.Reload)
	cmd.Start.Flags().BoolVar(&cmd.PreqOrderFlag, "preq-order", false, "start modules in the order of prerequisites")
	cmd.Start.Flags().BoolVar(&cmd.LogfileFlag, "logfile", false, "log modules' output to files")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	//if cfgFile != "" { // enable ability to specify config file via flag
	//	viper.SetConfigFile(cfgFile)
	//}

	viper.SetConfigName(".newAppName") // name of config file (without extension)
	viper.AddConfigPath("$HOME")       // adding home directory as first search path
	viper.AutomaticEnv()               // read in environment variables that match

	// If a config file is found, read it in.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Println("Using config file:", viper.ConfigFileUsed())
	}
}

func main() {
	//log.SetOutput(ioutil.Discard)

	// Get the command line args. We shortcut "--version" and "-v" to
	// just show the version.
	//args := os.Args[1:]
	//for _, arg := range args {
	//	if arg == "--" {
	//		break
	//	}
	//	if arg == "-v" || arg == "--version" {
	//		newArgs := make([]string, len(args)+1)
	//		newArgs[0] = "version"
	//		copy(newArgs[1:], args)
	//		args = newArgs
	//		break
	//	}
	//}

	//cli := &cli.CLI{
	//	Args:     args,
	//	Commands: Commands,
	//	//	HelpFunc: cli.BasicHelpFunc("consul"),
	//}
	//exitCode, err := cli.Run()
	//if err != nil {
	//	fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
	//	return 1
	//}

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Jan Broer. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main // import "github.com/janeczku/go-dnsmasq"

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog"
	"github.com/codegangsta/cli"
	"github.com/miekg/dns"

	"github.com/janeczku/go-dnsmasq/hostsfile"
	"github.com/janeczku/go-dnsmasq/resolvconf"
	"github.com/janeczku/go-dnsmasq/server"
	"github.com/janeczku/go-dnsmasq/stats"
)

// var Version string
const Version = "0.9.6"

var (
	nameservers   = []string{}
	searchDomains = []string{}
	stubzones     = ""
	hostPort      = ""
	listen        = ""
)

var exitErr error

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	app := cli.NewApp()
	app.Name = "go-dnsmasq"
	app.Usage = "Lightweight caching DNS proxy for Docker containers"
	app.Version = Version
	app.Author, app.Email = "", ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen, l",
			Value:  "127.0.0.1:53",
			Usage:  "listen address: ‘host[:port]‘",
			EnvVar: "DNSMASQ_LISTEN",
		},
		cli.BoolFlag{
			Name:   "default-resolver, d",
			Usage:  "make go-dnsmasq the local primary nameserver (updates /etc/resolv.conf)",
			EnvVar: "DNSMASQ_DEFAULT",
		},
		cli.StringFlag{
			Name:   "nameservers, n",
			Value:  "",
			Usage:  "comma-separated list of name servers: ‘host[:port]‘",
			EnvVar: "DNSMASQ_SERVERS",
		},
		cli.StringFlag{
			Name:   "stubzones, z",
			Value:  "",
			Usage:  "domains to resolve using a specific nameserver: ‘fqdn[,fqdn]/host[:port]‘",
			EnvVar: "DNSMASQ_STUB",
		},
		cli.StringFlag{
			Name:   "hostsfile, f",
			Value:  "",
			Usage:  "full path to hostsfile (e.g. ‘/etc/hosts‘)",
			EnvVar: "DNSMASQ_HOSTSFILE",
		},
		cli.IntFlag{
			Name:   "hostsfile-poll, p",
			Value:  0,
			Usage:  "how frequently to poll hostsfile (in seconds, ‘0‘ to disable)",
			EnvVar: "DNSMASQ_POLL",
		},
		cli.StringFlag{
			Name:   "search-domains, s",
			Value:  "",
			Usage:  "specify SEARCH domains taking precedence over /etc/resolv.conf: ‘fqdn[,fqdn]‘",
			EnvVar: "DNSMASQ_SEARCH",
		},
		cli.BoolFlag{
			Name:   "append-search-domains, a",
			Usage:  "enable suffixing single-label queries with SEARCH domains",
			EnvVar: "DNSMASQ_APPEND",
		},
		cli.IntFlag{
			Name:   "rcache, r",
			Value:  0,
			Usage:  "capacity of the response cache (‘0‘ to disable caching)",
			EnvVar: "DNSMASQ_RCACHE",
		},
		cli.IntFlag{
			Name:   "rcache-ttl",
			Value:  server.RCacheTtl,
			Usage:  "TTL of entries in the response cache",
			EnvVar: "DNSMASQ_RCACHE_TTL",
		},
		cli.BoolFlag{
			Name:   "no-rec",
			Usage:  "disable recursion",
			EnvVar: "DNSMASQ_NOREC",
		},
		cli.BoolFlag{
			Name:   "round-robin",
			Usage:  "enable round robin of A/AAAA replies",
			EnvVar: "DNSMASQ_RR",
		},
		cli.BoolFlag{
			Name:   "systemd",
			Usage:  "bind to socket(s) activated by systemd (ignores --listen)",
			EnvVar: "DNSMASQ_SYSTEMD",
		},
		cli.BoolFlag{
			Name:   "verbose",
			Usage:  "enable verbose logging",
			EnvVar: "DNSMASQ_VERBOSE",
		},
		cli.BoolFlag{
			Name:   "syslog",
			Usage:  "enable syslog logging",
			EnvVar: "DNSMASQ_SYSLOG",
		},
		cli.BoolFlag{
			Name:   "multithreading",
			Usage:  "enable multithreading (num physical CPU cores)",
			EnvVar: "DNSMASQ_MULTITHREADING",
		},
	}
	app.Action = func(c *cli.Context) {
		exitReason := make(chan error)
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
			sig := <-c
			log.Infoln("Application exit requested by signal:", sig)
			exitReason <- nil
		}()

		if c.Bool("multithreading") {
			runtime.GOMAXPROCS(runtime.NumCPU() + 1)
		}

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if c.Bool("syslog") {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true, DisableColors: true})
			hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DAEMON|syslog.LOG_INFO, "go-dnsmasq")
			if err != nil {
				log.Error("Unable to connect to local syslog daemon")
			} else {
				log.AddHook(hook)
			}
		} else {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true})
		}

		if ns := c.String("nameservers"); ns != "" {
			for _, hostPort := range strings.Split(ns, ",") {
				if !strings.Contains(hostPort, ":") {
					hostPort += ":53"
				}

				if err := validateHostPort(hostPort); err != nil {
					log.Fatalf("This nameserver is invalid: %s", err)
				}

				nameservers = append(nameservers, hostPort)
			}
		}

		if sd := c.String("search-domains"); sd != "" {
			for _, domain := range strings.Split(sd, ",") {

				if dns.CountLabel(domain) < 2 {
					log.Fatalf("This search domain is not a FQDN: %s", domain)
				}
				domain = dns.Fqdn(strings.ToLower(domain))
				searchDomains = append(searchDomains, domain)
			}
		}

		if listen = c.String("listen"); !strings.Contains(listen, ":") {
			listen += ":53"
		}

		if err := validateHostPort(listen); err != nil {
			log.Fatalf("The listen address is invalid: %s", err)
		}

		config := &server.Config{
			DnsAddr:         listen,
			DefaultResolver: c.Bool("default-resolver"),
			Nameservers:     nameservers,
			Systemd:         c.Bool("systemd"),
			SearchDomains:   searchDomains,
			AppendDomain:    c.Bool("append-search-domains"),
			Hostsfile:       c.String("hostsfile"),
			PollInterval:    c.Int("hostsfile-poll"),
			RoundRobin:      c.Bool("round-robin"),
			NoRec:           c.Bool("no-rec"),
			ReadTimeout:     0,
			RCache:          c.Int("rcache"),
			RCacheTtl:       c.Int("rcache-ttl"),
			Verbose:         c.Bool("verbose"),
		}

		if err := server.SetDefaults(config); err != nil {
			if !config.NoRec && len(config.Nameservers) == 0 {
				log.Fatalf("No nameservers found in resolv.conf and --nameservers option not supplied: %s", err)
			} else if config.AppendDomain && len(config.SearchDomains) == 0 {
				log.Fatalf("No search domains found in resolv.conf and --search-domains option not supplied: %s", err)
			} else {
				log.Warnf("Error parsing resolv.conf: %s", err)
			}
		}

		if stubzones = c.String("stubzones"); stubzones != "" {
			stubmap := make(map[string][]string)
			segments := strings.Split(stubzones, "/")
			if len(segments) != 2 || len(segments[0]) == 0 || len(segments[1]) == 0 {
				log.Fatalf("The --stubzones argument is invalid")
			}

			hostPort = segments[1]
			if !strings.Contains(hostPort, ":") {
				hostPort += ":53"
			}

			if err := validateHostPort(hostPort); err != nil {
				log.Fatalf("This stubzones server address invalid: %s", err)
			}

			for _, sdomain := range strings.Split(segments[0], ",") {
				if dns.CountLabel(sdomain) < 2 {
					log.Fatalf("This stubzones domain is not a FQDN: %s", sdomain)
				}
				sdomain = dns.Fqdn(sdomain)
				stubmap[sdomain] = append(stubmap[sdomain], hostPort)
			}

			config.Stub = &stubmap
		}

		log.Infof("Starting go-dnsmasq server %s ...", Version)
		if config.AppendDomain {
			log.Infof("Search domains in use: %v", config.SearchDomains)
		}

		hf, err := hosts.NewHostsfile(config.Hostsfile, &hosts.Config{
			Poll:    config.PollInterval,
			Verbose: config.Verbose,
		})
		if err != nil {
			log.Fatalf("Error loading hostsfile: %s", err)
		}

		s := server.New(hf, config, Version)

		defer s.Stop()

		stats.Collect()

		if config.DefaultResolver {
			address, _, _ := net.SplitHostPort(config.DnsAddr)
			err := resolvconf.StoreAddress(address)
			if err != nil {
				log.Warnf("Failed to register as default nameserver: %s", err)
			}
			defer resolvconf.Clean()
		}

		go func() {
			if err := s.Run(); err != nil {
				exitReason <- err
			}
		}()

		exitErr = <-exitReason
		if exitErr != nil {
			log.Fatalf("Server error: %s", err)
		}
	}

	app.Run(os.Args)
}

func validateHostPort(hostPort string) error {
	host, port, err := net.SplitHostPort(hostPort)
	if err != nil {
		return err
	}
	if ip := net.ParseIP(host); ip == nil {
		return fmt.Errorf("Bad IP address: %s", host)
	}

	if p, _ := strconv.Atoi(port); p < 1 || p > 65535 {
		return fmt.Errorf("Bad port number %s", port)
	}
	return nil
}
<commit_msg>Bump version 0.9.7<commit_after>// Copyright (c) 2015 Jan Broer. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main // import "github.com/janeczku/go-dnsmasq"

import (
	"fmt"
	"log/syslog"
	"net"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog"
	"github.com/codegangsta/cli"
	"github.com/miekg/dns"

	"github.com/janeczku/go-dnsmasq/hostsfile"
	"github.com/janeczku/go-dnsmasq/resolvconf"
	"github.com/janeczku/go-dnsmasq/server"
	"github.com/janeczku/go-dnsmasq/stats"
)

// var Version string
const Version = "0.9.7"

var (
	nameservers   = []string{}
	searchDomains = []string{}
	stubzones     = ""
	hostPort      = ""
	listen        = ""
)

var exitErr error

func init() {
	log.SetOutput(os.Stderr)
}

func main() {
	app := cli.NewApp()
	app.Name = "go-dnsmasq"
	app.Usage = "Lightweight caching DNS proxy for Docker containers"
	app.Version = Version
	app.Author, app.Email = "", ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen, l",
			Value:  "127.0.0.1:53",
			Usage:  "listen address: ‘host[:port]‘",
			EnvVar: "DNSMASQ_LISTEN",
		},
		cli.BoolFlag{
			Name:   "default-resolver, d",
			Usage:  "make go-dnsmasq the local primary nameserver (updates /etc/resolv.conf)",
			EnvVar: "DNSMASQ_DEFAULT",
		},
		cli.StringFlag{
			Name:   "nameservers, n",
			Value:  "",
			Usage:  "comma-separated list of name servers: ‘host[:port]‘",
			EnvVar: "DNSMASQ_SERVERS",
		},
		cli.StringFlag{
			Name:   "stubzones, z",
			Value:  "",
			Usage:  "domains to resolve using a specific nameserver: ‘fqdn[,fqdn]/host[:port]‘",
			EnvVar: "DNSMASQ_STUB",
		},
		cli.StringFlag{
			Name:   "hostsfile, f",
			Value:  "",
			Usage:  "full path to hostsfile (e.g. ‘/etc/hosts‘)",
			EnvVar: "DNSMASQ_HOSTSFILE",
		},
		cli.IntFlag{
			Name:   "hostsfile-poll, p",
			Value:  0,
			Usage:  "how frequently to poll hostsfile (in seconds, ‘0‘ to disable)",
			EnvVar: "DNSMASQ_POLL",
		},
		cli.StringFlag{
			Name:   "search-domains, s",
			Value:  "",
			Usage:  "specify SEARCH domains taking precedence over /etc/resolv.conf: ‘fqdn[,fqdn]‘",
			EnvVar: "DNSMASQ_SEARCH",
		},
		cli.BoolFlag{
			Name:   "append-search-domains, a",
			Usage:  "enable suffixing single-label queries with SEARCH domains",
			EnvVar: "DNSMASQ_APPEND",
		},
		cli.IntFlag{
			Name:   "rcache, r",
			Value:  0,
			Usage:  "capacity of the response cache (‘0‘ to disable caching)",
			EnvVar: "DNSMASQ_RCACHE",
		},
		cli.IntFlag{
			Name:   "rcache-ttl",
			Value:  server.RCacheTtl,
			Usage:  "TTL of entries in the response cache",
			EnvVar: "DNSMASQ_RCACHE_TTL",
		},
		cli.BoolFlag{
			Name:   "no-rec",
			Usage:  "disable recursion",
			EnvVar: "DNSMASQ_NOREC",
		},
		cli.BoolFlag{
			Name:   "round-robin",
			Usage:  "enable round robin of A/AAAA replies",
			EnvVar: "DNSMASQ_RR",
		},
		cli.BoolFlag{
			Name:   "systemd",
			Usage:  "bind to socket(s) activated by systemd (ignores --listen)",
			EnvVar: "DNSMASQ_SYSTEMD",
		},
		cli.BoolFlag{
			Name:   "verbose",
			Usage:  "enable verbose logging",
			EnvVar: "DNSMASQ_VERBOSE",
		},
		cli.BoolFlag{
			Name:   "syslog",
			Usage:  "enable syslog logging",
			EnvVar: "DNSMASQ_SYSLOG",
		},
		cli.BoolFlag{
			Name:   "multithreading",
			Usage:  "enable multithreading (num physical CPU cores)",
			EnvVar: "DNSMASQ_MULTITHREADING",
		},
	}
	app.Action = func(c *cli.Context) {
		exitReason := make(chan error)
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
			sig := <-c
			log.Infoln("Application exit requested by signal:", sig)
			exitReason <- nil
		}()

		if c.Bool("multithreading") {
			runtime.GOMAXPROCS(runtime.NumCPU() + 1)
		}

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if c.Bool("syslog") {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true, DisableColors: true})
			hook, err := logrus_syslog.NewSyslogHook("", "", syslog.LOG_DAEMON|syslog.LOG_INFO, "go-dnsmasq")
			if err != nil {
				log.Error("Unable to connect to local syslog daemon")
			} else {
				log.AddHook(hook)
			}
		} else {
			log.SetFormatter(&log.TextFormatter{DisableTimestamp: true})
		}

		if ns := c.String("nameservers"); ns != "" {
			for _, hostPort := range strings.Split(ns, ",") {
				if !strings.Contains(hostPort, ":") {
					hostPort += ":53"
				}

				if err := validateHostPort(hostPort); err != nil {
					log.Fatalf("This nameserver is invalid: %s", err)
				}

				nameservers = append(nameservers, hostPort)
			}
		}

		if sd := c.String("search-domains"); sd != "" {
			for _, domain := range strings.Split(sd, ",") {

				if dns.CountLabel(domain) < 2 {
					log.Fatalf("This search domain is not a FQDN: %s", domain)
				}
				domain = dns.Fqdn(strings.ToLower(domain))
				searchDomains = append(searchDomains, domain)
			}
		}

		if listen = c.String("listen"); !strings.Contains(listen, ":") {
			listen += ":53"
		}

		if err := validateHostPort(listen); err != nil {
			log.Fatalf("The listen address is invalid: %s", err)
		}

		config := &server.Config{
			DnsAddr:         listen,
			DefaultResolver: c.Bool("default-resolver"),
			Nameservers:     nameservers,
			Systemd:         c.Bool("systemd"),
			SearchDomains:   searchDomains,
			AppendDomain:    c.Bool("append-search-domains"),
			Hostsfile:       c.String("hostsfile"),
			PollInterval:    c.Int("hostsfile-poll"),
			RoundRobin:      c.Bool("round-robin"),
			NoRec:           c.Bool("no-rec"),
			ReadTimeout:     0,
			RCache:          c.Int("rcache"),
			RCacheTtl:       c.Int("rcache-ttl"),
			Verbose:         c.Bool("verbose"),
		}

		if err := server.SetDefaults(config); err != nil {
			if !config.NoRec && len(config.Nameservers) == 0 {
				log.Fatalf("No nameservers found in resolv.conf and --nameservers option not supplied: %s", err)
			} else if config.AppendDomain && len(config.SearchDomains) == 0 {
				log.Fatalf("No search domains found in resolv.conf and --search-domains option not supplied: %s", err)
			} else {
				log.Warnf("Error parsing resolv.conf: %s", err)
			}
		}

		if stubzones = c.String("stubzones"); stubzones != "" {
			stubmap := make(map[string][]string)
			segments := strings.Split(stubzones, "/")
			if len(segments) != 2 || len(segments[0]) == 0 || len(segments[1]) == 0 {
				log.Fatalf("The --stubzones argument is invalid")
			}

			hostPort = segments[1]
			if !strings.Contains(hostPort, ":") {
				hostPort += ":53"
			}

			if err := validateHostPort(hostPort); err != nil {
				log.Fatalf("This stubzones server address invalid: %s", err)
			}

			for _, sdomain := range strings.Split(segments[0], ",") {
				if dns.CountLabel(sdomain) < 2 {
					log.Fatalf("This stubzones domain is not a FQDN: %s", sdomain)
				}
				sdomain = dns.Fqdn(sdomain)
				stubmap[sdomain] = append(stubmap[sdomain], hostPort)
			}

			config.Stub = &stubmap
		}

		log.Infof("Starting go-dnsmasq server %s ...", Version)
		if config.AppendDomain {
			log.Infof("Search domains in use: %v", config.SearchDomains)
		}

		hf, err := hosts.NewHostsfile(config.Hostsfile, &hosts.Config{
			Poll:    config.PollInterval,
			Verbose: config.Verbose,
		})
		if err != nil {
			log.Fatalf("Error loading hostsfile: %s", err)
		}

		s := server.New(hf, config, Version)

		defer s.Stop()

		stats.Collect()

		if config.DefaultResolver {
			address, _, _ := net.SplitHostPort(config.DnsAddr)
			err := resolvconf.StoreAddress(address)
			if err != nil {
				log.Warnf("Failed to register as default nameserver: %s", err)
			}
			defer resolvconf.Clean()
		}

		go func() {
			if err := s.Run(); err != nil {
				exitReason <- err
			}
		}()

		exitErr = <-exitReason
		if exitErr != nil {
			log.Fatalf("Server error: %s", err)
		}
	}

	app.Run(os.Args)
}

func validateHostPort(hostPort string) error {
	host, port, err := net.SplitHostPort(hostPort)
	if err != nil {
		return err
	}
	if ip := net.ParseIP(host); ip == nil {
		return fmt.Errorf("Bad IP address: %s", host)
	}

	if p, _ := strconv.Atoi(port); p < 1 || p > 65535 {
		return fmt.Errorf("Bad port number %s", port)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/ssm"
	"github.com/jamietsao/aws-ssm-env/fetch"
)

var (
	paths []string
	tags  []string
	debug bool
)

func main() {

	// initialize command line flags
	initFlags()

	// fetch parameters
	start := time.Now()
	fetcher := fetch.NewFetcher(os.Getenv("SSM_REGION"), debug)
	params, err := fetcher.FetchParams(paths, tags)
	if err != nil {
		panic(err)
	}
	elapsed := time.Since(start)

	debugf("Params retrieved in %s\n", elapsed)

	// print as env variables
	printParams(params)
}

func initFlags() {
	pathsFlag := flag.String("paths", "", "comma delimited string of parameter path hierarchies (optional)")
	tagsFlag := flag.String("tags", "", "comma delimited string of tags to filter by (optional)")
	flag.BoolVar(&debug, "debug", false, "Enables debug logging when set to true")
	flag.Parse()

	if *pathsFlag == "" && *tagsFlag == "" {
		fmt.Print("Flag required: Either --paths or --tags is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	initPaths(pathsFlag)
	initTags(tagsFlag)
}

func initPaths(pathsFlag *string) {
	if *pathsFlag != "" {
		paths = strings.Split(*pathsFlag, ",")
	}

	// ensure only path hierarchies were given
	for _, path := range paths {
		if !strings.Contains(path, "/") {
			fmt.Printf("Invalid path: '%s' - only path hierarchies are supported (e.g. '/production/webapp/')\n", path)
			os.Exit(1)
		}
	}
}

func initTags(tagsFlag *string) {
	if *tagsFlag != "" {
		tags = strings.Split(*tagsFlag, ",")
	}
}

func printParams(params []*ssm.Parameter) {
	for _, param := range params {
		split := strings.Split(*param.Name, "/")
		name := split[len(split)-1]
		fmt.Printf("%s=%s\n", strings.ToUpper(name), *param.Value)
	}
}

func debugf(format string, a ...interface{}) {
	if debug {
		fmt.Printf("DEBUG -- %s", fmt.Sprintf(format, a...))
	}
}
<commit_msg>PR suggestion<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/ssm"
	"github.com/jamietsao/aws-ssm-env/fetch"
)

var (
	paths []string
	tags  []string
	debug bool
)

func main() {

	// initialize command line flags
	initFlags()

	// fetch parameters
	start := time.Now()
	fetcher := fetch.NewFetcher(os.Getenv("SSM_REGION"), debug)
	params, err := fetcher.FetchParams(paths, tags)
	if err != nil {
		panic(err)
	}
	elapsed := time.Since(start)

	debugf("Params retrieved in %s\n", elapsed)

	// print as env variables
	printParams(params)
}

func initFlags() {
	pathsFlag := flag.String("paths", "", "comma delimited string of parameter path hierarchies (optional)")
	tagsFlag := flag.String("tags", "", "comma delimited string of tags to filter by (optional)")
	flag.BoolVar(&debug, "debug", false, "Enables debug logging when set to true")
	flag.Parse()

	if *pathsFlag == "" && *tagsFlag == "" {
		fmt.Print("Flag required: Either --paths or --tags is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	initPaths(pathsFlag)
	initTags(tagsFlag)
}

func initPaths(pathsFlag *string) {
	if *pathsFlag != "" {
		paths = strings.Split(*pathsFlag, ",")
	}

	// ensure only path hierarchies were given
	for _, path := range paths {
		if !strings.Contains(path, "/") {
			fmt.Printf("Invalid path: '%s' - only path hierarchies are supported (e.g. '/production/webapp/')\n", path)
			os.Exit(1)
		}
	}
}

func initTags(tagsFlag *string) {
	if *tagsFlag != "" {
		tags = strings.Split(*tagsFlag, ",")
	}
}

func printParams(params []*ssm.Parameter) {
	for _, param := range params {
		split := strings.Split(*param.Name, "/")
		name := split[len(split)-1]
		fmt.Printf("%s=%s\n", strings.ToUpper(name), *param.Value)
	}
}

func debugf(format string, a ...interface{}) {
	if debug {
		fmt.Printf("DEBUG -- "+format, a...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"os"

	"io/ioutil"

	"fmt"
	"j2es/fetcher"
	"j2es/logger"
	"j2es/uploader"
	"strings"

	"github.com/uber-go/zap"
)

type config struct {
	EsHosts []string `json:"es_hosts"`
	Names   []string `json:"services"`
	Path    string   `json:"record_path"`
}

var (
	configInfo config
	records    map[string]string
)

func makeConfig(path string) error {
	// read config file to configInfo.
	if b, err := ioutil.ReadFile(path); err != nil {
		return fmt.Errorf("Error read config file: %s, %v", path, err)
	} else if err := json.Unmarshal(b, &configInfo); err != nil {
		return err
	}

	if len(configInfo.EsHosts) == 0 {
		return errors.New("\"es_hosts\" field is not existed")
	}

	if len(configInfo.Names) == 0 {
		return errors.New("\"services\" field is not existed")
	}

	if len(configInfo.Path) == 0 {
		return errors.New("\"record_path\" field is not existed")
	}

	// read records file to services.
	if b, err := ioutil.ReadFile(configInfo.Path); err == nil {
		if err := json.Unmarshal(b, &records); err != nil {
			return err
		}
	} else {
		// not exist
		records = make(map[string]string, 0)
	}

	return nil
}

func writeRecords(records map[string]string) error {
	b, err := json.Marshal(records)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(configInfo.Path, b, 0644); err != nil {
		return err
	}
	return nil
}

func getCommand(serviceName, cursor string) (string, []string) {
	var args []string
	args = append(args, "-u")
	args = append(args, serviceName)
	args = append(args, "-o")
	args = append(args, "json")

	if len(cursor) != 0 {
		args = append(args, fmt.Sprintf("--after-cursor=%s", cursor))
	}
	return "journalctl", args
}

func handleOne(serviceName string) error {
	cmd, args := getCommand(serviceName, records[serviceName])

	logger.Log.Debug(serviceName, zap.String("cmd", strings.Join(args, " ")))
	results, err := fetcher.GetMessages(serviceName, cmd, args...)
	l := len(results)
	if err != nil {
		return err
	} else if l == 0 {
		logger.Log.Info("", zap.String("args", strings.Join(args, " ")), zap.Int("count", l))
		return nil
	}

	for _, one := range results {
		uploader.AddRecord(serviceName, one.Message)
	}

	if err := uploader.PostToES(); err != nil {
		return err
	}

	records[serviceName] = results[l-1].Cursor
	if err := writeRecords(records); err != nil {
		return err
	}
	logger.Log.Info("", zap.String("args", strings.Join(args, " ")), zap.Int("count", l))

	return nil
}

func main() {
	var configPath string
	flag.StringVar(&configPath, "path", "", "The config file path.")
	flag.Parse()

	if len(configPath) == 0 {
		logger.Log.Error("path is empty.")
		os.Exit(1)
	}

	if err := makeConfig(configPath); err != nil {
		logger.Log.Error(err.Error())
		os.Exit(1)
	}

	if err := uploader.PrepareUploader(configInfo.EsHosts); err != nil {
		logger.Log.Error(err.Error())
		os.Exit(1)
	}

	exCode := 0
	for _, one := range configInfo.Names {
		if err := handleOne(one); err != nil {
			logger.Log.Error(err.Error())
			exCode = 1
		}
	}
	os.Exit(exCode)
}
<commit_msg>Add msg field with service name.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"os"

	"io/ioutil"

	"fmt"
	"j2es/fetcher"
	"j2es/logger"
	"j2es/uploader"
	"strings"

	"github.com/uber-go/zap"
)

type config struct {
	EsHosts []string `json:"es_hosts"`
	Names   []string `json:"services"`
	Path    string   `json:"record_path"`
}

var (
	configInfo config
	records    map[string]string
)

func makeConfig(path string) error {
	// read config file to configInfo.
	if b, err := ioutil.ReadFile(path); err != nil {
		return fmt.Errorf("Error read config file: %s, %v", path, err)
	} else if err := json.Unmarshal(b, &configInfo); err != nil {
		return err
	}

	if len(configInfo.EsHosts) == 0 {
		return errors.New("\"es_hosts\" field is not existed")
	}

	if len(configInfo.Names) == 0 {
		return errors.New("\"services\" field is not existed")
	}

	if len(configInfo.Path) == 0 {
		return errors.New("\"record_path\" field is not existed")
	}

	// read records file to services.
	if b, err := ioutil.ReadFile(configInfo.Path); err == nil {
		if err := json.Unmarshal(b, &records); err != nil {
			return err
		}
	} else {
		// not exist
		records = make(map[string]string, 0)
	}

	return nil
}

func writeRecords(records map[string]string) error {
	b, err := json.Marshal(records)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(configInfo.Path, b, 0644); err != nil {
		return err
	}
	return nil
}

func getCommand(serviceName, cursor string) (string, []string) {
	var args []string
	args = append(args, "-u")
	args = append(args, serviceName)
	args = append(args, "-o")
	args = append(args, "json")

	if len(cursor) != 0 {
		args = append(args, fmt.Sprintf("--after-cursor=%s", cursor))
	}
	return "journalctl", args
}

func handleOne(serviceName string) error {
	cmd, args := getCommand(serviceName, records[serviceName])

	logger.Log.Debug(serviceName, zap.String("cmd", strings.Join(args, " ")))
	results, err := fetcher.GetMessages(serviceName, cmd, args...)
	l := len(results)
	if err != nil {
		return err
	} else if l == 0 {
		logger.Log.Info(serviceName, zap.String("args", strings.Join(args, " ")), zap.Int("count", l))
		return nil
	}

	for _, one := range results {
		uploader.AddRecord(serviceName, one.Message)
	}

	if err := uploader.PostToES(); err != nil {
		return err
	}

	records[serviceName] = results[l-1].Cursor
	if err := writeRecords(records); err != nil {
		return err
	}
	logger.Log.Info(serviceName, zap.String("args", strings.Join(args, " ")), zap.Int("count", l))

	return nil
}

func main() {
	var configPath string
	flag.StringVar(&configPath, "path", "", "The config file path.")
	flag.Parse()

	if len(configPath) == 0 {
		logger.Log.Error("path is empty.")
		os.Exit(1)
	}

	if err := makeConfig(configPath); err != nil {
		logger.Log.Error(err.Error())
		os.Exit(1)
	}

	if err := uploader.PrepareUploader(configInfo.EsHosts); err != nil {
		logger.Log.Error(err.Error())
		os.Exit(1)
	}

	exCode := 0
	for _, one := range configInfo.Names {
		if err := handleOne(one); err != nil {
			logger.Log.Error(err.Error())
			exCode = 1
		}
	}
	os.Exit(exCode)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"html/template"
	"net/http"
	"net/url"

	"github.com/jtacoma/uritemplates"
)

var (
	homeTemplate = template.Must(template.New("home").Parse(`

				<title>don</title>

					home

				<form id="find-feed" method="get" action="/find-feed">
					user
					<input id="user" name="user" placeholder="e.g. your-username@mastodon.social" />
					<input type="submit" />
				</form>

	`))

	feedTemplate = template.Must(template.New("feed").Parse(`
		{{$feed := .Feed}}

				<title>{{$feed.Author}} @ don</title>

					home

{{$feed.Author}}

				{{range $entry := $feed.Entry}}
					{{if $object := $entry.Object}}

{{$object.Published}} (shared from {{$object.Author}} @ {{$entry.Published}})

						{{if $content := $object.Content}}

{{$content.HTML}}

						{{end}}
					{{else if $content := $entry.Content}}

{{$entry.Published}}

{{$content.HTML}}

					{{else if eq $entry.Verb "http://activitystrea.ms/schema/1.0/delete"}}

{{$entry.Published}}

						<s>(deleted @ {{$entry.Updated}})</s>
					{{end}}

				{{end}}

				{{if $link := $feed.GetLink "next"}}
					earlier
				{{end}}

	`))
)

var (
	addr = flag.String("addr", ":3000", "Address to listen on.")
)

func main() {
	flag.Parse()

	http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Set("content-type", "text/html")
		if err := homeTemplate.Execute(rw, nil); err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	http.HandleFunc("/find-feed", func(rw http.ResponseWriter, r *http.Request) {
		user := r.URL.Query().Get("user")

		acct, err := AcctFromString(user)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusBadRequest)
			return
		}

		hm, err := HostMetaFetch(acct.Host)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		lrdd := hm.GetLink("lrdd")
		if lrdd == nil {
			http.Error(rw, "no lrdd link found in host metadata", http.StatusInternalServerError)
			return
		}

		var lrddHref string
		switch {
		case lrdd.Href != "":
			lrddHref = lrdd.Href
		case lrdd.Template != "":
			lrddHrefTemplate, err := uritemplates.Parse(lrdd.Template)
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			s, err := lrddHrefTemplate.Expand(map[string]interface{}{"uri": acct.String()})
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			lrddHref = s
		}

		wf, err := WebfingerFetch(lrddHref)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		feedLink := wf.GetLink("http://schemas.google.com/g/2010#updates-from")
		if feedLink == nil {
			http.Error(rw, "no feed link found in webfinger response", http.StatusInternalServerError)
			return
		}

		rw.Header().Set("location", "/show-feed?"+url.Values{"url": []string{feedLink.Href}}.Encode())
		rw.WriteHeader(http.StatusSeeOther)
	})

	http.HandleFunc("/show-feed", func(rw http.ResponseWriter, r *http.Request) {
		feed, err := AtomFetch(r.URL.Query().Get("url"))
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		rw.Header().Set("content-type", "text/html")
		rw.WriteHeader(http.StatusOK)

		if err := feedTemplate.Execute(rw, map[string]interface{}{"Feed": feed}); err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	if err := http.ListenAndServe(*addr, nil); err != nil {
		panic(err)
	}
}
<commit_msg>make placeholder text clearer<commit_after>package main

import (
	"flag"
	"html/template"
	"net/http"
	"net/url"

	"github.com/jtacoma/uritemplates"
)

var (
	homeTemplate = template.Must(template.New("home").Parse(`

				<title>don</title>

					home

				<form id="find-feed" method="get" action="/find-feed">
					user
					<input id="user" name="user" placeholder="e.g. acct:your-username@mastodon.social" />
					<input type="submit" />
				</form>

	`))

	feedTemplate = template.Must(template.New("feed").Parse(`
		{{$feed := .Feed}}

				<title>{{$feed.Author}} @ don</title>

					home

{{$feed.Author}}

				{{range $entry := $feed.Entry}}
					{{if $object := $entry.Object}}

{{$object.Published}} (shared from {{$object.Author}} @ {{$entry.Published}})

						{{if $content := $object.Content}}

{{$content.HTML}}

						{{end}}
					{{else if $content := $entry.Content}}

{{$entry.Published}}

{{$content.HTML}}

					{{else if eq $entry.Verb "http://activitystrea.ms/schema/1.0/delete"}}

{{$entry.Published}}

						<s>(deleted @ {{$entry.Updated}})</s>
					{{end}}

				{{end}}

				{{if $link := $feed.GetLink "next"}}
					earlier
				{{end}}

	`))
)

var (
	addr = flag.String("addr", ":3000", "Address to listen on.")
)

func main() {
	flag.Parse()

	http.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
		rw.Header().Set("content-type", "text/html")
		if err := homeTemplate.Execute(rw, nil); err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	http.HandleFunc("/find-feed", func(rw http.ResponseWriter, r *http.Request) {
		user := r.URL.Query().Get("user")

		acct, err := AcctFromString(user)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusBadRequest)
			return
		}

		hm, err := HostMetaFetch(acct.Host)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		lrdd := hm.GetLink("lrdd")
		if lrdd == nil {
			http.Error(rw, "no lrdd link found in host metadata", http.StatusInternalServerError)
			return
		}

		var lrddHref string
		switch {
		case lrdd.Href != "":
			lrddHref = lrdd.Href
		case lrdd.Template != "":
			lrddHrefTemplate, err := uritemplates.Parse(lrdd.Template)
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			s, err := lrddHrefTemplate.Expand(map[string]interface{}{"uri": acct.String()})
			if err != nil {
				http.Error(rw, err.Error(), http.StatusInternalServerError)
				return
			}

			lrddHref = s
		}

		wf, err := WebfingerFetch(lrddHref)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		feedLink := wf.GetLink("http://schemas.google.com/g/2010#updates-from")
		if feedLink == nil {
			http.Error(rw, "no feed link found in webfinger response", http.StatusInternalServerError)
			return
		}

		rw.Header().Set("location", "/show-feed?"+url.Values{"url": []string{feedLink.Href}}.Encode())
		rw.WriteHeader(http.StatusSeeOther)
	})

	http.HandleFunc("/show-feed", func(rw http.ResponseWriter, r *http.Request) {
		feed, err := AtomFetch(r.URL.Query().Get("url"))
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		rw.Header().Set("content-type", "text/html")
		rw.WriteHeader(http.StatusOK)

		if err := feedTemplate.Execute(rw, map[string]interface{}{"Feed": feed}); err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}
	})

	if err := http.ListenAndServe(*addr, nil); err != nil {
		panic(err)
	}
}
<|endoftext|>