<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"cloud.google.com/go/storage"
	"github.com/Sirupsen/logrus"
	units "github.com/docker/go-units"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"golang.org/x/net/context"
)

var (
	provider string
	bucket   string

	s3AccessKey string
	s3SecretKey string
	s3Region    string

	port     string
	certFile string
	keyFile  string
)

func init() {
	flag.StringVar(&provider, "provider", "s3", "cloud provider (ex. s3, gcs)")
	flag.StringVar(&bucket, "bucket", "", "bucket path from which to serve files")

	flag.StringVar(&s3AccessKey, "s3key", "", "s3 access key")
	flag.StringVar(&s3SecretKey, "s3secret", "", "s3 access secret")
	flag.StringVar(&s3Region, "s3region", "us-west-2", "aws region for the bucket")

	flag.StringVar(&port, "p", "8080", "port for server to run on")

	flag.StringVar(&certFile, "cert", "", "path to ssl certificate")
	flag.StringVar(&keyFile, "key", "", "path to ssl key")

	flag.Parse()

	if provider != "s3" && provider != "gcs" {
		logrus.Fatalf("%s is not a valid provider, try `s3` or `gcs`.", provider)
	}
}

func main() {
	// create a new provider
	p, err := newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey)
	if err != nil {
		logrus.Fatalf("Creating new provider failed: %v", err)
	}

	// get the files
	max := 2000
	q := &storage.Query{
		Prefix:     p.Prefix(),
		MaxResults: max,
	}
	files, err := p.List(p.Prefix(), p.Prefix(), "", max, q)
	if err != nil {
		logrus.Fatalf("Listing all files in bucket failed: %v", err)
	}

	// create mux server
	mux := http.NewServeMux()

	// static files handler
	staticHandler := http.StripPrefix("/static/", http.FileServer(http.Dir("/src/static")))
	mux.Handle("/static/", staticHandler)

	// template handler
	h := handler{
		Files: files,
	}
	mux.HandleFunc("/", h.serveTemplate)

	// set up the server
	server := &http.Server{
		Addr:    ":" + port,
		Handler: mux,
	}
	logrus.Infof("Starting server on port %q", port)
	if certFile != "" && keyFile != "" {
		logrus.Fatal(server.ListenAndServeTLS(certFile, keyFile))
	} else {
		logrus.Fatal(server.ListenAndServe())
	}
}

type object struct {
	Name string
	Size int64
}

type s3Provider struct {
	bucket string
	prefix string
	client *s3.S3
	ctx    context.Context
	b      *s3.Bucket
}

type gcsProvider struct {
	bucket string
	prefix string
	client *storage.Client
	ctx    context.Context
	b      *storage.BucketHandle
}

type cloud interface {
	List(prefix, delimiter, marker string, max int, q *storage.Query) ([]object, error)
	Prefix() string
}

func newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey string) (cloud, error) {
	if provider == "s3" {
		// auth with aws
		auth, err := aws.GetAuth(s3AccessKey, s3SecretKey)
		if err != nil {
			return nil, err
		}

		// create the client
		region, err := getRegion(s3Region)
		if err != nil {
			return nil, err
		}

		p := s3Provider{bucket: bucket}
		p.client = s3.New(auth, region)
		bucket, p.prefix = cleanBucketName(p.bucket)
		p.b = p.client.Bucket(bucket)
		return &p, nil
	}

	p := gcsProvider{bucket: bucket}
	p.ctx = context.Background()
	client, err := storage.NewClient(p.ctx)
	if err != nil {
		return nil, err
	}
	p.client = client
	p.b = client.Bucket(bucket)
	_, p.prefix = cleanBucketName(p.bucket)
	return &p, nil
}

// List returns the files in an s3 bucket.
func (c *s3Provider) List(prefix, delimiter, marker string, max int, q *storage.Query) (files []object, err error) {
	resp, err := c.b.List(prefix, delimiter, marker, max)
	if err != nil {
		return nil, err
	}

	// append to files
	for _, f := range resp.Contents {
		files = append(files, object{
			Name: f.Key,
			Size: f.Size,
		})
	}

	// recursion for the recursion god
	if resp.IsTruncated && resp.NextMarker != "" {
		f, err := c.List(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, q)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}

// Prefix returns the prefix in an s3 bucket.
func (c *s3Provider) Prefix() string {
	return c.prefix
}

// List returns the files in an gcs bucket.
func (c *gcsProvider) List(prefix, delimiter, marker string, max int, q *storage.Query) (files []object, err error) {
	resp, err := c.b.List(c.ctx, q)
	if err != nil {
		return nil, err
	}

	// append to files
	for _, f := range resp.Results {
		files = append(files, object{
			Name: f.Name,
			Size: f.Size,
		})
	}

	// recursion for the recursion god
	if resp.Next != nil {
		f, err := c.List(prefix, delimiter, marker, max, resp.Next)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}

// Prefix returns the prefix in an gcs bucket.
func (c *gcsProvider) Prefix() string {
	return c.prefix
}

// cleanBucketName returns the bucket and prefix
// for a given s3bucket.
func cleanBucketName(bucket string) (string, string) {
	bucket = strings.TrimPrefix(bucket, "s3://")
	parts := strings.SplitN(bucket, "/", 2)
	if len(parts) == 1 {
		return bucket, "/"
	}

	return parts[0], parts[1]
}

// getRegion returns the aws region that is matches a given string.
func getRegion(name string) (aws.Region, error) {
	var regions = map[string]aws.Region{
		aws.APNortheast.Name:  aws.APNortheast,
		aws.APSoutheast.Name:  aws.APSoutheast,
		aws.APSoutheast2.Name: aws.APSoutheast2,
		aws.EUCentral.Name:    aws.EUCentral,
		aws.EUWest.Name:       aws.EUWest,
		aws.USEast.Name:       aws.USEast,
		aws.USWest.Name:       aws.USWest,
		aws.USWest2.Name:      aws.USWest2,
		aws.USGovWest.Name:    aws.USGovWest,
		aws.SAEast.Name:       aws.SAEast,
	}
	region, ok := regions[name]
	if !ok {
		return aws.Region{}, fmt.Errorf("No region matches %s", name)
	}
	return region, nil
}

// JSONResponse is a map[string]string
// response from the web server.
type JSONResponse map[string]string

// String returns the string representation of the
// JSONResponse object.
func (j JSONResponse) String() string {
	str, err := json.MarshalIndent(j, "", "  ")
	if err != nil {
		return fmt.Sprintf(`{
  "error": "%v"
}`, err)
	}

	return string(str)
}

// handler is the object which contains data to pass to the http handler functions.
type handler struct {
	Files []object
}

func (h *handler) serveTemplate(w http.ResponseWriter, r *http.Request) {
	templateDir := path.Join("/src", "templates")
	lp := path.Join(templateDir, "layout.html")

	// set up custom functions
	funcMap := template.FuncMap{
		"ext": func(name string) string {
			return strings.TrimPrefix(filepath.Ext(name), ".")
		},
		"base": func(name string) string {
			parts := strings.Split(name, "/")
			return parts[len(parts)-1]
		},
		"size": func(s int64) string {
			return units.HumanSize(float64(s))
		},
	}

	// parse & execute the template
	tmpl := template.Must(template.New("").Funcs(funcMap).ParseFiles(lp))
	if err := tmpl.ExecuteTemplate(w, "layout", h.Files); err != nil {
		writeError(w, fmt.Sprintf("Execute template failed: %v", err))
		return
	}
}

// writeError sends an error back to the requester
// and also logs the error.
func writeError(w http.ResponseWriter, msg string) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, JSONResponse{
		"error": msg,
	})
	logrus.Printf("writing error: %s", msg)
	return
}
<commit_msg>fix<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"cloud.google.com/go/storage"
	"github.com/Sirupsen/logrus"
	units "github.com/docker/go-units"
	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/s3"
	"golang.org/x/net/context"
)

var (
	provider string
	bucket   string

	s3AccessKey string
	s3SecretKey string
	s3Region    string

	port     string
	certFile string
	keyFile  string
)

func init() {
	flag.StringVar(&provider, "provider", "s3", "cloud provider (ex. s3, gcs)")
	flag.StringVar(&bucket, "bucket", "", "bucket path from which to serve files")

	flag.StringVar(&s3AccessKey, "s3key", "", "s3 access key")
	flag.StringVar(&s3SecretKey, "s3secret", "", "s3 access secret")
	flag.StringVar(&s3Region, "s3region", "us-west-2", "aws region for the bucket")

	flag.StringVar(&port, "p", "8080", "port for server to run on")

	flag.StringVar(&certFile, "cert", "", "path to ssl certificate")
	flag.StringVar(&keyFile, "key", "", "path to ssl key")

	flag.Parse()

	if provider != "s3" && provider != "gcs" {
		logrus.Fatalf("%s is not a valid provider, try `s3` or `gcs`.", provider)
	}
}

func main() {
	// create a new provider
	p, err := newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey)
	if err != nil {
		logrus.Fatalf("Creating new provider failed: %v", err)
	}

	// get the files
	max := 2000
	q := &storage.Query{
		Prefix:     p.Prefix(),
		MaxResults: max,
	}
	files, err := p.List(p.Prefix(), p.Prefix(), "", max, q)
	if err != nil {
		logrus.Fatalf("Listing all files in bucket failed: %v", err)
	}

	// create mux server
	mux := http.NewServeMux()

	// static files handler
	staticHandler := http.StripPrefix("/static/", http.FileServer(http.Dir("/src/static")))
	mux.Handle("/static/", staticHandler)

	// template handler
	h := handler{
		Files: files,
	}
	mux.HandleFunc("/", h.serveTemplate)

	// set up the server
	server := &http.Server{
		Addr:    ":" + port,
		Handler: mux,
	}
	logrus.Infof("Starting server on port %q", port)
	if certFile != "" && keyFile != "" {
		logrus.Fatal(server.ListenAndServeTLS(certFile, keyFile))
	} else {
		logrus.Fatal(server.ListenAndServe())
	}
}

type object struct {
	Name string
	Size int64
}

type s3Provider struct {
	bucket string
	prefix string
	client *s3.S3
	ctx    context.Context
	b      *s3.Bucket
}

type gcsProvider struct {
	bucket string
	prefix string
	client *storage.Client
	ctx    context.Context
	b      *storage.BucketHandle
}

type cloud interface {
	List(prefix, delimiter, marker string, max int, q *storage.Query) ([]object, error)
	Prefix() string
}

func newProvider(provider, bucket, s3Region, s3AccessKey, s3SecretKey string) (cloud, error) {
	if provider == "s3" {
		// auth with aws
		auth, err := aws.GetAuth(s3AccessKey, s3SecretKey)
		if err != nil {
			return nil, err
		}

		// create the client
		region, err := getRegion(s3Region)
		if err != nil {
			return nil, err
		}

		p := s3Provider{bucket: bucket}
		p.client = s3.New(auth, region)
		bucket, p.prefix = cleanBucketName(p.bucket)
		p.b = p.client.Bucket(bucket)
		return &p, nil
	}

	p := gcsProvider{bucket: bucket}
	p.ctx = context.Background()
	client, err := storage.NewClient(p.ctx)
	if err != nil {
		return nil, err
	}
	p.client = client
	p.b = client.Bucket(bucket)
	p.bucket, p.prefix = cleanBucketName(p.bucket)
	return &p, nil
}

// List returns the files in an s3 bucket.
func (c *s3Provider) List(prefix, delimiter, marker string, max int, q *storage.Query) (files []object, err error) {
	resp, err := c.b.List(prefix, delimiter, marker, max)
	if err != nil {
		return nil, err
	}

	// append to files
	for _, f := range resp.Contents {
		files = append(files, object{
			Name: f.Key,
			Size: f.Size,
		})
	}

	// recursion for the recursion god
	if resp.IsTruncated && resp.NextMarker != "" {
		f, err := c.List(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, q)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}

// Prefix returns the prefix in an s3 bucket.
func (c *s3Provider) Prefix() string {
	return c.prefix
}

// List returns the files in an gcs bucket.
func (c *gcsProvider) List(prefix, delimiter, marker string, max int, q *storage.Query) (files []object, err error) {
	resp, err := c.b.List(c.ctx, q)
	if err != nil {
		return nil, err
	}

	// append to files
	for _, f := range resp.Results {
		files = append(files, object{
			Name: f.Name,
			Size: f.Size,
		})
	}

	// recursion for the recursion god
	if resp.Next != nil {
		f, err := c.List(prefix, delimiter, marker, max, resp.Next)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}

// Prefix returns the prefix in an gcs bucket.
func (c *gcsProvider) Prefix() string {
	return c.prefix
}

// cleanBucketName returns the bucket and prefix
// for a given s3bucket.
func cleanBucketName(bucket string) (string, string) {
	bucket = strings.TrimPrefix(bucket, "s3://")
	bucket = strings.TrimPrefix(bucket, "gcs://")
	parts := strings.SplitN(bucket, "/", 2)
	if len(parts) == 1 {
		return bucket, "/"
	}

	return parts[0], parts[1]
}

// getRegion returns the aws region that is matches a given string.
func getRegion(name string) (aws.Region, error) {
	var regions = map[string]aws.Region{
		aws.APNortheast.Name:  aws.APNortheast,
		aws.APSoutheast.Name:  aws.APSoutheast,
		aws.APSoutheast2.Name: aws.APSoutheast2,
		aws.EUCentral.Name:    aws.EUCentral,
		aws.EUWest.Name:       aws.EUWest,
		aws.USEast.Name:       aws.USEast,
		aws.USWest.Name:       aws.USWest,
		aws.USWest2.Name:      aws.USWest2,
		aws.USGovWest.Name:    aws.USGovWest,
		aws.SAEast.Name:       aws.SAEast,
	}
	region, ok := regions[name]
	if !ok {
		return aws.Region{}, fmt.Errorf("No region matches %s", name)
	}
	return region, nil
}

// JSONResponse is a map[string]string
// response from the web server.
type JSONResponse map[string]string

// String returns the string representation of the
// JSONResponse object.
func (j JSONResponse) String() string {
	str, err := json.MarshalIndent(j, "", "  ")
	if err != nil {
		return fmt.Sprintf(`{
  "error": "%v"
}`, err)
	}

	return string(str)
}

// handler is the object which contains data to pass to the http handler functions.
type handler struct {
	Files []object
}

func (h *handler) serveTemplate(w http.ResponseWriter, r *http.Request) {
	templateDir := path.Join("/src", "templates")
	lp := path.Join(templateDir, "layout.html")

	// set up custom functions
	funcMap := template.FuncMap{
		"ext": func(name string) string {
			return strings.TrimPrefix(filepath.Ext(name), ".")
		},
		"base": func(name string) string {
			parts := strings.Split(name, "/")
			return parts[len(parts)-1]
		},
		"size": func(s int64) string {
			return units.HumanSize(float64(s))
		},
	}

	// parse & execute the template
	tmpl := template.Must(template.New("").Funcs(funcMap).ParseFiles(lp))
	if err := tmpl.ExecuteTemplate(w, "layout", h.Files); err != nil {
		writeError(w, fmt.Sprintf("Execute template failed: %v", err))
		return
	}
}

// writeError sends an error back to the requester
// and also logs the error.
func writeError(w http.ResponseWriter, msg string) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprint(w, JSONResponse{
		"error": msg,
	})
	logrus.Printf("writing error: %s", msg)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
<<<<<<< HEAD
	"flag"
=======
>>>>>>> upstream/master
	"sync"

	irc "github.com/fluffle/goirc/client"
)

type Bot struct {
	Nickname      string   `json:"nickname"`
	Username      string   `json:"username"`
	RealName      string   `json:"real_name"`
	NickservPass  string   `json:"nickserv_pass"`
	ServerName    string   `json:"server_name"`
	Port          int      `json:"port"`
	TLS           bool     `json:"tls"`
	Channels      []string `json:"channels"`
	Ignored       []string `json:"ignored"`
	Moderators    []string `json:"moderators"`
	MaxLineLength int      `json:"max_line_length"`
	Chance        int      `json:"chance"`
	Posting       bool     `json:"posting"`
	Trigger       string   `json:"trigger"`
	MinGap        int64    `json:"min_gap_seconds"`
	DBName        string   `json:"db_name"`
	DBUser        string   `json:"db_username"`
	DBPassword    string   `json:"db_password"`
	Training      bool     `json:"training"`
	TrainFile     string   `json:"trainfile"`
	DB            *sql.DB
	Conn          *irc.Conn
	LastTime      int64
	Mutex         sync.RWMutex
}

func main() {
	var bot Bot

<<<<<<< HEAD
	var configFilePath = flag.String("configFilePath", "timhortons.json", "The path to the JSON config file.")
	flag.Parse()

	bot.loadConfiguration(*configFilePath)
=======
	configFilePath := "./timhortons.json"
	bot.loadConfiguration(configFilePath)
>>>>>>> upstream/master
	bot.dialDB()
	if bot.Training {
		bot.processTrainingFile()
	}

	bot.dial()
	bot.run()

	select {}
}
<commit_msg>more conflict resolution<commit_after>package main

import (
	"database/sql"
	"flag"
	"sync"

	irc "github.com/fluffle/goirc/client"
)

type Bot struct {
	Nickname      string   `json:"nickname"`
	Username      string   `json:"username"`
	RealName      string   `json:"real_name"`
	NickservPass  string   `json:"nickserv_pass"`
	ServerName    string   `json:"server_name"`
	Port          int      `json:"port"`
	TLS           bool     `json:"tls"`
	Channels      []string `json:"channels"`
	Ignored       []string `json:"ignored"`
	Moderators    []string `json:"moderators"`
	MaxLineLength int      `json:"max_line_length"`
	Chance        int      `json:"chance"`
	Posting       bool     `json:"posting"`
	Trigger       string   `json:"trigger"`
	MinGap        int64    `json:"min_gap_seconds"`
	DBName        string   `json:"db_name"`
	DBUser        string   `json:"db_username"`
	DBPassword    string   `json:"db_password"`
	Training      bool     `json:"training"`
	TrainFile     string   `json:"trainfile"`
	DB            *sql.DB
	Conn          *irc.Conn
	LastTime      int64
	Mutex         sync.RWMutex
}

func main() {
	var bot Bot

	var configFilePath = flag.String("configFilePath", "timhortons.json", "The path to the JSON config file.")
	flag.Parse()

	bot.loadConfiguration(*configFilePath)
	bot.dialDB()
	if bot.Training {
		bot.processTrainingFile()
	}

	bot.dial()
	bot.run()

	select {}
}
<|endoftext|>


<commit_before>// Copyright © 2016 Sidharth Kshatriya
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/sidkshatriya/dontbug/cmd"
	"log"
)

func main() {
	log.SetFlags(log.Lshortfile)
	log.SetPrefix("dontbug: ")
	cmd.Execute()
}
<commit_msg>Add a light red background for the dontbug prefix for fatal errors<commit_after>// Copyright © 2016 Sidharth Kshatriya
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"github.com/sidkshatriya/dontbug/cmd"
	"log"
)

func main() {
	log.SetFlags(log.Lshortfile)
	// Light red background
	log.SetPrefix("\x1b[101mdontbug fatal error:\x1b[0m ")
	cmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"

	_ "image/gif"
	_ "image/png"
)

var errHomeNotFound = errors.New("wimg: user home directory not found")

func remove(r rune) bool {
	return unicode.Is(unicode.Mn, r)
}

func normalize(name string) string {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(remove), norm.NFC)
	name = strings.TrimSpace(name)
	name, _, err := transform.String(t, name)
	if err != nil {
		panic(err)
	}
	name = strings.ToLower(name)
	name = strings.Replace(name, " ", "_", -1)
	return name
}

func run(src, name string, quality int) error {
	home := os.Getenv("HOME")
	if home == "" {
		home = os.Getenv("USERPROFILE")
	}
	if home == "" {
		return errHomeNotFound
	}
	resp, err := http.Get(src)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	m, _, err := image.Decode(resp.Body)
	if err != nil {
		return err
	}
	bounds := m.Bounds()
	if name == "" {
		ext := path.Ext(src)
		name = path.Base(src)
		name = strings.TrimSuffix(name, ext)
	}
	name = normalize(name)
	name = fmt.Sprintf("%dx%d_%s.jpg", bounds.Dx(), bounds.Dy(), name)
	filename := filepath.Join(home, "img", name)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, m, nil)
}

func main() {
	var (
		name    = flag.String("name", "", "name of the image")
		quality = flag.Int("quality", 100, "jpeg quality encoding")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "wimg: invalid arguments")
		flag.PrintDefaults()
		os.Exit(1)
	}
	err := run(args[0], *name, *quality)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Handle normalization error even though it will likely never occur.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"

	_ "image/gif"
	_ "image/png"
)

var errHomeNotFound = errors.New("wimg: user home directory not found")

func remove(r rune) bool {
	return unicode.Is(unicode.Mn, r)
}

func normalize(name string) (string, error) {
	t := transform.Chain(norm.NFD, transform.RemoveFunc(remove), norm.NFC)
	name = strings.TrimSpace(name)
	name, _, err := transform.String(t, name)
	if err != nil {
		return "", err
	}
	name = strings.ToLower(name)
	name = strings.Replace(name, " ", "_", -1)
	return name, nil
}

func run(src, name string, quality int) error {
	home := os.Getenv("HOME")
	if home == "" {
		home = os.Getenv("USERPROFILE")
	}
	if home == "" {
		return errHomeNotFound
	}
	resp, err := http.Get(src)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	m, _, err := image.Decode(resp.Body)
	if err != nil {
		return err
	}
	bounds := m.Bounds()
	if name == "" {
		ext := path.Ext(src)
		name = path.Base(src)
		name = strings.TrimSuffix(name, ext)
	}
	name, err = normalize(name)
	if err != nil {
		return err
	}
	name = fmt.Sprintf("%dx%d_%s.jpg", bounds.Dx(), bounds.Dy(), name)
	filename := filepath.Join(home, "img", name)
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return jpeg.Encode(f, m, nil)
}

func main() {
	var (
		name    = flag.String("name", "", "name of the image")
		quality = flag.Int("quality", 100, "jpeg quality encoding")
	)
	flag.Parse()
	args := flag.Args()
	if len(args) != 1 {
		fmt.Fprintln(os.Stderr, "wimg: invalid arguments")
		flag.PrintDefaults()
		os.Exit(1)
	}
	err := run(args[0], *name, *quality)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Number One rule: it's compiling it's working, no tests needed
// Number Two: it's freaking imperative
// Copyright AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

// +build ignore

package main

import (
  ui "github.com/gizak/termui"
  "time"
  "strconv"
  "net/http"
  "io/ioutil"
  "encoding/json"
  "strings"
  "sort"
  "fmt"
  "math"
  "flag"
  "log"
  "errors"
)

type LangStatistic struct {
  Level string `json:"level"`
  Points float64 `json:"points"`
}

type Language struct {
  Name string
  Points float64
  Level int
  Percent float64
}

type PlatformStatistic struct {
  PercentWork float64 `json:"percent_work"`
  Points float64 `json:"points"`
  Time float64 `json:"time"`
}

type Platform struct {
  Name string
  PercentWork float64
  Points float64
  Time float64
}

type UserStatistic struct {
  CurrentLanguage string `json:"current_language"`
  FocusLevel string `json:"focus_level"`
  FocusPoints float64 `json:"focus_points"`
  Level string `json:"level"`
  MaxStreak float64 `json:"max_streak"`
  Name string `json:"name"`
  ProgrammingNow bool `json:"programming_now"`
  StreakingNow bool `json:"streaking_now"`
  TimeSpent float64 `json:"time_spent"`
  TotalDaysCoded float64 `json:"total_days_coded"`
  TotalFlowStates float64 `json:"total_flow_states"`
  Platforms map[string]PlatformStatistic `json:"platforms"`
  Languages map[string]LangStatistic `json:"languages"`
}

type LanguageByLevel []Language
func (v LanguageByLevel) Len() int { return len(v) }
func (v LanguageByLevel) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v LanguageByLevel) Less(i, j int) bool {
  if v[i].Level == v[j].Level {
    if v[i].Points == v[j].Points {
      return v[i].Name > v[j].Name
    } else {
      return v[i].Points > v[j].Points
    }

  } else {
    return v[i].Level > v[j].Level
  }
}

type PlatformByPoints []Platform
func (v PlatformByPoints) Len() int { return len(v) }
func (v PlatformByPoints) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v PlatformByPoints) Less(i, j int) bool {
  if v[i].Points == v[j].Points {
    if v[i].Time == v[j].Time {
      return v[i].Name > v[j].Name
    } else {
      return v[i].Time > v[j].Time
    }
  } else {
    return v[i].Points > v[j].Points
  }
}

type Level struct {
  Number int
  Percent float64
}

func ParseLevel(strLevel string) (Level, error) {
  splited := strings.Split(strLevel, ".")
  level, err := strconv.Atoi(splited[0])
  if err != nil {
    return Level{}, err
  }
  percent, err := strconv.ParseFloat(splited[1], 64)
  if err != nil {
    return Level{}, err
  }
  return Level{level, percent}, nil
}

func HandleMeLikeOneOfYourFrenchGirls(err error) {
  parUser := ui.NewPar(err.Error())
  parUser.Height = 3
  parUser.Border.Label = "Erro acontece nada ocorre feijoada"

  ui.Body.AddRows(
      ui.NewRow(ui.NewCol(12, 0, parUser)))
}

func main() {
  // yeah, if there's no username you'll see my profile :3
  var username string
  var numberOfLanguages int
  flag.StringVar(&username, "username", "schleumer", "your codeivate username")
  flag.IntVar(&numberOfLanguages, "len", 10, "number of languages to display")
  flag.Parse()

  err := ui.Init()
  if err != nil {
    panic(err)
  }
  defer ui.Close()

  ui.UseTheme("helloworld")

  done := make(chan bool)
  redraw := make(chan bool)
  error := make(chan string)

  ui.Body.Align()

  var startPoints float64
  firstRound := true

  update := func () {
    for {
      // restart body
      ui.Body = ui.NewGrid()
      ui.Body.Width = ui.TermWidth()

      client := &http.Client{}
      req, err := http.NewRequest("GET", fmt.Sprintf("http://codeivate.com/users/%s.json", username), nil)
      req.Close = true
      req.Header.Set("Content-Type", "application/json")
      req.Header.Set("User-Agent", "NOTICE ME SENPAI v0.1a")

      resp, err := client.Do(req)

      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      body, err := ioutil.ReadAll(resp.Body)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      var statistic UserStatistic
      err = json.Unmarshal(body, &statistic)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(errors.New(fmt.Sprintf("Error on unmarshaling, probably the user %s doesn't exists", username)))
        redraw <- true
        time.Sleep(time.Second * 60)
        continue
      }

      userLevel, err := ParseLevel(statistic.Level)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      // i have no idea what i'm doing, but it's fucking hardcore
      hours := math.Floor(statistic.TimeSpent / 3600)
      minutes := math.Floor((statistic.TimeSpent - (hours * 3600)) / 60)
      var totalPoints float64
      var pointsSinceFirstRound float64

      for _, lang := range statistic.Languages {
        totalPoints += lang.Points
      }

      if firstRound {
        startPoints = totalPoints
      }

      pointsSinceFirstRound = totalPoints - startPoints

      parUser := ui.NewPar(
        fmt.Sprintf("Level: %d - Percent: %.0f\nTime: %.0f hours %.0f minutes - Current Language: %s\nPoints: %.0f(%.0f)",
          userLevel.Number,
          userLevel.Percent,
          hours,
          minutes,
          statistic.CurrentLanguage,
          totalPoints,
          pointsSinceFirstRound))

      parUser.Height = 5
      parUser.Border.Label = "User Info"

      ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, parUser)))

      var platformsContent []string

      var platforms []Platform

      for name, platform := range statistic.Platforms {
        platforms = append(platforms, Platform{name, platform.PercentWork, platform.Points, platform.Time})
      }

      sort.Sort(PlatformByPoints(platforms))

      for _, platform := range platforms {
        hours := math.Floor(platform.Time / 3600)
        minutes := math.Floor((platform.Time - (hours * 3600)) / 60)
        platformsContent = append(platformsContent, fmt.Sprintf("%s - Percent: %.2f - Points: %.2f - Time: %.0f hours %.0f minutes", platform.Name, platform.PercentWork, platform.Points, hours, minutes))
      }

      parWorkspace := ui.NewPar(strings.Join(platformsContent, "\n"))
      parWorkspace.Height = len(statistic.Platforms) + 2
      parWorkspace.Border.Label = "Workspace Info"

      ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, parWorkspace)))

      var languages []Language

      for name, lang := range statistic.Languages {
        level, err := ParseLevel(lang.Level)
        if err != nil {
          // don't give a fuck
          continue
        }
        languages = append(languages, Language{name, lang.Points, level.Number, level.Percent})
      }

      sort.Sort(LanguageByLevel(languages))

      for _, lang := range languages[:numberOfLanguages] {
        g := ui.NewGauge()
        g.Percent = int(lang.Percent)
        g.Height = 3
        g.Border.Label = fmt.Sprintf("%s Level: %d - Points: %.0f", lang.Name, lang.Level, lang.Points)
        g.BarColor = ui.ColorGreen

        ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, g)))
      }

      ui.Render(ui.Body)
      ui.Body.Align()

      redraw <- true
      time.Sleep(time.Second * 10)
    }
  }

  evt := ui.EventCh()

  ui.Render(ui.Body)
  go update()

  for {
    select {
      case e := <-evt:
        if e.Type == ui.EventKey && (e.Ch == 'q' || e.Ch == 'Q' /* HEHEHEHE */) {
          log.Print("Everything went better than expected")
          return
        }
        if e.Type == ui.EventResize {
          ui.Body.Width = ui.TermWidth()
          ui.Body.Align()
          go func() { redraw <- true }()
        }
      case <-done:
        log.Print("Everything went better than expected")
        return
      case e := <-error:
        log.Fatal(e)
        return
      case <-redraw:
        ui.Body.Align()
        ui.Render(ui.Body)
    }
  }
}<commit_msg>added even more crap code<commit_after>// Number One rule: it's compiling it's working, no tests needed
// Number Two: it's freaking imperative
// Copyright AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
// AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

// +build ignore

package main

import (
  ui "github.com/gizak/termui"
  "time"
  "strconv"
  "net/http"
  "io/ioutil"
  "encoding/json"
  "strings"
  "sort"
  "fmt"
  "math"
  "flag"
  "log"
  "errors"
)

type LangStatistic struct {
  Level string `json:"level"`
  Points float64 `json:"points"`
}

type Language struct {
  Name string
  Points float64
  Level int
  Percent float64
}

type PlatformStatistic struct {
  PercentWork float64 `json:"percent_work"`
  Points float64 `json:"points"`
  Time float64 `json:"time"`
}

type Platform struct {
  Name string
  PercentWork float64
  Points float64
  Time float64
}

type UserStatistic struct {
  CurrentLanguage string `json:"current_language"`
  FocusLevel string `json:"focus_level"`
  FocusPoints float64 `json:"focus_points"`
  Level string `json:"level"`
  MaxStreak float64 `json:"max_streak"`
  Name string `json:"name"`
  ProgrammingNow bool `json:"programming_now"`
  StreakingNow bool `json:"streaking_now"`
  TimeSpent float64 `json:"time_spent"`
  TotalDaysCoded float64 `json:"total_days_coded"`
  TotalFlowStates float64 `json:"total_flow_states"`
  Platforms map[string]PlatformStatistic `json:"platforms"`
  Languages map[string]LangStatistic `json:"languages"`
}

type LanguageByLevel []Language
func (v LanguageByLevel) Len() int { return len(v) }
func (v LanguageByLevel) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v LanguageByLevel) Less(i, j int) bool {
  if v[i].Level == v[j].Level {
    if v[i].Points == v[j].Points {
      return v[i].Name > v[j].Name
    } else {
      return v[i].Points > v[j].Points
    }

  } else {
    return v[i].Level > v[j].Level
  }
}

type PlatformByPoints []Platform
func (v PlatformByPoints) Len() int { return len(v) }
func (v PlatformByPoints) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
func (v PlatformByPoints) Less(i, j int) bool {
  if v[i].Points == v[j].Points {
    if v[i].Time == v[j].Time {
      return v[i].Name > v[j].Name
    } else {
      return v[i].Time > v[j].Time
    }
  } else {
    return v[i].Points > v[j].Points
  }
}

type Level struct {
  Number int
  Percent float64
}

func ParseLevel(strLevel string) (Level, error) {
  splited := strings.Split(strLevel, ".")
  level, err := strconv.Atoi(splited[0])
  if err != nil {
    return Level{}, err
  }
  percent, err := strconv.ParseFloat(splited[1], 64)
  if err != nil {
    return Level{}, err
  }
  return Level{level, percent}, nil
}

func HandleMeLikeOneOfYourFrenchGirls(err error) {
  parUser := ui.NewPar(err.Error())
  parUser.Height = 3
  parUser.Border.Label = "Erro acontece nada ocorre feijoada"

  ui.Body.AddRows(
      ui.NewRow(ui.NewCol(12, 0, parUser)))
}

func main() {
  // yeah, if there's no username you'll see my profile :3
  var username string
  var numberOfLanguages int
  flag.StringVar(&username, "username", "schleumer", "your codeivate username")
  flag.IntVar(&numberOfLanguages, "len", 10, "number of languages to display")
  flag.Parse()

  err := ui.Init()
  if err != nil {
    panic(err)
  }
  defer ui.Close()

  ui.UseTheme("helloworld")

  done := make(chan bool)
  redraw := make(chan bool)
  error := make(chan string)

  ui.Body.Align()

  var startPoints float64
  firstRound := true

  update := func () {
    for {
      // restart body
      ui.Body = ui.NewGrid()
      ui.Body.Width = ui.TermWidth()

      client := &http.Client{}
      req, err := http.NewRequest("GET", fmt.Sprintf("http://codeivate.com/users/%s.json", username), nil)
      req.Close = true
      req.Header.Set("Content-Type", "application/json")
      req.Header.Set("User-Agent", "NOTICE ME SENPAI v0.1a")

      resp, err := client.Do(req)

      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      body, err := ioutil.ReadAll(resp.Body)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      var statistic UserStatistic
      err = json.Unmarshal(body, &statistic)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(errors.New(fmt.Sprintf("Error on unmarshaling, probably the user %s doesn't exists", username)))
        redraw <- true
        time.Sleep(time.Second * 60)
        continue
      }

      userLevel, err := ParseLevel(statistic.Level)
      if err != nil {
        HandleMeLikeOneOfYourFrenchGirls(err)
        redraw <- true
        time.Sleep(time.Second * 5)
        continue
      }

      // i have no idea what i'm doing, but it's fucking hardcore
      hours := math.Floor(statistic.TimeSpent / 3600)
      minutes := math.Floor((statistic.TimeSpent - (hours * 3600)) / 60)
      var totalPoints float64
      var pointsSinceFirstRound float64

      for _, lang := range statistic.Languages {
        totalPoints += lang.Points
      }

      if firstRound {
        startPoints = totalPoints
      }

      pointsSinceFirstRound = totalPoints - startPoints

      parUser := ui.NewPar(
        fmt.Sprintf("Level: %d - Percent: %.0f\nTime: %.0f hours %.0f minutes - Current Language: %s\nPoints: %.0f(%.0f)",
          userLevel.Number,
          userLevel.Percent,
          hours,
          minutes,
          statistic.CurrentLanguage,
          totalPoints,
          pointsSinceFirstRound))

      parUser.Height = 5
      parUser.Border.Label = "User Info"

      ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, parUser)))

      var platformsContent []string

      var platforms []Platform

      for name, platform := range statistic.Platforms {
        platforms = append(platforms, Platform{name, platform.PercentWork, platform.Points, platform.Time})
      }

      sort.Sort(PlatformByPoints(platforms))

      for _, platform := range platforms {
        hours := math.Floor(platform.Time / 3600)
        minutes := math.Floor((platform.Time - (hours * 3600)) / 60)
        platformsContent = append(platformsContent, fmt.Sprintf("%s - Percent: %.2f - Points: %.2f - Time: %.0f hours %.0f minutes", platform.Name, platform.PercentWork, platform.Points, hours, minutes))
      }

      parWorkspace := ui.NewPar(strings.Join(platformsContent, "\n"))
      parWorkspace.Height = len(statistic.Platforms) + 2
      parWorkspace.Border.Label = "Workspace Info"

      ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, parWorkspace)))

      var languages []Language

      for name, lang := range statistic.Languages {
        level, err := ParseLevel(lang.Level)
        if err != nil {
          // don't give a fuck
          continue
        }
        languages = append(languages, Language{name, lang.Points, level.Number, level.Percent})
      }

      sort.Sort(LanguageByLevel(languages))

      for _, lang := range languages[:numberOfLanguages] {
        g := ui.NewGauge()
        g.Percent = int(lang.Percent)
        g.Height = 3
        g.Border.Label = fmt.Sprintf("%s Level: %d - Points: %.0f", lang.Name, lang.Level, lang.Points)
        g.BarColor = ui.ColorGreen

        ui.Body.AddRows(
          ui.NewRow(ui.NewCol(12, 0, g)))
      }

      ui.Render(ui.Body)
      ui.Body.Align()

      redraw <- true

      // if everything went ok
      if firstRound {
        firstRound = false
      }
      time.Sleep(time.Second * 10)
    }
  }

  evt := ui.EventCh()

  ui.Render(ui.Body)
  go update()

  for {
    select {
      case e := <-evt:
        if e.Type == ui.EventKey && (e.Ch == 'q' || e.Ch == 'Q' /* HEHEHEHE */) {
          log.Print("Everything went better than expected")
          return
        }
        if e.Type == ui.EventResize {
          ui.Body.Width = ui.TermWidth()
          ui.Body.Align()
          go func() { redraw <- true }()
        }
      case <-done:
        log.Print("Everything went better than expected")
        return
      case e := <-error:
        log.Fatal(e)
        return
      case <-redraw:
        ui.Body.Align()
        ui.Render(ui.Body)
    }
  }
}<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/caelifer/dups/finder"
)

// Internal constant to control number of Worker threads in balancer's worker pool
const workerPoolMultiplier = 1 << 3 // Use eight times the available cores

// Start of execution
func main() {
	// Flags
	var (
		cpuprofile   = flag.String("cpuprofile", "", "write cpu profile to file")
		memprofile   = flag.String("memprofile", "", "write memory profile to file")
		workerCount  = flag.Int("workers", runtime.NumCPU()*workerPoolMultiplier, "Number of parallel jobs")
		bufferedJobs = flag.Int("jbuffer", 1<<10, "Number of pending work units")
		output       = flag.String("output", "-", "write output to a file. Default: STDOUT")
		stats        = flag.Bool("stats", false, "display runtime statistics on STDERR")
	)

	// First parse flags
	flag.Parse()

	// Prep runtime to use the workerCount real threads
	runtime.GOMAXPROCS(*workerCount)

	// CPU profile
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Memory profile
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()
	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Get output writer
	out, err := getOutput(*output)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	// Trace time spent
	t1 := time.Now()

	// Find all dups and report to output
	find := finder.New(*workerCount, *bufferedJobs)
	for d := range find.AllDups(paths) {
		fmt.Fprintln(out, d)
	}

	// Update stats
	find.SetTimeSpent(time.Since(t1))

	// Display runtime stats if requested
	if *stats {
		log.Println(find.Stats())
	}
}

// Get output handle
func getOutput(path string) (io.WriteCloser, error) {
	switch path {
	case "-":
		return os.Stdout, nil // default
	case "/dev/null":
		return os.OpenFile(os.DevNull, os.O_CREATE|os.O_WRONLY, 0666)
	default:
		return os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	}
}
<commit_msg>Use simple constant<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/caelifer/dups/finder"
)

// Internal constant to control number of Worker threads in balancer's worker pool
const workerPoolMultiplier = 8 // Use eight times the available cores

// Start of execution
func main() {
	// Flags
	var (
		cpuprofile   = flag.String("cpuprofile", "", "write cpu profile to file")
		memprofile   = flag.String("memprofile", "", "write memory profile to file")
		workerCount  = flag.Int("workers", runtime.NumCPU()*workerPoolMultiplier, "Number of parallel jobs")
		bufferedJobs = flag.Int("jbuffer", 1<<10, "Number of pending work units")
		output       = flag.String("output", "-", "write output to a file. Default: STDOUT")
		stats        = flag.Bool("stats", false, "display runtime statistics on STDERR")
	)

	// First parse flags
	flag.Parse()

	// Prep runtime to use the workerCount real threads
	runtime.GOMAXPROCS(*workerCount)

	// CPU profile
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Memory profile
	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		defer func() {
			pprof.WriteHeapProfile(f)
			f.Close()
		}()
	}

	// Process command line params
	paths := flag.Args()
	if len(paths) == 0 {
		// Default is current directory
		paths = []string{"."}
	}

	// Get output writer
	out, err := getOutput(*output)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	// Trace time spent
	t1 := time.Now()

	// Find all dups and report to output
	find := finder.New(*workerCount, *bufferedJobs)
	for d := range find.AllDups(paths) {
		fmt.Fprintln(out, d)
	}

	// Update stats
	find.SetTimeSpent(time.Since(t1))

	// Display runtime stats if requested
	if *stats {
		log.Println(find.Stats())
	}
}

// Get output handle
func getOutput(path string) (io.WriteCloser, error) {
	switch path {
	case "-":
		return os.Stdout, nil // default
	case "/dev/null":
		return os.OpenFile(os.DevNull, os.O_CREATE|os.O_WRONLY, 0666)
	default:
		return os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"time"

	"os"

	"github.com/42wim/ipisp"
	"github.com/dustin/go-humanize"
	"github.com/miekg/dns"
	"text/tabwriter"
)

var (
	resolver = "8.8.8.8:53"
)

type NSInfo struct {
	Name string
	IP   []net.IP
}

type IPInfo struct {
	Loc string
	ASN ipisp.ASN
	ISP string
}

type KeyInfo struct {
	Start int64
	End   int64
}

func ipinfo(ip net.IP) (IPInfo, error) {
	client, _ := ipisp.NewDNSClient()
	resp, err := client.LookupIP(net.ParseIP(ip.String()))
	if err != nil {
		return IPInfo{}, err
	}
	return IPInfo{resp.Country, resp.ASN, resp.Name.Raw}, nil
}

func typeSOA(q string, server string) (time.Duration, *dns.SOA, error) {
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{q + ".", dns.TypeSOA, dns.ClassINET}
	in, rtt, err := c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return 0, new(dns.SOA), err
	}
	return rtt, in.Answer[0].(*dns.SOA), nil
}

func getIP(host string, qtype uint16) []net.IP {
	var ips []net.IP
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{host, qtype, dns.ClassINET}
	in, _, err := c.Exchange(m, resolver)
	if err != nil {
		return ips
	}
	for _, a := range in.Answer {
		switch a.(type) {
		case *dns.A:
			ips = append(ips, a.(*dns.A).A)
		case *dns.AAAA:
			ips = append(ips, a.(*dns.AAAA).AAAA)
		}
	}
	return ips
}

func typeA(host string) []net.IP {
	return getIP(host, dns.TypeA)
}

func typeAAAA(host string) []net.IP {
	return getIP(host, dns.TypeAAAA)
}

func typeDNSKEY(q string, server string) (bool, KeyInfo, error) {
	c := new(dns.Client)
	m := prepMsg()
	m.SetEdns0(4096, true)
	m.Question[0] = dns.Question{q + ".", dns.TypeDNSKEY, dns.ClassINET}
	in, _, err := c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return false, KeyInfo{}, err
	}
	keys := []*dns.DNSKEY{}
	for _, a := range in.Answer {
		switch a.(type) {
		case *dns.DNSKEY:
			keys = append(keys, a.(*dns.DNSKEY))
		}
	}

	// ask dnssec
	m = prepMsg()
	m.SetEdns0(4096, true)
	m.Question[0] = dns.Question{q + ".", dns.TypeNS, dns.ClassINET}
	in, _, err = c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return false, KeyInfo{}, err
	}
	return validateRR(keys, in.Answer)
}

func validateRR(keys []*dns.DNSKEY, rrset []dns.RR) (bool, KeyInfo, error) {
	if len(rrset) == 0 {
		return false, KeyInfo{}, nil
	}
	var sig *dns.RRSIG
	var cleanset []dns.RR
	for _, v := range rrset {
		_, ok := v.(*dns.RRSIG)
		if ok {
			sig = v.(*dns.RRSIG)
		} else {
			cleanset = append(cleanset, v)
		}
	}
	for _, key := range keys {
		// zone signing key
		if key.Flags == 256 {
			err := sig.Verify(key, cleanset)
			if err == nil {
				ti, te := explicitValid(sig)
				if sig.ValidityPeriod(time.Now()) {
					return true, KeyInfo{ti, te}, nil
				}
				return false, KeyInfo{ti, te}, nil
			}
		}
	}
	return false, KeyInfo{}, nil
}

func explicitValid(rr *dns.RRSIG) (int64, int64) {
	t := time.Now()
	var utc int64
	var year68 = int64(1 << 31)
	if t.IsZero() {
		utc = time.Now().UTC().Unix()
	} else {
		utc = t.UTC().Unix()
	}
	modi := (int64(rr.Inception) - utc) / year68
	mode := (int64(rr.Expiration) - utc) / year68
	ti := int64(rr.Inception) + (modi * year68)
	te := int64(rr.Expiration) + (mode * year68)
	return ti, te
}

func findNS(domain string) []NSInfo {
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{domain, dns.TypeNS, dns.ClassINET}
	in, _, _ := c.Exchange(m, resolver)
	var ips []net.IP
	var nsinfos []NSInfo
	for _, a := range in.Answer {
		nsinfo := NSInfo{}
		nsinfo.Name = a.(*dns.NS).Ns
		ips = append(ips, typeA(a.(*dns.NS).Ns)...)
		ips = append(ips, typeAAAA(a.(*dns.NS).Ns)...)
		nsinfo.IP = ips
		nsinfos = append(nsinfos, nsinfo)
		ips = []net.IP{}
	}
	return nsinfos
}

func prepMsg() *dns.Msg {
	m := new(dns.Msg)
	m.Id = dns.Id()
	m.RecursionDesired = true
	m.Question = make([]dns.Question, 1)
	return m
}

func main() {
	if len(os.Args) == 1 {
		fmt.Println("please enter a domain. (e.g. google.com)")
		return
	}
	domain := os.Args[1]
	nsinfos := findNS(dns.Fqdn(domain))
	if len(nsinfos) == 0 {
		fmt.Println("no nameservers found for", domain)
		return
	}
	const padding = 1
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.Debug)
	fmt.Fprintf(w, "NS\tIP\tLOC\tASN\tISP\trtt\tSerial\tDNSSEC\tValidFrom\tValidUntil\n")
	for _, nsinfo := range nsinfos {
		fmt.Fprintf(w, "%s\t", nsinfo.Name)
		i := 0
		for _, ip := range nsinfo.IP {
			info, _ := ipinfo(ip)
			if i > 0 {
				fmt.Fprintf(w, "\t%s\t", ip.String())
			} else {
				fmt.Fprintf(w, "%s\t", ip.String())
			}
			fmt.Fprintf(w, "%v\tASN %#v\t%v\t", info.Loc, info.ASN, fmt.Sprintf("%.40s", info.ISP))
			rtt, soa, err := typeSOA(domain, ip.String())
			if err != nil {
				fmt.Fprintf(w, "%s\t%v\t", "error", "error")
			} else {
				fmt.Fprintf(w, "%s\t%v\t", rtt.String(), int64(soa.Serial))
			}
			valid, keyinfo, err := typeDNSKEY(domain, ip.String())
			if valid {
				fmt.Fprintf(w, "%v\t%s\t%s", "valid", humanize.Time(time.Unix(keyinfo.Start, 0)), humanize.Time(time.Unix(keyinfo.End, 0)))
			} else {
				if err != nil {
					fmt.Fprintf(w, "%v\t%s\t%s", "error", "", "")
				} else {
					fmt.Fprintf(w, "%v\t%s\t%s", "invalid", "", "")
				}
			}
			fmt.Fprintln(w)
			i++
		}
	}
	w.Flush()
	fmt.Println()
}
<commit_msg>Add concurrency. Query every NS concurrently<commit_after>package main

import (
	"fmt"
	"net"
	"sync"
	"text/tabwriter"
	"time"

	"os"

	"github.com/42wim/ipisp"
	"github.com/dustin/go-humanize"
	"github.com/miekg/dns"
)

var (
	resolver = "8.8.8.8:53"
	wc       chan string
	done     chan struct{}
)

type NSInfo struct {
	Name string
	IP   []net.IP
}

type IPInfo struct {
	Loc string
	ASN ipisp.ASN
	ISP string
}

type KeyInfo struct {
	Start int64
	End   int64
}

func ipinfo(ip net.IP) (IPInfo, error) {
	client, _ := ipisp.NewDNSClient()
	resp, err := client.LookupIP(net.ParseIP(ip.String()))
	if err != nil {
		return IPInfo{}, err
	}
	return IPInfo{resp.Country, resp.ASN, resp.Name.Raw}, nil
}

func typeSOA(q string, server string) (time.Duration, *dns.SOA, error) {
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{q + ".", dns.TypeSOA, dns.ClassINET}
	in, rtt, err := c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return 0, new(dns.SOA), err
	}
	return rtt, in.Answer[0].(*dns.SOA), nil
}

func getIP(host string, qtype uint16) []net.IP {
	var ips []net.IP
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{host, qtype, dns.ClassINET}
	in, _, err := c.Exchange(m, resolver)
	if err != nil {
		return ips
	}
	for _, a := range in.Answer {
		switch a.(type) {
		case *dns.A:
			ips = append(ips, a.(*dns.A).A)
		case *dns.AAAA:
			ips = append(ips, a.(*dns.AAAA).AAAA)
		}
	}
	return ips
}

func typeA(host string) []net.IP {
	return getIP(host, dns.TypeA)
}

func typeAAAA(host string) []net.IP {
	return getIP(host, dns.TypeAAAA)
}

func typeDNSKEY(q string, server string) (bool, KeyInfo, error) {
	c := new(dns.Client)
	m := prepMsg()
	m.SetEdns0(4096, true)
	m.Question[0] = dns.Question{q + ".", dns.TypeDNSKEY, dns.ClassINET}
	in, _, err := c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return false, KeyInfo{}, err
	}
	keys := []*dns.DNSKEY{}
	for _, a := range in.Answer {
		switch a.(type) {
		case *dns.DNSKEY:
			keys = append(keys, a.(*dns.DNSKEY))
		}
	}

	// ask dnssec
	m = prepMsg()
	m.SetEdns0(4096, true)
	m.Question[0] = dns.Question{q + ".", dns.TypeNS, dns.ClassINET}
	in, _, err = c.Exchange(m, net.JoinHostPort(server, "53"))
	if err != nil {
		return false, KeyInfo{}, err
	}
	return validateRR(keys, in.Answer)
}

func validateRR(keys []*dns.DNSKEY, rrset []dns.RR) (bool, KeyInfo, error) {
	if len(rrset) == 0 {
		return false, KeyInfo{}, nil
	}
	var sig *dns.RRSIG
	var cleanset []dns.RR
	for _, v := range rrset {
		_, ok := v.(*dns.RRSIG)
		if ok {
			sig = v.(*dns.RRSIG)
		} else {
			cleanset = append(cleanset, v)
		}
	}
	for _, key := range keys {
		// zone signing key
		if key.Flags == 256 {
			err := sig.Verify(key, cleanset)
			if err == nil {
				ti, te := explicitValid(sig)
				if sig.ValidityPeriod(time.Now()) {
					return true, KeyInfo{ti, te}, nil
				}
				return false, KeyInfo{ti, te}, nil
			}
		}
	}
	return false, KeyInfo{}, nil
}

func explicitValid(rr *dns.RRSIG) (int64, int64) {
	t := time.Now()
	var utc int64
	var year68 = int64(1 << 31)
	if t.IsZero() {
		utc = time.Now().UTC().Unix()
	} else {
		utc = t.UTC().Unix()
	}
	modi := (int64(rr.Inception) - utc) / year68
	mode := (int64(rr.Expiration) - utc) / year68
	ti := int64(rr.Inception) + (modi * year68)
	te := int64(rr.Expiration) + (mode * year68)
	return ti, te
}

func findNS(domain string) []NSInfo {
	c := new(dns.Client)
	m := prepMsg()
	m.Question[0] = dns.Question{domain, dns.TypeNS, dns.ClassINET}
	in, _, _ := c.Exchange(m, resolver)
	var ips []net.IP
	var nsinfos []NSInfo
	for _, a := range in.Answer {
		nsinfo := NSInfo{}
		nsinfo.Name = a.(*dns.NS).Ns
		ips = append(ips, typeA(a.(*dns.NS).Ns)...)
		ips = append(ips, typeAAAA(a.(*dns.NS).Ns)...)
		nsinfo.IP = ips
		nsinfos = append(nsinfos, nsinfo)
		ips = []net.IP{}
	}
	return nsinfos
}

func prepMsg() *dns.Msg {
	m := new(dns.Msg)
	m.Id = dns.Id()
	m.RecursionDesired = true
	m.Question = make([]dns.Question, 1)
	return m
}

func outputter() {
	const padding = 1
	w := tabwriter.NewWriter(os.Stdout, 0, 0, padding, ' ', tabwriter.Debug)
	for input := range wc {
		fmt.Fprintf(w, input)
	}
	w.Flush()
	done <- struct{}{}
}

func main() {
	if len(os.Args) == 1 {
		fmt.Println("please enter a domain. (e.g. google.com)")
		return
	}
	domain := os.Args[1]
	nsinfos := findNS(dns.Fqdn(domain))
	if len(nsinfos) == 0 {
		fmt.Println("no nameservers found for", domain)
		return
	}
	wc = make(chan string)
	done = make(chan struct{})
	var wg sync.WaitGroup
	go outputter()

	wc <- fmt.Sprintf("NS\tIP\tLOC\tASN\tISP\trtt\tSerial\tDNSSEC\tValidFrom\tValidUntil\n")
	for _, nsinfo := range nsinfos {
		wg.Add(1)
		go func(nsinfo NSInfo) {
			output := fmt.Sprintf("%s\t", nsinfo.Name)
			i := 0
			for _, ip := range nsinfo.IP {
				info, _ := ipinfo(ip)
				if i > 0 {
					output = output + fmt.Sprintf("\t%s\t", ip.String())
				} else {
					output = output + fmt.Sprintf("%s\t", ip.String())
				}
				output = output + fmt.Sprintf("%v\tASN %#v\t%v\t", info.Loc, info.ASN, fmt.Sprintf("%.40s", info.ISP))
				rtt, soa, err := typeSOA(domain, ip.String())
				if err != nil {
					output = output + fmt.Sprintf("%s\t%v\t", "error", "error")
				} else {
					output = output + fmt.Sprintf("%s\t%v\t", rtt.String(), int64(soa.Serial))
				}
				valid, keyinfo, err := typeDNSKEY(domain, ip.String())
				if valid {
					output = output + fmt.Sprintf("%v\t%s\t%s", "valid", humanize.Time(time.Unix(keyinfo.Start, 0)), humanize.Time(time.Unix(keyinfo.End, 0)))
				} else {
					if err != nil {
						output = output + fmt.Sprintf("%v\t%s\t%s", "error", "", "")
					} else {
						if keyinfo.Start == 0 {
							output = output + fmt.Sprintf("%v\t%s\t%s", "disabled", "", "")
						} else {
							output = output + fmt.Sprintf("%v\t%s\t%s", "invalid", "", "")
						}
					}
				}
				output = output + fmt.Sprintln()
				i++
			}
			wc <- output
			wg.Done()
		}(nsinfo)
	}
	wg.Wait()
	close(wc)
	<-done
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"time"
)

var dbRoot = flag.String("root", "db", "Root directory for database files.")
var flushTime = flag.Duration("flushDelay", time.Second*5,
	"Maximum amount of time to wait before flushing")
var maxOpQueue = flag.Int("maxOpQueue", 1000,
	"Maximum number of queued items before flushing")
var staticPath = flag.String("static", "static", "Path to static data")
var queryTimeout = flag.Duration("maxQueryTime", time.Minute*5,
	"Maximum amount of time a query is allowed to process.")
var queryBacklog = flag.Int("queryBacklog", 0, "Query scan/group backlog size")
var docBacklog = flag.Int("docBacklog", 0, "MR group request backlog size")
var cacheAddr = flag.String("memcache", "", "Memcached server to connect to")
var cacheBacklog = flag.Int("cacheBacklog", 1000, "Cache backlog size")
var cacheWorkers = flag.Int("cacheWorkers", 4, "Number of cache workers")
var verbose = flag.Bool("v", false, "Verbose logging")
var useSyslog = flag.Bool("syslog", false, "Log to syslog")

// Profiling
var pprofFile = flag.String("proFile", "", "File to write profiling info into")
var pprofStart = flag.Duration("proStart", 5*time.Second,
	"How long after startup to start profiling")
var pprofDuration = flag.Duration("proDuration", 5*time.Minute,
	"How long to run the profiler before shutting it down")

type routeHandler func(parts []string, w http.ResponseWriter, req *http.Request)

type routingEntry struct {
	Method   string
	Path     *regexp.Regexp
	Handler  routeHandler
	Deadline time.Duration
}

const dbMatch = "[-%+()$_a-zA-Z0-9]+"

var defaultDeadline = time.Millisecond * 50

var routingTable []routingEntry = []routingEntry{
	routingEntry{"GET", regexp.MustCompile("^/$"),
		serverInfo, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_static/(.*)"),
		staticHandler, defaultDeadline},
	// Database stuff
	routingEntry{"GET", regexp.MustCompile("^/_all_dbs$"),
		listDatabases, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_(.*)"),
		reservedHandler, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		dbInfo, defaultDeadline},
	routingEntry{"HEAD", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		checkDB, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_changes$"),
		dbChanges, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_query$"),
		query, *queryTimeout},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/_bulk$"),
		deleteBulk, *queryTimeout},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_all"),
		allDocs, *queryTimeout},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/_compact"),
		compact, time.Second * 30},
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		createDB, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		deleteDB, defaultDeadline},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		newDocument, defaultDeadline},
	// Document stuff
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		putDocument, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		getDocument, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		rmDocument, defaultDeadline},
}

func mustEncode(status int, w http.ResponseWriter, ob interface{}) {
	b, err := json.Marshal(ob)
	if err != nil {
		log.Fatalf("Error encoding %v.", ob)
	}
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(b)))
	w.WriteHeader(status)
	w.Write(b)
}

func emitError(status int, w http.ResponseWriter, e, reason string) {
	m := map[string]string{"error": e, "reason": reason}
	mustEncode(status, w, m)
}

func staticHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	w.Header().Del("Content-type")
	http.StripPrefix("/_static/",
		http.FileServer(http.Dir(*staticPath))).ServeHTTP(w, req)
}

func reservedHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,
		w, "illegal_database_name",
		"Only lowercase characters (a-z), digits (0-9), "+
			"and any of the characters _, $, (, ), +, -, and / are allowed. "+
			"Must begin with a letter.")

}

func defaultHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,

		w, "no_handler",
		fmt.Sprintf("Can't handle %v to %v\n", req.Method, req.URL.Path))

}

func findHandler(method, path string) (routingEntry, []string) {
	for _, r := range routingTable {
		if r.Method == method {
			matches := r.Path.FindAllStringSubmatch(path, 1)
			if len(matches) > 0 {
				return r, matches[0][1:]
			}
		}
	}
	return routingEntry{"DEFAULT", nil, defaultHandler, defaultDeadline},
		[]string{}
}

func handler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	start := time.Now()
	route, hparts := findHandler(req.Method, req.URL.Path)
	wd := time.AfterFunc(route.Deadline, func() {
		log.Printf("%v:%v is taking longer than %v",
			req.Method, req.URL.Path, route.Deadline)
	})

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-type", "application/json")
	route.Handler(hparts, w, req)

	if !wd.Stop() {
		log.Printf("%v:%v eventually finished in %v",
			req.Method, req.URL.Path, time.Since(start))
	}
}

func startProfiler() {
	time.Sleep(*pprofStart)
	log.Printf("Starting profiler")
	f, err := os.OpenFile(*pprofFile, os.O_WRONLY|os.O_CREATE, 0666)
	if err == nil {
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatalf("Can't start profiler")
		}
		time.AfterFunc(*pprofDuration, func() {
			log.Printf("Shutting down profiler")
			pprof.StopCPUProfile()
			f.Close()
		})
	} else {
		log.Printf("Can't open profilefile")
	}

}

func main() {
	halfProcs := runtime.GOMAXPROCS(0) / 2
	if halfProcs < 1 {
		halfProcs = 1
	}
	queryWorkers := flag.Int("queryWorkers", halfProcs,
		"Number of query tree walkers.")
	docWorkers := flag.Int("docWorkers", halfProcs,
		"Number of document mapreduce workers.")

	addr := flag.String("addr", ":3133", "Address to bind to")
	mcaddr := flag.String("memcbind", "", "Memcached server bind address")
	flag.Parse()

	if *useSyslog {
		sl, err := syslog.New(syslog.LOG_INFO, "seriesly")
		if err != nil {
			log.Fatalf("Can't initialize syslog: %v", err)
		}
		log.SetOutput(sl)
		log.SetFlags(0)
	}

	if err := os.MkdirAll(*dbRoot, 0777); err != nil {
		log.Fatalf("Could not create %v: %v", *dbRoot, err)
	}

	// Update the query handler deadline to the query timeout
	found := false
	for i := range routingTable {
		matches := routingTable[i].Path.FindAllStringSubmatch("/x/_query", 1)
		if len(matches) > 0 {
			routingTable[i].Deadline = *queryTimeout
			found = true
			break
		}
	}
	if !found {
		log.Fatalf("Programming error:  Could not find query handler")
	}

	processorInput = make(chan *processIn, *docBacklog)
	for i := 0; i < *docWorkers; i++ {
		go docProcessor(processorInput)
	}

	if *cacheAddr == "" {
		cacheInput = processorInput
		// Note: cacheInputSet will be null here, there should be no caching
	} else {
		cacheInput = make(chan *processIn, *cacheBacklog)
		cacheInputSet = make(chan *processOut, *cacheBacklog)
		for i := 0; i < *cacheWorkers; i++ {
			go cacheProcessor(cacheInput, cacheInputSet)
		}
	}

	queryInput = make(chan *queryIn, *queryBacklog)
	for i := 0; i < *queryWorkers; i++ {
		go queryExecutor(queryInput)
	}

	if *pprofFile != "" {
		go startProfiler()
	}

	if *mcaddr != "" {
		go listenMC(*mcaddr)
	}

	s := &http.Server{
		Addr:        *addr,
		Handler:     http.HandlerFunc(handler),
		ReadTimeout: 5 * time.Second,
	}
	log.Printf("Listening to web requests on %s", *addr)
	log.Fatal(s.ListenAndServe())
}
<commit_msg>add flag for access log<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"net/http"
	"os"
	"regexp"
	"runtime"
	"runtime/pprof"
	"time"
)

var dbRoot = flag.String("root", "db", "Root directory for database files.")
var flushTime = flag.Duration("flushDelay", time.Second*5,
	"Maximum amount of time to wait before flushing")
var maxOpQueue = flag.Int("maxOpQueue", 1000,
	"Maximum number of queued items before flushing")
var staticPath = flag.String("static", "static", "Path to static data")
var queryTimeout = flag.Duration("maxQueryTime", time.Minute*5,
	"Maximum amount of time a query is allowed to process.")
var queryBacklog = flag.Int("queryBacklog", 0, "Query scan/group backlog size")
var docBacklog = flag.Int("docBacklog", 0, "MR group request backlog size")
var cacheAddr = flag.String("memcache", "", "Memcached server to connect to")
var cacheBacklog = flag.Int("cacheBacklog", 1000, "Cache backlog size")
var cacheWorkers = flag.Int("cacheWorkers", 4, "Number of cache workers")
var verbose = flag.Bool("v", false, "Verbose logging")
var logAccess = flag.Bool("logAccess", false, "Log HTTP Requests")
var useSyslog = flag.Bool("syslog", false, "Log to syslog")

// Profiling
var pprofFile = flag.String("proFile", "", "File to write profiling info into")
var pprofStart = flag.Duration("proStart", 5*time.Second,
	"How long after startup to start profiling")
var pprofDuration = flag.Duration("proDuration", 5*time.Minute,
	"How long to run the profiler before shutting it down")

type routeHandler func(parts []string, w http.ResponseWriter, req *http.Request)

type routingEntry struct {
	Method   string
	Path     *regexp.Regexp
	Handler  routeHandler
	Deadline time.Duration
}

const dbMatch = "[-%+()$_a-zA-Z0-9]+"

var defaultDeadline = time.Millisecond * 50

var routingTable []routingEntry = []routingEntry{
	routingEntry{"GET", regexp.MustCompile("^/$"),
		serverInfo, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_static/(.*)"),
		staticHandler, defaultDeadline},
	// Database stuff
	routingEntry{"GET", regexp.MustCompile("^/_all_dbs$"),
		listDatabases, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/_(.*)"),
		reservedHandler, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		dbInfo, defaultDeadline},
	routingEntry{"HEAD", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		checkDB, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_changes$"),
		dbChanges, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_query$"),
		query, *queryTimeout},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/_bulk$"),
		deleteBulk, *queryTimeout},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/_all"),
		allDocs, *queryTimeout},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/_compact"),
		compact, time.Second * 30},
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		createDB, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		deleteDB, defaultDeadline},
	routingEntry{"POST", regexp.MustCompile("^/(" + dbMatch + ")/?$"),
		newDocument, defaultDeadline},
	// Document stuff
	routingEntry{"PUT", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		putDocument, defaultDeadline},
	routingEntry{"GET", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		getDocument, defaultDeadline},
	routingEntry{"DELETE", regexp.MustCompile("^/(" + dbMatch + ")/([^/]+)$"),
		rmDocument, defaultDeadline},
}

func mustEncode(status int, w http.ResponseWriter, ob interface{}) {
	b, err := json.Marshal(ob)
	if err != nil {
		log.Fatalf("Error encoding %v.", ob)
	}
	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(b)))
	w.WriteHeader(status)
	w.Write(b)
}

func emitError(status int, w http.ResponseWriter, e, reason string) {
	m := map[string]string{"error": e, "reason": reason}
	mustEncode(status, w, m)
}

func staticHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	w.Header().Del("Content-type")
	http.StripPrefix("/_static/",
		http.FileServer(http.Dir(*staticPath))).ServeHTTP(w, req)
}

func reservedHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,
		w, "illegal_database_name",
		"Only lowercase characters (a-z), digits (0-9), "+
			"and any of the characters _, $, (, ), +, -, and / are allowed. "+
			"Must begin with a letter.")

}

func defaultHandler(parts []string, w http.ResponseWriter, req *http.Request) {
	emitError(400,

		w, "no_handler",
		fmt.Sprintf("Can't handle %v to %v\n", req.Method, req.URL.Path))

}

func findHandler(method, path string) (routingEntry, []string) {
	for _, r := range routingTable {
		if r.Method == method {
			matches := r.Path.FindAllStringSubmatch(path, 1)
			if len(matches) > 0 {
				return r, matches[0][1:]
			}
		}
	}
	return routingEntry{"DEFAULT", nil, defaultHandler, defaultDeadline},
		[]string{}
}

func handler(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()
	start := time.Now()
	if *logAccess {
		log.Printf("%s %s %s", req.RemoteAddr, req.Method, req.URL)
	}
	route, hparts := findHandler(req.Method, req.URL.Path)
	wd := time.AfterFunc(route.Deadline, func() {
		log.Printf("%v:%v is taking longer than %v",
			req.Method, req.URL.Path, route.Deadline)
	})

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-type", "application/json")
	route.Handler(hparts, w, req)

	if !wd.Stop() {
		log.Printf("%v:%v eventually finished in %v",
			req.Method, req.URL.Path, time.Since(start))
	}
}

func startProfiler() {
	time.Sleep(*pprofStart)
	log.Printf("Starting profiler")
	f, err := os.OpenFile(*pprofFile, os.O_WRONLY|os.O_CREATE, 0666)
	if err == nil {
		err = pprof.StartCPUProfile(f)
		if err != nil {
			log.Fatalf("Can't start profiler")
		}
		time.AfterFunc(*pprofDuration, func() {
			log.Printf("Shutting down profiler")
			pprof.StopCPUProfile()
			f.Close()
		})
	} else {
		log.Printf("Can't open profilefile")
	}

}

func main() {
	halfProcs := runtime.GOMAXPROCS(0) / 2
	if halfProcs < 1 {
		halfProcs = 1
	}
	queryWorkers := flag.Int("queryWorkers", halfProcs,
		"Number of query tree walkers.")
	docWorkers := flag.Int("docWorkers", halfProcs,
		"Number of document mapreduce workers.")

	addr := flag.String("addr", ":3133", "Address to bind to")
	mcaddr := flag.String("memcbind", "", "Memcached server bind address")
	flag.Parse()

	if *useSyslog {
		sl, err := syslog.New(syslog.LOG_INFO, "seriesly")
		if err != nil {
			log.Fatalf("Can't initialize syslog: %v", err)
		}
		log.SetOutput(sl)
		log.SetFlags(0)
	}

	if err := os.MkdirAll(*dbRoot, 0777); err != nil {
		log.Fatalf("Could not create %v: %v", *dbRoot, err)
	}

	// Update the query handler deadline to the query timeout
	found := false
	for i := range routingTable {
		matches := routingTable[i].Path.FindAllStringSubmatch("/x/_query", 1)
		if len(matches) > 0 {
			routingTable[i].Deadline = *queryTimeout
			found = true
			break
		}
	}
	if !found {
		log.Fatalf("Programming error:  Could not find query handler")
	}

	processorInput = make(chan *processIn, *docBacklog)
	for i := 0; i < *docWorkers; i++ {
		go docProcessor(processorInput)
	}

	if *cacheAddr == "" {
		cacheInput = processorInput
		// Note: cacheInputSet will be null here, there should be no caching
	} else {
		cacheInput = make(chan *processIn, *cacheBacklog)
		cacheInputSet = make(chan *processOut, *cacheBacklog)
		for i := 0; i < *cacheWorkers; i++ {
			go cacheProcessor(cacheInput, cacheInputSet)
		}
	}

	queryInput = make(chan *queryIn, *queryBacklog)
	for i := 0; i < *queryWorkers; i++ {
		go queryExecutor(queryInput)
	}

	if *pprofFile != "" {
		go startProfiler()
	}

	if *mcaddr != "" {
		go listenMC(*mcaddr)
	}

	s := &http.Server{
		Addr:        *addr,
		Handler:     http.HandlerFunc(handler),
		ReadTimeout: 5 * time.Second,
	}
	log.Printf("Listening to web requests on %s", *addr)
	log.Fatal(s.ListenAndServe())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/bitrise-tools/go-steputils/input"

	"github.com/bitrise-io/go-utils/command"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/go-utils/retry"
	"github.com/bitrise-tools/go-android/sdk"
	"github.com/bitrise-tools/go-android/sdkcomponent"
	"github.com/bitrise-tools/go-android/sdkmanager"
)

// ConfigsModel ...
type ConfigsModel struct {
	GradlewPath string
	AndroidHome string
}

func createConfigsModelFromEnvs() ConfigsModel {
	return ConfigsModel{
		GradlewPath: os.Getenv("gradlew_path"),
		AndroidHome: os.Getenv("ANDROID_HOME"),
	}
}

func (configs ConfigsModel) print() {
	log.Infof("Configs:")
	log.Printf("- GradlewPath: %s", configs.GradlewPath)
	log.Printf("- AndroidHome: %s", configs.AndroidHome)
}

func (configs ConfigsModel) validate() error {
	if err := input.ValidateIfPathExists(configs.GradlewPath); err != nil {
		return errors.New("Issue with input GradlewPath: " + err.Error())
	}

	if err := input.ValidateIfNotEmpty(configs.AndroidHome); err != nil {
		return errors.New("Issue with input AndroidHome: " + err.Error())
	}

	return nil
}

// -----------------------
// --- Functions
// -----------------------

func ensureAndroidLicences(androidHome string, isLegacySDK bool) error {
	if !isLegacySDK {
		licensesCmd := command.New(filepath.Join(androidHome, "tools/bin/sdkmanager"), "--licenses")
		licensesCmd.SetStdin(bytes.NewReader([]byte(strings.Repeat("y\n", 1000))))
		if err := licensesCmd.Run(); err != nil {
			log.Warnf("Failed to install licenses using $(sdkmanager --licenses) command")
			log.Printf("Continue using legacy license installation...")
			fmt.Println()
		} else {
			return nil
		}
	}

	licenceMap := map[string]string{
		"android-sdk-license":           "8933bad161af4178b1185d1a37fbf41ea5269c55\n\nd56f5187479451eabf01fb78af6dfcb131a6481e",
		"android-googletv-license":      "\n601085b94cd77f0b54ff86406957099ebe79c4d6",
		"android-sdk-preview-license":   "\n84831b9409646a918e30573bab4c9c91346d8abd",
		"intel-android-extra-license":   "\nd975f751698a77b662f1254ddbeed3901e976f5a",
		"google-gdk-license":            "\n33b6a2b64607f11b759f320ef9dff4ae5c47d97a",
		"mips-android-sysimage-license": "\ne9acab5b5fbb560a72cfaecce8946896ff6aab9d",
	}

	licencesDir := filepath.Join(androidHome, "licenses")
	if exist, err := pathutil.IsDirExists(licencesDir); err != nil {
		return err
	} else if !exist {
		if err := os.MkdirAll(licencesDir, os.ModePerm); err != nil {
			return err
		}
	}

	for name, content := range licenceMap {
		pth := filepath.Join(licencesDir, name)

		if err := fileutil.WriteStringToFile(pth, content); err != nil {
			return err
		}
	}

	return nil
}

func failf(format string, v ...interface{}) {
	log.Errorf(format, v...)
	os.Exit(1)
}

// -----------------------
// --- Main
// -----------------------

func main() {
	// Input validation
	configs := createConfigsModelFromEnvs()

	fmt.Println()
	configs.print()

	if err := configs.validate(); err != nil {
		failf(err.Error())
	}

	fmt.Println()
	log.Infof("Preparation")

	// Set executable permission for gradlew
	log.Printf("Set executable permission for gradlew")
	if err := os.Chmod(configs.GradlewPath, 0770); err != nil {
		failf("Failed to set executable permission for gradlew, error: %s", err)
	}

	// Initialize Android SDK
	log.Printf("Initialize Android SDK")
	androidSdk, err := sdk.New(configs.AndroidHome)
	if err != nil {
		failf("Failed to initialize Android SDK, error: %s", err)
	}

	sdkManager, err := sdkmanager.New(androidSdk)
	if err != nil {
		failf("Failed to create SDK manager, error: %s", err)
	}

	// Ensure android licences
	log.Printf("Ensure android licences")
	if err := ensureAndroidLicences(configs.AndroidHome, sdkManager.IsLegacySDK()); err != nil {
		failf("Failed to ensure android licences, error: %s", err)
	}

	// Ensure required Android SDK components
	fmt.Println()
	log.Infof("Ensure required Android SDK components")

	retryCount := 0
	for true {
		gradleCmd := command.New("./gradlew", "dependencies")
		gradleCmd.SetStdin(strings.NewReader("y"))
		gradleCmd.SetDir(filepath.Dir(configs.GradlewPath))

		log.Printf("Searching for missing SDK components using:")
		log.Printf("$ %s", gradleCmd.PrintableCommandArgs())

		if out, err := gradleCmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
			reader := strings.NewReader(out)
			scanner := bufio.NewScanner(reader)

			missingSDKComponentFound := false

			for scanner.Scan() {
				line := scanner.Text()
				{
					// failed to find target with hash string 'android-22'
					targetPattern := `failed to find target with hash string 'android-(?P<version>.*)'\s*`
					targetRe := regexp.MustCompile(targetPattern)
					if matches := targetRe.FindStringSubmatch(line); len(matches) == 2 {
						missingSDKComponentFound = true

						targetVersion := "android-" + matches[1]

						log.Warnf("Missing platform version found: %s", targetVersion)

						platformComponent := sdkcomponent.Platform{
							Version: targetVersion,
						}

						cmd := sdkManager.InstallCommand(platformComponent)
						cmd.SetStdin(strings.NewReader("y"))

						log.Printf("Installing platform version using:")
						log.Printf("$ %s", cmd.PrintableCommandArgs())

						if err := retry.Times(1).Wait(time.Second).Try(func(attempt uint) error {
							if attempt > 0 {
								log.Warnf("Retrying...")
							}

							if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
								if attempt > 0 {
									return fmt.Errorf("output: %s, error: %s", out, err)
								}
								return err
							}

							return nil
						}); err != nil {
							log.Errorf("Failed to install platform:")
							failf("%s", err)
						}
					}
				}

				{
					// failed to find Build Tools revision 22.0.1
					buildToolsPattern := `failed to find Build Tools revision (?P<version>[0-9.]*)\s*`
					buildToolsRe := regexp.MustCompile(buildToolsPattern)
					if matches := buildToolsRe.FindStringSubmatch(line); len(matches) == 2 {
						missingSDKComponentFound = true

						buildToolsVersion := matches[1]

						log.Warnf("Missing build tools version found: %s", buildToolsVersion)

						buildToolsComponent := sdkcomponent.BuildTool{
							Version: buildToolsVersion,
						}

						cmd := sdkManager.InstallCommand(buildToolsComponent)
						cmd.SetStdin(strings.NewReader("y"))

						log.Printf("Installing build tools version using:")
						log.Printf("$ %s", cmd.PrintableCommandArgs())

						if err := retry.Times(1).Wait(time.Second).Try(func(attempt uint) error {
							if attempt > 0 {
								log.Warnf("Retrying...")
							}

							if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
								if attempt > 0 {
									return fmt.Errorf("output: %s, error: %s", out, err)
								}
								return err
							}

							return nil
						}); err != nil {
							log.Errorf("Failed to install build tools:")
							failf("%s", err)
						}
					}
				}
			}

			if err := scanner.Err(); err != nil {
				failf("failed to analyze gradle output, error: %s", err)
			}

			if !missingSDKComponentFound {
				if retryCount < 2 {
					log.Errorf("Failed to find missing components, retrying...")
					retryCount++
					continue
				}
				log.Printf(out)
				failf("%s", err)
			}
		} else {
			break
		}
	}

	fmt.Println()
	log.Donef("Required SDK components are installed")
}
<commit_msg>Install support repositories and missing packages (#32)<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/bitrise-tools/go-steputils/input"

	"github.com/bitrise-io/go-utils/command"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/go-utils/retry"
	"github.com/bitrise-tools/go-android/sdk"
	"github.com/bitrise-tools/go-android/sdkcomponent"
	"github.com/bitrise-tools/go-android/sdkmanager"
)

// ConfigsModel ...
type ConfigsModel struct {
	GradlewPath string
	AndroidHome string
}

func createConfigsModelFromEnvs() ConfigsModel {
	return ConfigsModel{
		GradlewPath: os.Getenv("gradlew_path"),
		AndroidHome: os.Getenv("ANDROID_HOME"),
	}
}

func (configs ConfigsModel) print() {
	log.Infof("Configs:")
	log.Printf("- GradlewPath: %s", configs.GradlewPath)
	log.Printf("- AndroidHome: %s", configs.AndroidHome)
}

func (configs ConfigsModel) validate() error {
	if err := input.ValidateIfPathExists(configs.GradlewPath); err != nil {
		return errors.New("Issue with input GradlewPath: " + err.Error())
	}

	if err := input.ValidateIfNotEmpty(configs.AndroidHome); err != nil {
		return errors.New("Issue with input AndroidHome: " + err.Error())
	}

	return nil
}

// -----------------------
// --- Functions
// -----------------------

func ensureAndroidLicences(androidHome string, isLegacySDK bool) error {
	if !isLegacySDK {
		licensesCmd := command.New(filepath.Join(androidHome, "tools/bin/sdkmanager"), "--licenses")
		licensesCmd.SetStdin(bytes.NewReader([]byte(strings.Repeat("y\n", 1000))))
		if err := licensesCmd.Run(); err != nil {
			log.Warnf("Failed to install licenses using $(sdkmanager --licenses) command")
			log.Printf("Continue using legacy license installation...")
			fmt.Println()
		} else {
			return nil
		}
	}

	licenceMap := map[string]string{
		"android-sdk-license":           "8933bad161af4178b1185d1a37fbf41ea5269c55\n\nd56f5187479451eabf01fb78af6dfcb131a6481e",
		"android-googletv-license":      "\n601085b94cd77f0b54ff86406957099ebe79c4d6",
		"android-sdk-preview-license":   "\n84831b9409646a918e30573bab4c9c91346d8abd",
		"intel-android-extra-license":   "\nd975f751698a77b662f1254ddbeed3901e976f5a",
		"google-gdk-license":            "\n33b6a2b64607f11b759f320ef9dff4ae5c47d97a",
		"mips-android-sysimage-license": "\ne9acab5b5fbb560a72cfaecce8946896ff6aab9d",
	}

	licencesDir := filepath.Join(androidHome, "licenses")
	if exist, err := pathutil.IsDirExists(licencesDir); err != nil {
		return err
	} else if !exist {
		if err := os.MkdirAll(licencesDir, os.ModePerm); err != nil {
			return err
		}
	}

	for name, content := range licenceMap {
		pth := filepath.Join(licencesDir, name)

		if err := fileutil.WriteStringToFile(pth, content); err != nil {
			return err
		}
	}

	return nil
}

func failf(format string, v ...interface{}) {
	log.Errorf(format, v...)
	os.Exit(1)
}

// -----------------------
// --- Main
// -----------------------

func main() {
	// Input validation
	configs := createConfigsModelFromEnvs()

	fmt.Println()
	configs.print()

	if err := configs.validate(); err != nil {
		failf(err.Error())
	}

	fmt.Println()
	log.Infof("Preparation")

	// Set executable permission for gradlew
	log.Printf("Set executable permission for gradlew")
	if err := os.Chmod(configs.GradlewPath, 0770); err != nil {
		failf("Failed to set executable permission for gradlew, error: %s", err)
	}

	// Initialize Android SDK
	log.Printf("Initialize Android SDK")
	androidSdk, err := sdk.New(configs.AndroidHome)
	if err != nil {
		failf("Failed to initialize Android SDK, error: %s", err)
	}

	sdkManager, err := sdkmanager.New(androidSdk)
	if err != nil {
		failf("Failed to create SDK manager, error: %s", err)
	}

	// Ensure android licences
	log.Printf("Ensure android licences")
	if err := ensureAndroidLicences(configs.AndroidHome, sdkManager.IsLegacySDK()); err != nil {
		failf("Failed to ensure android licences, error: %s", err)
	}

	// Ensure required Android SDK components
	fmt.Println()
	log.Infof("Ensure required Android SDK components")

	retryCount := 0
	for true {
		gradleCmd := command.New("./gradlew", "dependencies")
		gradleCmd.SetStdin(strings.NewReader("y"))
		gradleCmd.SetDir(filepath.Dir(configs.GradlewPath))

		log.Printf("Searching for missing SDK components using:")
		log.Printf("$ %s", gradleCmd.PrintableCommandArgs())

		if out, err := gradleCmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
			reader := strings.NewReader(out)
			scanner := bufio.NewScanner(reader)

			missingSDKComponentFound := false

			for scanner.Scan() {
				line := scanner.Text()
				{
					// failed to find target with hash string 'android-22'
					targetPattern := `failed to find target with hash string 'android-(?P<version>.*)'\s*`
					targetRe := regexp.MustCompile(targetPattern)
					if matches := targetRe.FindStringSubmatch(line); len(matches) == 2 {
						missingSDKComponentFound = true

						targetVersion := "android-" + matches[1]

						log.Warnf("Missing platform version found: %s", targetVersion)

						platformComponent := sdkcomponent.Platform{
							Version: targetVersion,
						}

						cmd := sdkManager.InstallCommand(platformComponent)
						cmd.SetStdin(strings.NewReader("y"))

						log.Printf("Installing platform version using:")
						log.Printf("$ %s", cmd.PrintableCommandArgs())

						if err := retry.Times(1).Wait(time.Second).Try(func(attempt uint) error {
							if attempt > 0 {
								log.Warnf("Retrying...")
							}

							if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
								if attempt > 0 {
									return fmt.Errorf("output: %s, error: %s", out, err)
								}
								return err
							}

							return nil
						}); err != nil {
							log.Errorf("Failed to install platform:")
							failf("%s", err)
						}
					}
				}

				{
					// failed to find Build Tools revision 22.0.1
					buildToolsPattern := `failed to find Build Tools revision (?P<version>[0-9.]*)\s*`
					buildToolsRe := regexp.MustCompile(buildToolsPattern)
					if matches := buildToolsRe.FindStringSubmatch(line); len(matches) == 2 {
						missingSDKComponentFound = true

						buildToolsVersion := matches[1]

						log.Warnf("Missing build tools version found: %s", buildToolsVersion)

						buildToolsComponent := sdkcomponent.BuildTool{
							Version: buildToolsVersion,
						}

						cmd := sdkManager.InstallCommand(buildToolsComponent)
						cmd.SetStdin(strings.NewReader("y"))

						log.Printf("Installing build tools version using:")
						log.Printf("$ %s", cmd.PrintableCommandArgs())

						if err := retry.Times(1).Wait(time.Second).Try(func(attempt uint) error {
							if attempt > 0 {
								log.Warnf("Retrying...")
							}

							if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
								if attempt > 0 {
									return fmt.Errorf("output: %s, error: %s", out, err)
								}
								return err
							}

							return nil
						}); err != nil {
							log.Errorf("Failed to install build tools:")
							failf("%s", err)
						}
					}
				}

				{
					// Example: "Could not find com.android.support.constraint:constraint-layout:1.0.2."
					extrasPattern := `Could not find (?P<package>com\.android\.support\..*)\.`
					extrasRe := regexp.MustCompile(extrasPattern)
					if matches := extrasRe.FindStringSubmatch(line); len(matches) == 2 {
						missingSDKComponentFound = true

						log.Warnf("Missing extras library found: %s", matches[1])

						lib := matches[1]
						firstColon := strings.Index(lib, ":")
						lib = strings.Replace(lib[:firstColon], ".", ";", -1) + strings.Replace(lib[firstColon:], ":", ";", -1)

						extrasComponents := sdkcomponent.SupportLibraryInstallComponents()
						extrasComponents = append(extrasComponents, sdkcomponent.Extras{
							Provider:    "m2repository",
							PackageName: lib,
						})
						for _, e := range extrasComponents {
							cmd := sdkManager.InstallCommand(e)
							cmd.SetStdin(strings.NewReader("y"))

							log.Printf("Installing extras using:")
							log.Printf("$ %s", cmd.PrintableCommandArgs())

							if err := retry.Times(1).Wait(time.Second).Try(func(attempt uint) error {
								if attempt > 0 {
									log.Warnf("Retrying...")
								}

								if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil {
									if attempt > 0 {
										return fmt.Errorf("output: %s, error: %s", out, err)
									}
									return err
								}

								return nil
							}); err != nil {
								log.Errorf("Failed to install support library dependency:")
								failf("%s", err)
							}
						}
					}
				}
			}

			if err := scanner.Err(); err != nil {
				failf("failed to analyze gradle output, error: %s", err)
			}

			if !missingSDKComponentFound {
				if retryCount < 2 {
					log.Errorf("Failed to find missing components, retrying...")
					retryCount++
					continue
				}
				log.Printf(out)
				failf("%s", err)
			}
		} else {
			break
		}
	}

	fmt.Println()
	log.Donef("Required SDK components are installed")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Get it
type Data struct {
	WebhookEvent string
	User         struct {
		Name        string
		AvatarUrls  map[string]string
		DisplayName string
	}
	Issue struct {
		Self   string
		Key    string
		Fields struct {
			Issuetype struct {
				IconUrl string
				Name    string
			}
			Summary string
		}
	}
	Comment struct {
		Body string
	}
	Changelog struct {
		Items []struct {
			Field      string
			FromString string
			ToString   string
		}
	}
}

// Send it
type Message struct {
	Text     string
	Username string
	Icon_url string
}

func getMessage(request *http.Request) []byte {
	// Parse JSON from JIRA
	decoder := json.NewDecoder(request.Body)
	var data Data
	decoder.Decode(&data)

	// Get JIRA URL from "issue" section in JSON
	u, _ := url.Parse(data.Issue.Self)

	// Select action
	var action, comment string
	switch data.WebhookEvent {
	case "jira:issue_created":
		action = "created"
	case "jira:issue_updated":
		action = "updated"
	case "jira:issue_deleted":
		action = "deleted"
	}

	//Process new comment
	if len(data.Comment.Body) > 0 {
		comment = fmt.Sprintf("\nComment:\n>%s", data.Comment.Body)
	}

	// Process changelog
	var changelog string
	if len(data.Changelog.Items) > 0 {
		for _, item := range data.Changelog.Items {
			itemName := strings.ToUpper(string(item.Field[0])) + item.Field[1:]
			if item.FromString == "" {
				item.FromString = "None"
			}
			changelog += fmt.Sprintf(
				"\n%s: ~~%s~~ %s",
				itemName,
				item.FromString,
				item.ToString,
			)
		}
	}

	// Create message for Mattermost
	text := fmt.Sprintf(
		//Message format:
		//![user_icon](user_icon_link)[UserFirstName UserSecondName](user_link) commented task ![task_icon](task_icon link)[TSK-42](issue_link) "Test task"
		//Status: ~~Done~~ Finished
		//>Comment text
		"![user_icon](%s) [%s](%s://%s/secure/ViewProfile.jspa?name=%s) %s %s ![task_icon](%s) [%s](%s://%s/browse/%s) \"%s\"%s%s",
		data.User.AvatarUrls["16x16"],
		data.User.DisplayName,
		u.Scheme,
		u.Host,
		data.User.Name,
		action,
		strings.ToLower(data.Issue.Fields.Issuetype.Name),
		data.Issue.Fields.Issuetype.IconUrl,
		data.Issue.Key,
		u.Scheme,
		u.Host,
		data.Issue.Key,
		data.Issue.Fields.Summary,
		changelog,
		comment,
	)

	message := Message{
		Text:     text,
		Username: "JIRA",
		Icon_url: "https://design.atlassian.com/images/logo/favicon.png",
	}

	JSONMessage, _ := json.Marshal(message)

	return JSONMessage
}

func index(_ http.ResponseWriter, r *http.Request) {

	// Get mattermost URL
	mattermostHookURL := r.URL.Query().Get("mattermost_hook_url")

	if len(mattermostHookURL) > 0 {
		// Get message from JIRA JSON request
		message := getMessage(r)

		// Create http-client
		req, _ := http.NewRequest("POST", mattermostHookURL, bytes.NewBuffer(message))
		req.Header.Set("Content-Type", "application/json")

		// Send data to Mattermost
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
	}
}

func main() {

	port := os.Getenv("PORT")
	if port == "" {
		port = "5000"
	}
	http.HandleFunc("/", index)
	http.ListenAndServe(":"+port, nil)
}
<commit_msg>Fix for golint<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Data struct for JIRA JSON parsing
type Data struct {
	WebhookEvent string
	User         struct {
		Name        string
		AvatarUrls  map[string]string
		DisplayName string
	}
	Issue struct {
		Self   string
		Key    string
		Fields struct {
			Issuetype struct {
				IconUrl string
				Name    string
			}
			Summary string
		}
	}
	Comment struct {
		Body string
	}
	Changelog struct {
		Items []struct {
			Field      string
			FromString string
			ToString   string
		}
	}
}

// Message structure for Mattermost JSON creation
type Message struct {
	Text     string `json:"text"`
	Username string `json:"username"`
	IconUrl  string `json:"icon_url"`
}

func getMessage(request *http.Request) []byte {
	// Parse JSON from JIRA
	decoder := json.NewDecoder(request.Body)
	var data Data
	decoder.Decode(&data)

	// Get JIRA URL from "issue" section in JSON
	u, _ := url.Parse(data.Issue.Self)

	// Select action
	var action, comment string
	switch data.WebhookEvent {
	case "jira:issue_created":
		action = "created"
	case "jira:issue_updated":
		action = "updated"
	case "jira:issue_deleted":
		action = "deleted"
	}

	//Process new comment
	if len(data.Comment.Body) > 0 {
		comment = fmt.Sprintf("\nComment:\n>%s", data.Comment.Body)
	}

	// Process changelog
	var changelog string
	if len(data.Changelog.Items) > 0 {
		for _, item := range data.Changelog.Items {
			itemName := strings.ToUpper(string(item.Field[0])) + item.Field[1:]
			if item.FromString == "" {
				item.FromString = "None"
			}
			changelog += fmt.Sprintf(
				"\n%s: ~~%s~~ %s",
				itemName,
				item.FromString,
				item.ToString,
			)
		}
	}

	// Create message for Mattermost
	text := fmt.Sprintf(
		//Message format:
		//![user_icon](user_icon_link)[UserFirstName UserSecondName](user_link) commented task ![task_icon](task_icon link)[TSK-42](issue_link) "Test task"
		//Status: ~~Done~~ Finished
		//>Comment text
		"![user_icon](%s) [%s](%s://%s/secure/ViewProfile.jspa?name=%s) %s %s ![task_icon](%s) [%s](%s://%s/browse/%s) \"%s\"%s%s",
		data.User.AvatarUrls["16x16"],
		data.User.DisplayName,
		u.Scheme,
		u.Host,
		data.User.Name,
		action,
		strings.ToLower(data.Issue.Fields.Issuetype.Name),
		data.Issue.Fields.Issuetype.IconUrl,
		data.Issue.Key,
		u.Scheme,
		u.Host,
		data.Issue.Key,
		data.Issue.Fields.Summary,
		changelog,
		comment,
	)

	message := Message{
		Text:     text,
		Username: "JIRA",
		IconUrl:  "https://design.atlassian.com/images/logo/favicon.png",
	}

	JSONMessage, _ := json.Marshal(message)

	return JSONMessage
}

func index(_ http.ResponseWriter, r *http.Request) {

	// Get mattermost URL
	mattermostHookURL := r.URL.Query().Get("mattermost_hook_url")

	if len(mattermostHookURL) > 0 {
		// Get message from JIRA JSON request
		message := getMessage(r)

		// Create http-client
		req, _ := http.NewRequest("POST", mattermostHookURL, bytes.NewBuffer(message))
		req.Header.Set("Content-Type", "application/json")

		// Send data to Mattermost
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
	}
}

func main() {

	port := os.Getenv("PORT")
	if port == "" {
		port = "5000"
	}
	http.HandleFunc("/", index)
	http.ListenAndServe(":"+port, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	connStr         = kingpin.Arg("conn", "PostgreSQL connection string in URL format").Required().String()
	schema          = kingpin.Flag("schema", "PostgreSQL schema name").Default("public").String()
	pkgName         = kingpin.Flag("package", "package name").Default("main").String()
	typeMapFilePath = kingpin.Flag("typemap", "column type and go type map file path").String()
)

func main() {
	kingpin.Parse()

	conn, err := OpenDB(*connStr)
	if err != nil {
		log.Fatal(err)
	}

	src, err := PgCreateStruct(conn, *schema, *typeMapFilePath, *pkgName)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s", src)
}
<commit_msg>Add output file option<commit_after>package main

import (
	"log"
	"os"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	connStr         = kingpin.Arg("conn", "PostgreSQL connection string in URL format").Required().String()
	schema          = kingpin.Flag("schema", "PostgreSQL schema name").Default("public").Short('s').String()
	pkgName         = kingpin.Flag("package", "package name").Default("main").Short('p').String()
	typeMapFilePath = kingpin.Flag("typemap", "column type and go type map file path").Short('t').String()
	outFile         = kingpin.Flag("output", "output file path").Short('o').String()
)

func main() {
	kingpin.Parse()

	conn, err := OpenDB(*connStr)
	if err != nil {
		log.Fatal(err)
	}

	src, err := PgCreateStruct(conn, *schema, *typeMapFilePath, *pkgName)
	if err != nil {
		log.Fatal(err)
	}

	var out os.File
	if *outFile != "" {
		out, err := os.Create(*outFile)
		if err != nil {
			log.Fatal(err)
		}
		defer out.Close()

		if _, err := out.Write(src); err != nil {
			log.Fatal(err)
		}
		out.Sync()
		return
	}
	out = *os.Stdout
	if _, err := out.Write(src); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>