<commit_before>package route

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Parse loads a routing table from a set of route commands.
//
// The commands are parsed in order and order matters.
// Deleting a route that has not been created yet yields
// a different result than the other way around.
//
// Route commands can have the following form:
//
// route add <svc> <src> <dst> weight <w> tags "<t1>,<t2>,..."
//   - Add route for service svc from src to dst and assign weight and tags
//
// route add <svc> <src> <dst> weight <w>
//   - Add route for service svc from src to dst and assign weight
//
// route add <svc> <src> <dst> tags "<t1>,<t2>,..."
//   - Add route for service svc from src to dst and assign tags
//
// route add <svc> <src> <dst>
//   - Add route for service svc from src to dst
//
// route del <svc> <src> <dst>
//   - Remove route matching svc, src and dst
//
// route del <svc> <src>
//   - Remove all routes of services matching svc and src
//
// route del <svc>
//   - Remove all routes of service matching svc
//
// route weight <svc> <src> weight <w> tags "<t1>,<t2>,..."
//   - Route w% of traffic to all services matching svc, src and tags
//
// route weight <src> weight <w> tags "<t1>,<t2>,..."
//   - Route w% of traffic to all services matching src and tags
//
// route weight <svc> <src> weight <w>
//   - Route w% of traffic to all services matching svc and src
//
// route weight service host/path weight w tags "tag1,tag2"
//   - Route w% of traffic to all services matching service, host/path and tags
//
//     w is a float > 0 describing a percentage, e.g. 0.5 == 50%
//     w <= 0: means no fixed weighting. Traffic is evenly distributed
//     w > 0: route will receive n% of traffic. If sum(w) > 1 then w is normalized.
//     sum(w) >= 1: only matching services will receive traffic
//
//    Note that the total sum of traffic sent to all matching routes is w%.
//
func Parse(r io.Reader) (Table, error) {
	p := &parser{t: make(Table)}
	if err := p.parse(r); err != nil {
		return nil, err
	}
	return p.t, nil
}

// ParseFile loads a routing table from a file.
func ParseFile(path string) (Table, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return Parse(f)
}

// ParseString loads a routing table from a string.
func ParseString(s string) (Table, error) {
	return Parse(strings.NewReader(s))
}

type parser struct {
	t          Table
	lineNumber int
	line       string
}

func (p *parser) parse(r io.Reader) error {
	cmds := map[string]func(s string) error{
		"route add ":    p.routeAdd,
		"route del ":    p.routeDel,
		"route weight ": p.routeWeight,
	}

	sc := bufio.NewScanner(r)
	for sc.Scan() {
		p.lineNumber++
		p.line = strings.TrimSpace(sc.Text())
		if p.line == "" || strings.HasPrefix(p.line, "#") {
			continue
		}
		for cmd, fn := range cmds {
			if !strings.HasPrefix(p.line, cmd) {
				continue
			}
			if err := fn(p.line); err != nil {
				return err
			}
			break
		}
	}
	return nil
}

var (
	// route add <svc> <src> <dst> weight <w> tags "<t1>,<t2>,..."
	routeAddSvcWeightTags = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) weight (\S+) tags "([^"]*)"$`)

	// route add <svc> <src> <dst> weight <w>
	routeAddSvcWeight = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) weight (\S+)$`)

	// route add <svc> <src> <dst> tags "<t1>,<t2>,..."
	routeAddSvcTags = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) tags "([^"]*)"$`)

	// route add <svc> <src> <dst>
	routeAddSvc = regexp.MustCompile(`^route add (\S+) (\S+) (\S+)$`)
)

func (p *parser) routeAdd(s string) error {
	var svc, src, dst string
	var tags []string
	var w float64
	var err error

	// test most to least specific
	if m := routeAddSvcWeightTags.FindStringSubmatch(s); m != nil {
		svc, src, dst, tags = m[1], m[2], m[4], strings.Split(m[5], ",")
		w, err = p.parseWeight(m[3])
	} else if m := routeAddSvcWeight.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
		w, err = p.parseWeight(m[4])
	} else if m := routeAddSvcTags.FindStringSubmatch(s); m != nil {
		svc, src, dst, tags = m[1], m[2], m[3], strings.Split(m[4], ",")
	} else if m := routeAddSvc.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.AddRoute(svc, src, dst, w, tags)
	return nil
}

var (
	// route del <svc> <src> <dst>
	routeDelSvcSrcDst = regexp.MustCompile(`^route del (\S+) (\S+) (\S+)$`)

	// route del <svc> <src>
	routeDelSvcSrc = regexp.MustCompile(`^route del (\S+) (\S+)$`)

	// route del <svc>
	routeDelSvc = regexp.MustCompile(`^route del (\S+)$`)
)

func (p *parser) routeDel(s string) error {
	var svc, src, dst string
	var err error

	// test most to least specific
	if m := routeDelSvcSrcDst.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
	} else if m := routeDelSvcSrc.FindStringSubmatch(s); m != nil {
		svc, src = m[1], m[2]
	} else if m := routeDelSvc.FindStringSubmatch(s); m != nil {
		svc = m[1]
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.DelRoute(svc, src, dst)
	return nil
}

var (
	// route weight <svc> <src> weight <w> tags "<t1>,<t2>,..."'
	routeWeightSvcSrcTags = regexp.MustCompile(`^route weight (\S+) (\S+) weight (\S+) tags "([^"]*)"$`)

	// route weight <src> weight <w> tags "<t1>,<t2>,..."'
	routeWeightSrcTags = regexp.MustCompile(`^route weight (\S+) weight (\S+) tags "([^"]+)"$`)

	// route weight <svc> <src> weight <w>'
	routeWeightSvcSrc = regexp.MustCompile(`^route weight (\S+) (\S+) weight (\S+)$`)
)

func (p *parser) routeWeight(s string) error {
	var svc, src string
	var tags []string
	var w float64
	var err error

	// test most to least specific
	if m := routeWeightSvcSrcTags.FindStringSubmatch(s); m != nil {
		svc, src, tags = m[1], m[2], strings.Split(m[4], ",")
		w, err = p.parseWeight(m[3])
	} else if m := routeWeightSvcSrc.FindStringSubmatch(s); m != nil {
		svc, src = m[1], m[2]
		w, err = p.parseWeight(m[3])
	} else if m := routeWeightSrcTags.FindStringSubmatch(s); m != nil {
		src, tags = m[1], strings.Split(m[3], ",")
		w, err = p.parseWeight(m[2])
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.AddRouteWeight(svc, src, w, tags)
	return nil
}

func (p *parser) parseWeight(s string) (float64, error) {
	n, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, p.errorf("invalid weight: %s", s)
	}
	return n, nil
}

func (p *parser) syntaxError() error {
	return fmt.Errorf("route: line %d: syntax error in %s", p.lineNumber, p.line)
}

func (p *parser) errorf(msg string, args ...string) error {
	return fmt.Errorf("route: line %d: %s", p.lineNumber, fmt.Sprintf(msg, args))
}
<commit_msg>Detect syntax error for invalid command.<commit_after>package route

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Parse loads a routing table from a set of route commands.
//
// The commands are parsed in order and order matters.
// Deleting a route that has not been created yet yields
// a different result than the other way around.
//
// Route commands can have the following form:
//
// route add <svc> <src> <dst> weight <w> tags "<t1>,<t2>,..."
//   - Add route for service svc from src to dst and assign weight and tags
//
// route add <svc> <src> <dst> weight <w>
//   - Add route for service svc from src to dst and assign weight
//
// route add <svc> <src> <dst> tags "<t1>,<t2>,..."
//   - Add route for service svc from src to dst and assign tags
//
// route add <svc> <src> <dst>
//   - Add route for service svc from src to dst
//
// route del <svc> <src> <dst>
//   - Remove route matching svc, src and dst
//
// route del <svc> <src>
//   - Remove all routes of services matching svc and src
//
// route del <svc>
//   - Remove all routes of service matching svc
//
// route weight <svc> <src> weight <w> tags "<t1>,<t2>,..."
//   - Route w% of traffic to all services matching svc, src and tags
//
// route weight <src> weight <w> tags "<t1>,<t2>,..."
//   - Route w% of traffic to all services matching src and tags
//
// route weight <svc> <src> weight <w>
//   - Route w% of traffic to all services matching svc and src
//
// route weight service host/path weight w tags "tag1,tag2"
//   - Route w% of traffic to all services matching service, host/path and tags
//
//     w is a float > 0 describing a percentage, e.g. 0.5 == 50%
//     w <= 0: means no fixed weighting. Traffic is evenly distributed
//     w > 0: route will receive n% of traffic. If sum(w) > 1 then w is normalized.
//     sum(w) >= 1: only matching services will receive traffic
//
//    Note that the total sum of traffic sent to all matching routes is w%.
//
func Parse(r io.Reader) (Table, error) {
	p := &parser{t: make(Table)}
	if err := p.parse(r); err != nil {
		return nil, err
	}
	return p.t, nil
}

// ParseFile loads a routing table from a file.
func ParseFile(path string) (Table, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return Parse(f)
}

// ParseString loads a routing table from a string.
func ParseString(s string) (Table, error) {
	return Parse(strings.NewReader(s))
}

type parser struct {
	t          Table
	lineNumber int
	line       string
}

type cmdFn func(s string) error

func (p *parser) parse(r io.Reader) error {
	cmds := map[string]cmdFn{
		"route add ":    p.routeAdd,
		"route del ":    p.routeDel,
		"route weight ": p.routeWeight,
	}

	sc := bufio.NewScanner(r)
	for sc.Scan() {
		p.lineNumber++
		p.line = strings.TrimSpace(sc.Text())
		if p.line == "" || strings.HasPrefix(p.line, "#") {
			continue
		}
		var fn cmdFn
		for cmd := range cmds {
			if strings.HasPrefix(p.line, cmd) {
				fn = cmds[cmd]
				break
			}
		}
		if fn == nil {
			return p.syntaxError()
		}
		if err := fn(p.line); err != nil {
			return err
		}
	}
	return nil
}

var (
	// route add <svc> <src> <dst> weight <w> tags "<t1>,<t2>,..."
	routeAddSvcWeightTags = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) weight (\S+) tags "([^"]*)"$`)

	// route add <svc> <src> <dst> weight <w>
	routeAddSvcWeight = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) weight (\S+)$`)

	// route add <svc> <src> <dst> tags "<t1>,<t2>,..."
	routeAddSvcTags = regexp.MustCompile(`^route add (\S+) (\S+) (\S+) tags "([^"]*)"$`)

	// route add <svc> <src> <dst>
	routeAddSvc = regexp.MustCompile(`^route add (\S+) (\S+) (\S+)$`)
)

func (p *parser) routeAdd(s string) error {
	var svc, src, dst string
	var tags []string
	var w float64
	var err error

	// test most to least specific
	if m := routeAddSvcWeightTags.FindStringSubmatch(s); m != nil {
		svc, src, dst, tags = m[1], m[2], m[4], strings.Split(m[5], ",")
		w, err = p.parseWeight(m[3])
	} else if m := routeAddSvcWeight.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
		w, err = p.parseWeight(m[4])
	} else if m := routeAddSvcTags.FindStringSubmatch(s); m != nil {
		svc, src, dst, tags = m[1], m[2], m[3], strings.Split(m[4], ",")
	} else if m := routeAddSvc.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.AddRoute(svc, src, dst, w, tags)
	return nil
}

var (
	// route del <svc> <src> <dst>
	routeDelSvcSrcDst = regexp.MustCompile(`^route del (\S+) (\S+) (\S+)$`)

	// route del <svc> <src>
	routeDelSvcSrc = regexp.MustCompile(`^route del (\S+) (\S+)$`)

	// route del <svc>
	routeDelSvc = regexp.MustCompile(`^route del (\S+)$`)
)

func (p *parser) routeDel(s string) error {
	var svc, src, dst string
	var err error

	// test most to least specific
	if m := routeDelSvcSrcDst.FindStringSubmatch(s); m != nil {
		svc, src, dst = m[1], m[2], m[3]
	} else if m := routeDelSvcSrc.FindStringSubmatch(s); m != nil {
		svc, src = m[1], m[2]
	} else if m := routeDelSvc.FindStringSubmatch(s); m != nil {
		svc = m[1]
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.DelRoute(svc, src, dst)
	return nil
}

var (
	// route weight <svc> <src> weight <w> tags "<t1>,<t2>,..."'
	routeWeightSvcSrcTags = regexp.MustCompile(`^route weight (\S+) (\S+) weight (\S+) tags "([^"]*)"$`)

	// route weight <src> weight <w> tags "<t1>,<t2>,..."'
	routeWeightSrcTags = regexp.MustCompile(`^route weight (\S+) weight (\S+) tags "([^"]+)"$`)

	// route weight <svc> <src> weight <w>'
	routeWeightSvcSrc = regexp.MustCompile(`^route weight (\S+) (\S+) weight (\S+)$`)
)

func (p *parser) routeWeight(s string) error {
	var svc, src string
	var tags []string
	var w float64
	var err error

	// test most to least specific
	if m := routeWeightSvcSrcTags.FindStringSubmatch(s); m != nil {
		svc, src, tags = m[1], m[2], strings.Split(m[4], ",")
		w, err = p.parseWeight(m[3])
	} else if m := routeWeightSvcSrc.FindStringSubmatch(s); m != nil {
		svc, src = m[1], m[2]
		w, err = p.parseWeight(m[3])
	} else if m := routeWeightSrcTags.FindStringSubmatch(s); m != nil {
		src, tags = m[1], strings.Split(m[3], ",")
		w, err = p.parseWeight(m[2])
	} else {
		err = p.syntaxError()
	}

	if err != nil {
		return err
	}

	p.t.AddRouteWeight(svc, src, w, tags)
	return nil
}

func (p *parser) parseWeight(s string) (float64, error) {
	n, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, p.errorf("invalid weight: %s", s)
	}
	return n, nil
}

func (p *parser) syntaxError() error {
	return fmt.Errorf("route: line %d: syntax error in %s", p.lineNumber, p.line)
}

func (p *parser) errorf(msg string, args ...string) error {
	return fmt.Errorf("route: line %d: %s", p.lineNumber, fmt.Sprintf(msg, args))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
	"github.com/mat/besticon/lettericon"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "" || r.URL.Path == "/" {
		renderHTMLTemplate(w, 200, indexHTML, nil)
	} else {
		renderHTMLTemplate(w, 404, notFoundHTML, nil)
	}
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	finder := besticon.IconFinder{}

	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	e, icons := finder.FetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) == 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue("url")
	if len(url) == 0 {
		writeAPIError(w, 400, errors.New("need url parameter"), true)
		return
	}

	size := r.FormValue("size")
	if size == "" {
		writeAPIError(w, 400, errors.New("need size parameter"), true)
		return
	}
	minSize, err := strconv.Atoi(size)
	if err != nil || minSize < 0 || minSize > 1000 {
		writeAPIError(w, 400, errors.New("bad size parameter"), true)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	err, _ = finder.FetchIcons(url)
	if err != nil {
		writeAPIError(w, 404, err, true)
		return
	}

	icon := finder.IconWithMinSize(minSize)
	if icon != nil {
		http.Redirect(w, r, icon.URL, 302)
		return
	}

	iconColor := finder.MainColorForIcons()
	letter := lettericon.MainLetterFromURL(url)
	redirectPath := lettericon.IconPath(letter, size, iconColor)
	http.Redirect(w, r, redirectPath, 302)
}

func popularHandler(w http.ResponseWriter, r *http.Request) {
	iconSize, err := strconv.Atoi(r.FormValue("iconsize"))
	if iconSize > 1000 || iconSize < 10 || err != nil {
		iconSize = 120
	}

	pageInfo := struct {
		URLs        []string
		IconSize    int
		DisplaySize int
	}{
		besticon.PopularSites,
		iconSize,
		iconSize / 2,
	}
	renderHTMLTemplate(w, 200, popularHTML, pageInfo)
}

const (
	urlParam    = "url"
	prettyParam = "pretty"
	maxAge      = "max_age"
)

const defaultMaxAge = time.Duration(604800) * time.Second // 7 days

func alliconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL, true)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	e, icons := finder.FetchIcons(url)
	if e != nil {
		writeAPIError(w, 404, e, true)
		return
	}

	pretty, err := strconv.ParseBool(r.FormValue(prettyParam))
	prettyPrint := (err == nil) && pretty

	writeAPIIcons(w, url, icons, prettyPrint)
}

func lettericonHandler(w http.ResponseWriter, r *http.Request) {
	charParam, col, size := lettericon.ParseIconPath(r.URL.Path)

	w.Header().Add("Content-Type", "image/png")
	lettericon.Render(charParam, col, size, w)
}

func obsoleteAPIHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("i_am_feeling_lucky") == "yes" {
		http.Redirect(w, r, fmt.Sprintf("/icon?size=120&%s", r.URL.RawQuery), 302)
	} else {
		http.Redirect(w, r, fmt.Sprintf("/allicons.json?%s", r.URL.RawQuery), 302)
	}
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error, pretty bool) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}

	if pretty {
		renderJSONResponsePretty(w, httpStatus, data)
	} else {
		renderJSONResponse(w, httpStatus, data)
	}
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon, pretty bool) {
	// Don't return whole image data
	newIcons := []besticon.Icon{}
	for _, ico := range icons {
		newIcon := ico
		newIcon.ImageData = nil
		newIcons = append(newIcons, newIcon)
	}

	data := &struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		newIcons,
	}

	if pretty {
		renderJSONResponsePretty(w, 200, data)
	} else {
		renderJSONResponse(w, 200, data)
	}
}

const (
	contentType     = "Content-Type"
	applicationJSON = "application/json"
	cacheControl    = "Cache-Control"
)

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

func renderJSONResponsePretty(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	b, _ := json.MarshalIndent(data, "", "  ")
	w.Write(b)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add(contentType, "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port string) {
	registerGzipHandler("/", indexHandler)
	registerGzipHandler("/icons", iconsHandler)
	registerHandler("/icon", iconHandler)
	registerGzipHandler("/popular", popularHandler)
	registerGzipHandler("/allicons.json", alliconsHandler)
	registerHandler("/lettericons/", lettericonHandler)
	registerHandler("/api/icons", obsoleteAPIHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css", oneYear)
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css", oneYear)
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css", oneYear)

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg", oneYear)
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico", oneYear)
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png", oneYear)
	serveAsset("/robots.txt", "besticon/iconserver/assets/robots.txt", nocache)
	serveAsset("/test-lettericons", "besticon/iconserver/assets/test-lettericons.html", nocache)

	addr := "0.0.0.0:" + port
	logger.Print("Starting server on ", addr, "...")
	e := http.ListenAndServe(addr, newLoggingMux())
	if e != nil {
		logger.Fatalf("cannot start server: %s\n", e)
	}
}

const (
	oneYear = 365 * 24 * 3600
	nocache = -1
)

func serveAsset(path string, assetPath string, maxAgeSeconds int) {
	registerGzipHandler(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		if maxAgeSeconds != nocache {
			w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", maxAgeSeconds))
		}

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(),
			bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func registerHandler(path string, f http.HandlerFunc) {
	http.Handle(path, newExpvarHandler(path, f))
}

func registerGzipHandler(path string, f http.HandlerFunc) {
	http.Handle(path, gziphandler.GzipHandler(newExpvarHandler(path, f)))
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	startServer(port)
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	popularHTML = templateFromAsset("besticon/iconserver/assets/popular.html", "popular.html")
	notFoundHTML = templateFromAsset("besticon/iconserver/assets/not_found.html", "not_found.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var popularHTML *template.Template
var notFoundHTML *template.Template

var funcMap = template.FuncMap{
	"ImgWidth": imgWidth,
}

func imgWidth(i *besticon.Icon) int {
	return i.Width / 2.0
}

func init() {
	besticon.SetCacheMaxSize(128)

	expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
	expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
	expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
	expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
	expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
}
<commit_msg>Print version on server start<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"expvar"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/NYTimes/gziphandler"
	"github.com/mat/besticon/besticon"
	"github.com/mat/besticon/besticon/iconserver/assets"
	"github.com/mat/besticon/lettericon"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "" || r.URL.Path == "/" {
		renderHTMLTemplate(w, 200, indexHTML, nil)
	} else {
		renderHTMLTemplate(w, 404, notFoundHTML, nil)
	}
}

func iconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		http.Redirect(w, r, "/", 302)
		return
	}

	finder := besticon.IconFinder{}

	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	e, icons := finder.FetchIcons(url)
	switch {
	case e != nil:
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: e})
	case len(icons) == 0:
		errNoIcons := errors.New("this poor site has no icons at all :-(")
		renderHTMLTemplate(w, 404, iconsHTML, pageInfo{URL: url, Error: errNoIcons})
	default:
		renderHTMLTemplate(w, 200, iconsHTML, pageInfo{Icons: icons, URL: url})
	}
}

func iconHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue("url")
	if len(url) == 0 {
		writeAPIError(w, 400, errors.New("need url parameter"), true)
		return
	}

	size := r.FormValue("size")
	if size == "" {
		writeAPIError(w, 400, errors.New("need size parameter"), true)
		return
	}
	minSize, err := strconv.Atoi(size)
	if err != nil || minSize < 0 || minSize > 1000 {
		writeAPIError(w, 400, errors.New("bad size parameter"), true)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	err, _ = finder.FetchIcons(url)
	if err != nil {
		writeAPIError(w, 404, err, true)
		return
	}

	icon := finder.IconWithMinSize(minSize)
	if icon != nil {
		http.Redirect(w, r, icon.URL, 302)
		return
	}

	iconColor := finder.MainColorForIcons()
	letter := lettericon.MainLetterFromURL(url)
	redirectPath := lettericon.IconPath(letter, size, iconColor)
	http.Redirect(w, r, redirectPath, 302)
}

func popularHandler(w http.ResponseWriter, r *http.Request) {
	iconSize, err := strconv.Atoi(r.FormValue("iconsize"))
	if iconSize > 1000 || iconSize < 10 || err != nil {
		iconSize = 120
	}

	pageInfo := struct {
		URLs        []string
		IconSize    int
		DisplaySize int
	}{
		besticon.PopularSites,
		iconSize,
		iconSize / 2,
	}
	renderHTMLTemplate(w, 200, popularHTML, pageInfo)
}

const (
	urlParam    = "url"
	prettyParam = "pretty"
	maxAge      = "max_age"
)

const defaultMaxAge = time.Duration(604800) * time.Second // 7 days

func alliconsHandler(w http.ResponseWriter, r *http.Request) {
	url := r.FormValue(urlParam)
	if len(url) == 0 {
		errMissingURL := errors.New("need url query parameter")
		writeAPIError(w, 400, errMissingURL, true)
		return
	}

	finder := besticon.IconFinder{}
	formats := r.FormValue("formats")
	if formats != "" {
		finder.FormatsAllowed = strings.Split(r.FormValue("formats"), ",")
	}

	e, icons := finder.FetchIcons(url)
	if e != nil {
		writeAPIError(w, 404, e, true)
		return
	}

	pretty, err := strconv.ParseBool(r.FormValue(prettyParam))
	prettyPrint := (err == nil) && pretty

	writeAPIIcons(w, url, icons, prettyPrint)
}

func lettericonHandler(w http.ResponseWriter, r *http.Request) {
	charParam, col, size := lettericon.ParseIconPath(r.URL.Path)

	w.Header().Add("Content-Type", "image/png")
	lettericon.Render(charParam, col, size, w)
}

func obsoleteAPIHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("i_am_feeling_lucky") == "yes" {
		http.Redirect(w, r, fmt.Sprintf("/icon?size=120&%s", r.URL.RawQuery), 302)
	} else {
		http.Redirect(w, r, fmt.Sprintf("/allicons.json?%s", r.URL.RawQuery), 302)
	}
}

func writeAPIError(w http.ResponseWriter, httpStatus int, e error, pretty bool) {
	data := struct {
		Error string `json:"error"`
	}{
		e.Error(),
	}

	if pretty {
		renderJSONResponsePretty(w, httpStatus, data)
	} else {
		renderJSONResponse(w, httpStatus, data)
	}
}

func writeAPIIcons(w http.ResponseWriter, url string, icons []besticon.Icon, pretty bool) {
	// Don't return whole image data
	newIcons := []besticon.Icon{}
	for _, ico := range icons {
		newIcon := ico
		newIcon.ImageData = nil
		newIcons = append(newIcons, newIcon)
	}

	data := &struct {
		URL   string          `json:"url"`
		Icons []besticon.Icon `json:"icons"`
	}{
		url,
		newIcons,
	}

	if pretty {
		renderJSONResponsePretty(w, 200, data)
	} else {
		renderJSONResponse(w, 200, data)
	}
}

const (
	contentType     = "Content-Type"
	applicationJSON = "application/json"
	cacheControl    = "Cache-Control"
)

func renderJSONResponse(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	enc := json.NewEncoder(w)
	enc.Encode(data)
}

func renderJSONResponsePretty(w http.ResponseWriter, httpStatus int, data interface{}) {
	w.Header().Add(contentType, applicationJSON)
	w.WriteHeader(httpStatus)
	b, _ := json.MarshalIndent(data, "", "  ")
	w.Write(b)
}

type pageInfo struct {
	URL   string
	Icons []besticon.Icon
	Error error
}

func (pi pageInfo) Host() string {
	u := pi.URL
	url, _ := url.Parse(u)
	if url != nil && url.Host != "" {
		return url.Host
	}
	return pi.URL
}

func (pi pageInfo) Best() string {
	if len(pi.Icons) > 0 {
		best := pi.Icons[0]
		return best.URL
	}
	return ""
}

func renderHTMLTemplate(w http.ResponseWriter, httpStatus int, templ *template.Template, data interface{}) {
	w.Header().Add(contentType, "text/html; charset=utf-8")
	w.WriteHeader(httpStatus)

	err := templ.Execute(w, data)
	if err != nil {
		err = fmt.Errorf("server: could not generate output: %s", err)
		logger.Print(err)
		w.Write([]byte(err.Error()))
	}
}

func startServer(port string) {
	registerGzipHandler("/", indexHandler)
	registerGzipHandler("/icons", iconsHandler)
	registerHandler("/icon", iconHandler)
	registerGzipHandler("/popular", popularHandler)
	registerGzipHandler("/allicons.json", alliconsHandler)
	registerHandler("/lettericons/", lettericonHandler)
	registerHandler("/api/icons", obsoleteAPIHandler)

	serveAsset("/pure-0.5.0-min.css", "besticon/iconserver/assets/pure-0.5.0-min.css", oneYear)
	serveAsset("/grids-responsive-0.5.0-min.css", "besticon/iconserver/assets/grids-responsive-0.5.0-min.css", oneYear)
	serveAsset("/main-min.css", "besticon/iconserver/assets/main-min.css", oneYear)

	serveAsset("/icon.svg", "besticon/iconserver/assets/icon.svg", oneYear)
	serveAsset("/favicon.ico", "besticon/iconserver/assets/favicon.ico", oneYear)
	serveAsset("/apple-touch-icon.png", "besticon/iconserver/assets/apple-touch-icon.png", oneYear)
	serveAsset("/robots.txt", "besticon/iconserver/assets/robots.txt", nocache)
	serveAsset("/test-lettericons", "besticon/iconserver/assets/test-lettericons.html", nocache)

	addr := "0.0.0.0:" + port
	logger.Print("Starting server on ", addr, "...")
	e := http.ListenAndServe(addr, newLoggingMux())
	if e != nil {
		logger.Fatalf("cannot start server: %s\n", e)
	}
}

const (
	oneYear = 365 * 24 * 3600
	nocache = -1
)

func serveAsset(path string, assetPath string, maxAgeSeconds int) {
	registerGzipHandler(path, func(w http.ResponseWriter, r *http.Request) {
		assetInfo, err := assets.AssetInfo(assetPath)
		if err != nil {
			panic(err)
		}

		if maxAgeSeconds != nocache {
			w.Header().Add(cacheControl, fmt.Sprintf("max-age=%d", maxAgeSeconds))
		}

		http.ServeContent(w, r, assetInfo.Name(), assetInfo.ModTime(),
			bytes.NewReader(assets.MustAsset(assetPath)))
	})
}

func registerHandler(path string, f http.HandlerFunc) {
	http.Handle(path, newExpvarHandler(path, f))
}

func registerGzipHandler(path string, f http.HandlerFunc) {
	http.Handle(path, gziphandler.GzipHandler(newExpvarHandler(path, f)))
}

func main() {
	fmt.Printf("iconserver %s (%s) - http://icons.better-idea.org\n", besticon.VersionString, runtime.Version())
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	startServer(port)
}

func init() {
	indexHTML = templateFromAsset("besticon/iconserver/assets/index.html", "index.html")
	iconsHTML = templateFromAsset("besticon/iconserver/assets/icons.html", "icons.html")
	popularHTML = templateFromAsset("besticon/iconserver/assets/popular.html", "popular.html")
	notFoundHTML = templateFromAsset("besticon/iconserver/assets/not_found.html", "not_found.html")
}

func templateFromAsset(assetPath, templateName string) *template.Template {
	bytes := assets.MustAsset(assetPath)
	return template.Must(template.New(templateName).Funcs(funcMap).Parse(string(bytes)))
}

var indexHTML *template.Template
var iconsHTML *template.Template
var popularHTML *template.Template
var notFoundHTML *template.Template

var funcMap = template.FuncMap{
	"ImgWidth": imgWidth,
}

func imgWidth(i *besticon.Icon) int {
	return i.Width / 2.0
}

func init() {
	besticon.SetCacheMaxSize(128)

	expvar.Publish("cacheBytes", expvar.Func(func() interface{} { return besticon.GetCacheStats().Bytes }))
	expvar.Publish("cacheItems", expvar.Func(func() interface{} { return besticon.GetCacheStats().Items }))
	expvar.Publish("cacheGets", expvar.Func(func() interface{} { return besticon.GetCacheStats().Gets }))
	expvar.Publish("cacheHits", expvar.Func(func() interface{} { return besticon.GetCacheStats().Hits }))
	expvar.Publish("cacheEvictions", expvar.Func(func() interface{} { return besticon.GetCacheStats().Evictions }))
}
<|endoftext|>


<commit_before>package sourcegraph

import (
	"errors"
	"fmt"

	"strconv"
)

func (s *BuildSpec) RouteVars() map[string]string {
	m := s.Repo.RouteVars()
	m["Attempt"] = fmt.Sprintf("%d", s.Attempt)
	m["CommitID"] = fmt.Sprintf("%s", s.CommitID)
	return m
}

func (s *TaskSpec) RouteVars() map[string]string {
	v := s.BuildSpec.RouteVars()
	v["TaskID"] = fmt.Sprintf("%d", s.TaskID)
	return v
}

func (b *Build) Spec() BuildSpec {
	return BuildSpec{
		Repo:     RepoSpec{URI: b.Repo},
		Attempt:  b.Attempt,
		CommitID: b.CommitID,
	}
}

// IDString returns a succinct string that uniquely identifies this build.
func (b BuildSpec) IDString() string {
	return fmt.Sprintf("B%s/%s/%d", b.Repo.URI, b.CommitID, b.Attempt)
}

// Build task ops.
const ImportTaskOp = "import"

func (t *BuildTask) Spec() TaskSpec {
	return TaskSpec{
		BuildSpec: BuildSpec{
			Repo: RepoSpec{
				URI: t.Repo,
			},
			Attempt:  t.Attempt,
			CommitID: t.CommitID,
		},
		TaskID: t.TaskID,
	}
}

// IDString returns a succinct string that uniquely identifies this build task.
func (t TaskSpec) IDString() string {
	return t.BuildSpec.IDString() + "-T" + strconv.FormatInt(t.TaskID, 36)
}

var ErrBuildNotFound = errors.New("build not found")
<commit_msg>Removed B from IDString<commit_after>package sourcegraph

import (
	"errors"
	"fmt"

	"strconv"
)

func (s *BuildSpec) RouteVars() map[string]string {
	m := s.Repo.RouteVars()
	m["Attempt"] = fmt.Sprintf("%d", s.Attempt)
	m["CommitID"] = fmt.Sprintf("%s", s.CommitID)
	return m
}

func (s *TaskSpec) RouteVars() map[string]string {
	v := s.BuildSpec.RouteVars()
	v["TaskID"] = fmt.Sprintf("%d", s.TaskID)
	return v
}

func (b *Build) Spec() BuildSpec {
	return BuildSpec{
		Repo:     RepoSpec{URI: b.Repo},
		Attempt:  b.Attempt,
		CommitID: b.CommitID,
	}
}

// IDString returns a succinct string that uniquely identifies this build.
func (b BuildSpec) IDString() string {
	return fmt.Sprintf("%s/%s/%d", b.Repo.URI, b.CommitID, b.Attempt)
}

// Build task ops.
const ImportTaskOp = "import"

func (t *BuildTask) Spec() TaskSpec {
	return TaskSpec{
		BuildSpec: BuildSpec{
			Repo: RepoSpec{
				URI: t.Repo,
			},
			Attempt:  t.Attempt,
			CommitID: t.CommitID,
		},
		TaskID: t.TaskID,
	}
}

// IDString returns a succinct string that uniquely identifies this build task.
func (t TaskSpec) IDString() string {
	return t.BuildSpec.IDString() + "-T" + strconv.FormatInt(t.TaskID, 36)
}

var ErrBuildNotFound = errors.New("build not found")
<|endoftext|>


<commit_before>package client

import (
	"fmt"

	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/build_db"
)

type Build build_db.Build

type BuildsService interface {
	Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

type buildsService struct {
	client *Client
}

var _ BuildsService = &buildsService{}

type BuildSpec struct {
	Repo RepositorySpec
	BID  int64
}

type BuildGetOptions struct{}

func (s *buildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuild, map[string]string{"RepoURI": build.Repo.URI, "BID": fmt.Sprintf("%d", build.BID)}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var build_ *Build
	resp, err := s.client.Do(req, &build_)
	if err != nil {
		return nil, resp, err
	}

	return build_, nil, nil
}

type BuildListByRepositoryOptions struct{}

func (s *buildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuilds, map[string]string{"RepoURI": repo.URI}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var builds []*Build
	resp, err := s.client.Do(req, &builds)
	if err != nil {
		return nil, resp, err
	}

	return builds, resp, nil
}

type MockBuildsService struct {
	Get_              func(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository_ func(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

var _ BuildsService = MockBuildsService{}

func (s MockBuildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(build, opt)
}
func (s MockBuildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	if s.ListByRepository_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.ListByRepository_(repo, opt)
}
<commit_msg>builds api<commit_after>package client

import (
	"errors"
	"fmt"
	"time"

	"sourcegraph.com/sourcegraph/api_router"
	"sourcegraph.com/sourcegraph/srcgraph/db_common"
	"sourcegraph.com/sourcegraph/srcgraph/repo"
)

type BuildsService interface {
	Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

type buildsService struct {
	client *Client
}

var _ BuildsService = &buildsService{}

type BuildSpec struct {
	Repo RepositorySpec
	BID  int64
}

// A Build represents a scheduled, completed, or failed repository analysis and
// import job.
type Build struct {
	BID       int64
	Repo      repo.RID
	CommitID  string             `db:"commit_id"`
	CreatedAt time.Time          `db:"created_at"`
	StartedAt db_common.NullTime `db:"started_at"`
	EndedAt   db_common.NullTime `db:"ended_at"`
	Success   bool
	Failure   bool
}

var ErrBuildNotFound = errors.New("build not found")

type BuildGetOptions struct{}

func (s *buildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuild, map[string]string{"RepoURI": build.Repo.URI, "BID": fmt.Sprintf("%d", build.BID)}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var build_ *Build
	resp, err := s.client.Do(req, &build_)
	if err != nil {
		return nil, resp, err
	}

	return build_, nil, nil
}

type BuildListByRepositoryOptions struct{ ListOptions }

func (s *buildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	url, err := s.client.url(api_router.RepositoryBuilds, map[string]string{"RepoURI": repo.URI}, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := s.client.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, nil, err
	}

	var builds []*Build
	resp, err := s.client.Do(req, &builds)
	if err != nil {
		return nil, resp, err
	}

	return builds, resp, nil
}

type MockBuildsService struct {
	Get_              func(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error)
	ListByRepository_ func(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error)
}

var _ BuildsService = MockBuildsService{}

func (s MockBuildsService) Get(build BuildSpec, opt *BuildGetOptions) (*Build, Response, error) {
	if s.Get_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.Get_(build, opt)
}
func (s MockBuildsService) ListByRepository(repo RepositorySpec, opt *BuildListByRepositoryOptions) ([]*Build, Response, error) {
	if s.ListByRepository_ == nil {
		return nil, &HTTPResponse{}, nil
	}
	return s.ListByRepository_(repo, opt)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Unconvert removes redundant type conversions from Go packages.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"runtime/pprof"
	"sort"
	"sync"
	"unicode"

	"golang.org/x/tools/container/intsets"
	"golang.org/x/tools/go/loader"
)

// Unnecessary conversions are identified by the position
// of their left parenthesis within a source file.

func apply(file string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	// Note: We modify edits during the walk.
	v := editor{edits: edits, file: fset.File(f.Package)}
	ast.Walk(&v, f)
	if !edits.IsEmpty() {
		log.Printf("%s: missing edits %s", file, edits)
	}

	// TODO(mdempsky): Write to temporary file and rename.
	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	if err != nil {
		log.Fatal(err)
	}

	err = ioutil.WriteFile(file, buf.Bytes(), 0)
	if err != nil {
		log.Fatal(err)
	}
}

type editor struct {
	edits *intsets.Sparse
	file  *token.File
}

func (e *editor) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	v := reflect.ValueOf(n).Elem()
	for i, n := 0, v.NumField(); i < n; i++ {
		switch f := v.Field(i).Addr().Interface().(type) {
		case *ast.Expr:
			e.rewrite(f)
		case *[]ast.Expr:
			for i := range *f {
				e.rewrite(&(*f)[i])
			}
		}
	}
	return e
}

func (e *editor) rewrite(f *ast.Expr) {
	n, ok := (*f).(*ast.CallExpr)
	if !ok {
		return
	}
	off := e.file.Offset(n.Lparen)
	if !e.edits.Has(off) {
		return
	}
	*f = n.Args[0]
	e.edits.Remove(off)
}

func print(name string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	buf, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatal(err)
	}

	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, name, buf, 0)
	if err != nil {
		log.Fatal(err)
	}

	file := fset.File(f.Package)
	for _, p := range edits.AppendTo(nil) {
		pos := file.Position(file.Pos(p))
		fmt.Printf("%s:%d:%d:\n", pos.Filename, pos.Line, pos.Column)
		line := lineForOffset(buf, pos.Offset)
		fmt.Printf("%s\n", line)
		fmt.Printf("%s^\n", rub(line[:pos.Column-1]))
	}
}

func rub(buf []byte) []byte {
	// TODO(mdempsky): Handle combining characters?
	// TODO(mdempsky): Handle East Asian wide characters?
	var res bytes.Buffer
	for _, c := range string(buf) {
		if !unicode.IsSpace(c) {
			c = ' '
		}
		res.WriteRune(c)
	}
	return res.Bytes()
}

func lineForOffset(buf []byte, off int) []byte {
	sol := bytes.LastIndexByte(buf[:off], '\n')
	if sol < 0 {
		sol = 0
	} else {
		sol += 1
	}
	eol := bytes.IndexByte(buf[off:], '\n')
	if eol < 0 {
		eol = len(buf)
	} else {
		eol += off
	}
	return buf[sol:eol]
}

var (
	flagAll        = flag.Bool("all", false, "type check all GOOS and GOARCH combinations")
	flagApply      = flag.Bool("apply", false, "apply edits to source files")
	flagCPUProfile = flag.String("cpuprofile", "", "write CPU profile to file")
)

func main() {
	flag.Parse()

	if *flagCPUProfile != "" {
		f, err := os.Create(*flagCPUProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var m map[string]*intsets.Sparse
	if *flagAll {
		m = mergeEdits()
	} else {
		m = computeEdits(build.Default.GOOS, build.Default.GOARCH, build.Default.CgoEnabled)
	}

	if *flagApply {
		var wg sync.WaitGroup
		for f, e := range m {
			wg.Add(1)
			f, e := f, e
			go func() {
				defer wg.Done()
				apply(f, e)
			}()
		}
		wg.Wait()
	} else {
		var files []string
		for f := range m {
			files = append(files, f)
		}
		sort.Strings(files)
		for _, f := range files {
			print(f, m[f])
		}
	}
}

var plats = [...]struct {
	goos, goarch string
}{
	// TODO(mdempsky): buildall.bash also builds linux-386-387 and linux-arm-arm5.
	{"linux", "386"},
	{"linux", "amd64"},
	{"linux", "arm"},
	{"linux", "arm64"},
	{"linux", "mips64"},
	{"linux", "mips64le"},
	{"linux", "ppc64"},
	{"linux", "ppc64le"},
	{"nacl", "386"},
	{"nacl", "amd64p32"},
	{"nacl", "arm"},
	{"android", "386"},
	{"android", "amd64"},
	{"darwin", "386"},
	{"darwin", "amd64"},
	{"dragonfly", "amd64"},
	{"freebsd", "386"},
	{"freebsd", "amd64"},
	{"freebsd", "arm"},
	{"netbsd", "386"},
	{"netbsd", "amd64"},
	{"netbsd", "arm"},
	{"openbsd", "386"},
	{"openbsd", "amd64"},
	{"openbsd", "arm"},
	{"plan9", "386"},
	{"plan9", "amd64"},
	{"solaris", "amd64"},
	{"windows", "386"},
	{"windows", "amd64"},
}

func mergeEdits() map[string]*intsets.Sparse {
	m := make(map[string]*intsets.Sparse)
	for _, plat := range plats {
		for f, e := range computeEdits(plat.goos, plat.goarch, false) {
			if e0, ok := m[f]; ok {
				e0.IntersectionWith(e)
			} else {
				m[f] = e
			}
		}
	}
	return m
}

type noImporter struct{}

func (noImporter) Import(path string) (*types.Package, error) {
	panic("golang.org/x/tools/go/loader said this wouldn't be called")
}

func computeEdits(os, arch string, cgoEnabled bool) map[string]*intsets.Sparse {
	ctxt := build.Default
	ctxt.GOOS = os
	ctxt.GOARCH = arch
	ctxt.CgoEnabled = cgoEnabled

	var conf loader.Config
	conf.Build = &ctxt
	conf.TypeChecker.Importer = noImporter{}
	for _, arg := range flag.Args() {
		conf.Import(arg)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	type res struct {
		file  string
		edits *intsets.Sparse
	}
	ch := make(chan res)
	var wg sync.WaitGroup
	for _, pkg := range prog.InitialPackages() {
		for _, file := range pkg.Files {
			pkg, file := pkg, file
			wg.Add(1)
			go func() {
				defer wg.Done()
				v := visitor{pkg: pkg, file: conf.Fset.File(file.Package)}
				ast.Walk(&v, file)
				ch <- res{v.file.Name(), &v.edits}
			}()
		}
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for r := range ch {
		m[r.file] = r.edits
	}
	return m
}

type visitor struct {
	pkg   *loader.PackageInfo
	file  *token.File
	edits intsets.Sparse
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if call, ok := node.(*ast.CallExpr); ok {
		v.unconvert(call)
	}
	return v
}

func (v *visitor) unconvert(call *ast.CallExpr) {
	// TODO(mdempsky): Handle useless multi-conversions.

	// Conversions have exactly one argument.
	if len(call.Args) != 1 || call.Ellipsis != token.NoPos {
		return
	}
	ft, ok := v.pkg.Types[call.Fun]
	if !ok {
		fmt.Println("Missing type for function")
		return
	}
	if !ft.IsType() {
		// Function call; not a conversion.
		return
	}
	at, ok := v.pkg.Types[call.Args[0]]
	if !ok {
		fmt.Println("Missing type for argument")
	}
	if isUntypedValue(call.Args[0], &v.pkg.Info) {
		// Workaround golang.org/issue/13061.
		return
	}
	if !types.Identical(ft.Type, at.Type) {
		// A real conversion.
		return
	}

	v.edits.Insert(v.file.Offset(call.Lparen))
}

func isUntypedValue(n ast.Expr, info *types.Info) (res bool) {
	switch n := n.(type) {
	case *ast.BinaryExpr:
		switch n.Op {
		case token.SHL, token.SHR:
			// Shifts yield an untyped value if their LHS is untyped.
			return isUntypedValue(n.X, info)
		case token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ:
			// Comparisons yield an untyped boolean value.
			return true
		case token.ADD, token.SUB, token.MUL, token.QUO, token.REM,
			token.AND, token.OR, token.XOR, token.AND_NOT,
			token.LAND, token.LOR:
			return isUntypedValue(n.X, info) && isUntypedValue(n.Y, info)
		}
	case *ast.UnaryExpr:
		switch n.Op {
		case token.ADD, token.SUB, token.NOT, token.XOR:
			return isUntypedValue(n.X, info)
		}
	case *ast.BasicLit:
		// Basic literals are always untyped.
		return true
	case *ast.ParenExpr:
		return isUntypedValue(n.X, info)
	case *ast.SelectorExpr:
		return isUntypedValue(n.Sel, info)
	case *ast.Ident:
		if obj, ok := info.Uses[n]; ok {
			if obj.Pkg() == nil && obj.Name() == "nil" {
				// The universal untyped zero value.
				return true
			}
			if b, ok := obj.Type().(*types.Basic); ok && b.Info()&types.IsUntyped != 0 {
				// Reference to an untyped constant.
				return true
			}
		}
	case *ast.CallExpr:
		if b, ok := asBuiltin(n.Fun, info); ok {
			switch b.Name() {
			case "real", "imag":
				return isUntypedValue(n.Args[0], info)
			case "complex":
				return isUntypedValue(n.Args[0], info) && isUntypedValue(n.Args[1], info)
			}
		}
	}

	return false
}

func asBuiltin(n ast.Expr, info *types.Info) (*types.Builtin, bool) {
	for {
		paren, ok := n.(*ast.ParenExpr)
		if !ok {
			break
		}
		n = paren.X
	}

	ident, ok := n.(*ast.Ident)
	if !ok {
		return nil, false
	}

	obj, ok := info.Uses[ident]
	if !ok {
		return nil, false
	}

	b, ok := obj.(*types.Builtin)
	return b, ok
}
<commit_msg>unconvert: better -help message<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Unconvert removes redundant type conversions from Go packages.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"go/types"
	"io/ioutil"
	"log"
	"os"
	"reflect"
	"runtime/pprof"
	"sort"
	"sync"
	"unicode"

	"golang.org/x/tools/container/intsets"
	"golang.org/x/tools/go/loader"
)

// Unnecessary conversions are identified by the position
// of their left parenthesis within a source file.

func apply(file string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, file, nil, parser.ParseComments)
	if err != nil {
		log.Fatal(err)
	}

	// Note: We modify edits during the walk.
	v := editor{edits: edits, file: fset.File(f.Package)}
	ast.Walk(&v, f)
	if !edits.IsEmpty() {
		log.Printf("%s: missing edits %s", file, edits)
	}

	// TODO(mdempsky): Write to temporary file and rename.
	var buf bytes.Buffer
	err = format.Node(&buf, fset, f)
	if err != nil {
		log.Fatal(err)
	}

	err = ioutil.WriteFile(file, buf.Bytes(), 0)
	if err != nil {
		log.Fatal(err)
	}
}

type editor struct {
	edits *intsets.Sparse
	file  *token.File
}

func (e *editor) Visit(n ast.Node) ast.Visitor {
	if n == nil {
		return nil
	}
	v := reflect.ValueOf(n).Elem()
	for i, n := 0, v.NumField(); i < n; i++ {
		switch f := v.Field(i).Addr().Interface().(type) {
		case *ast.Expr:
			e.rewrite(f)
		case *[]ast.Expr:
			for i := range *f {
				e.rewrite(&(*f)[i])
			}
		}
	}
	return e
}

func (e *editor) rewrite(f *ast.Expr) {
	n, ok := (*f).(*ast.CallExpr)
	if !ok {
		return
	}
	off := e.file.Offset(n.Lparen)
	if !e.edits.Has(off) {
		return
	}
	*f = n.Args[0]
	e.edits.Remove(off)
}

func print(name string, edits *intsets.Sparse) {
	if edits.IsEmpty() {
		return
	}

	buf, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatal(err)
	}

	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, name, buf, 0)
	if err != nil {
		log.Fatal(err)
	}

	file := fset.File(f.Package)
	for _, p := range edits.AppendTo(nil) {
		pos := file.Position(file.Pos(p))
		fmt.Printf("%s:%d:%d:\n", pos.Filename, pos.Line, pos.Column)
		line := lineForOffset(buf, pos.Offset)
		fmt.Printf("%s\n", line)
		fmt.Printf("%s^\n", rub(line[:pos.Column-1]))
	}
}

func rub(buf []byte) []byte {
	// TODO(mdempsky): Handle combining characters?
	// TODO(mdempsky): Handle East Asian wide characters?
	var res bytes.Buffer
	for _, c := range string(buf) {
		if !unicode.IsSpace(c) {
			c = ' '
		}
		res.WriteRune(c)
	}
	return res.Bytes()
}

func lineForOffset(buf []byte, off int) []byte {
	sol := bytes.LastIndexByte(buf[:off], '\n')
	if sol < 0 {
		sol = 0
	} else {
		sol += 1
	}
	eol := bytes.IndexByte(buf[off:], '\n')
	if eol < 0 {
		eol = len(buf)
	} else {
		eol += off
	}
	return buf[sol:eol]
}

var (
	flagAll        = flag.Bool("all", false, "type check all GOOS and GOARCH combinations")
	flagApply      = flag.Bool("apply", false, "apply edits to source files")
	flagCPUProfile = flag.String("cpuprofile", "", "write CPU profile to file")
)

func usage() {
	fmt.Fprintf(os.Stderr, "usage: unconvert [flags] [package ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if *flagCPUProfile != "" {
		f, err := os.Create(*flagCPUProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var m map[string]*intsets.Sparse
	if *flagAll {
		m = mergeEdits()
	} else {
		m = computeEdits(build.Default.GOOS, build.Default.GOARCH, build.Default.CgoEnabled)
	}

	if *flagApply {
		var wg sync.WaitGroup
		for f, e := range m {
			wg.Add(1)
			f, e := f, e
			go func() {
				defer wg.Done()
				apply(f, e)
			}()
		}
		wg.Wait()
	} else {
		var files []string
		for f := range m {
			files = append(files, f)
		}
		sort.Strings(files)
		for _, f := range files {
			print(f, m[f])
		}
	}
}

var plats = [...]struct {
	goos, goarch string
}{
	// TODO(mdempsky): buildall.bash also builds linux-386-387 and linux-arm-arm5.
	{"linux", "386"},
	{"linux", "amd64"},
	{"linux", "arm"},
	{"linux", "arm64"},
	{"linux", "mips64"},
	{"linux", "mips64le"},
	{"linux", "ppc64"},
	{"linux", "ppc64le"},
	{"nacl", "386"},
	{"nacl", "amd64p32"},
	{"nacl", "arm"},
	{"android", "386"},
	{"android", "amd64"},
	{"darwin", "386"},
	{"darwin", "amd64"},
	{"dragonfly", "amd64"},
	{"freebsd", "386"},
	{"freebsd", "amd64"},
	{"freebsd", "arm"},
	{"netbsd", "386"},
	{"netbsd", "amd64"},
	{"netbsd", "arm"},
	{"openbsd", "386"},
	{"openbsd", "amd64"},
	{"openbsd", "arm"},
	{"plan9", "386"},
	{"plan9", "amd64"},
	{"solaris", "amd64"},
	{"windows", "386"},
	{"windows", "amd64"},
}

func mergeEdits() map[string]*intsets.Sparse {
	m := make(map[string]*intsets.Sparse)
	for _, plat := range plats {
		for f, e := range computeEdits(plat.goos, plat.goarch, false) {
			if e0, ok := m[f]; ok {
				e0.IntersectionWith(e)
			} else {
				m[f] = e
			}
		}
	}
	return m
}

type noImporter struct{}

func (noImporter) Import(path string) (*types.Package, error) {
	panic("golang.org/x/tools/go/loader said this wouldn't be called")
}

func computeEdits(os, arch string, cgoEnabled bool) map[string]*intsets.Sparse {
	ctxt := build.Default
	ctxt.GOOS = os
	ctxt.GOARCH = arch
	ctxt.CgoEnabled = cgoEnabled

	var conf loader.Config
	conf.Build = &ctxt
	conf.TypeChecker.Importer = noImporter{}
	for _, arg := range flag.Args() {
		conf.Import(arg)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	type res struct {
		file  string
		edits *intsets.Sparse
	}
	ch := make(chan res)
	var wg sync.WaitGroup
	for _, pkg := range prog.InitialPackages() {
		for _, file := range pkg.Files {
			pkg, file := pkg, file
			wg.Add(1)
			go func() {
				defer wg.Done()
				v := visitor{pkg: pkg, file: conf.Fset.File(file.Package)}
				ast.Walk(&v, file)
				ch <- res{v.file.Name(), &v.edits}
			}()
		}
	}
	go func() {
		wg.Wait()
		close(ch)
	}()

	m := make(map[string]*intsets.Sparse)
	for r := range ch {
		m[r.file] = r.edits
	}
	return m
}

type visitor struct {
	pkg   *loader.PackageInfo
	file  *token.File
	edits intsets.Sparse
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if call, ok := node.(*ast.CallExpr); ok {
		v.unconvert(call)
	}
	return v
}

func (v *visitor) unconvert(call *ast.CallExpr) {
	// TODO(mdempsky): Handle useless multi-conversions.

	// Conversions have exactly one argument.
	if len(call.Args) != 1 || call.Ellipsis != token.NoPos {
		return
	}
	ft, ok := v.pkg.Types[call.Fun]
	if !ok {
		fmt.Println("Missing type for function")
		return
	}
	if !ft.IsType() {
		// Function call; not a conversion.
		return
	}
	at, ok := v.pkg.Types[call.Args[0]]
	if !ok {
		fmt.Println("Missing type for argument")
	}
	if isUntypedValue(call.Args[0], &v.pkg.Info) {
		// Workaround golang.org/issue/13061.
		return
	}
	if !types.Identical(ft.Type, at.Type) {
		// A real conversion.
		return
	}

	v.edits.Insert(v.file.Offset(call.Lparen))
}

func isUntypedValue(n ast.Expr, info *types.Info) (res bool) {
	switch n := n.(type) {
	case *ast.BinaryExpr:
		switch n.Op {
		case token.SHL, token.SHR:
			// Shifts yield an untyped value if their LHS is untyped.
			return isUntypedValue(n.X, info)
		case token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ:
			// Comparisons yield an untyped boolean value.
			return true
		case token.ADD, token.SUB, token.MUL, token.QUO, token.REM,
			token.AND, token.OR, token.XOR, token.AND_NOT,
			token.LAND, token.LOR:
			return isUntypedValue(n.X, info) && isUntypedValue(n.Y, info)
		}
	case *ast.UnaryExpr:
		switch n.Op {
		case token.ADD, token.SUB, token.NOT, token.XOR:
			return isUntypedValue(n.X, info)
		}
	case *ast.BasicLit:
		// Basic literals are always untyped.
		return true
	case *ast.ParenExpr:
		return isUntypedValue(n.X, info)
	case *ast.SelectorExpr:
		return isUntypedValue(n.Sel, info)
	case *ast.Ident:
		if obj, ok := info.Uses[n]; ok {
			if obj.Pkg() == nil && obj.Name() == "nil" {
				// The universal untyped zero value.
				return true
			}
			if b, ok := obj.Type().(*types.Basic); ok && b.Info()&types.IsUntyped != 0 {
				// Reference to an untyped constant.
				return true
			}
		}
	case *ast.CallExpr:
		if b, ok := asBuiltin(n.Fun, info); ok {
			switch b.Name() {
			case "real", "imag":
				return isUntypedValue(n.Args[0], info)
			case "complex":
				return isUntypedValue(n.Args[0], info) && isUntypedValue(n.Args[1], info)
			}
		}
	}

	return false
}

func asBuiltin(n ast.Expr, info *types.Info) (*types.Builtin, bool) {
	for {
		paren, ok := n.(*ast.ParenExpr)
		if !ok {
			break
		}
		n = paren.X
	}

	ident, ok := n.(*ast.Ident)
	if !ok {
		return nil, false
	}

	obj, ok := info.Uses[ident]
	if !ok {
		return nil, false
	}

	b, ok := obj.(*types.Builtin)
	return b, ok
}
<|endoftext|>


<commit_before>package main

import (
	cryptorand "crypto/rand"
	"crypto/sha512"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"runtime"
	"time"
)

const (
	BSIZE    = 65536
	BSIZE7   = BSIZE - BSIZEMOD
	BSIZEMOD = BSIZE % 7
	MULTI    = 4
	SOURCES  = 8
)

var (
	narg    = int(0)
	phrase  = ""
	pool    = make(chan []byte, SOURCES*MULTI)
	sources [SOURCES]rand.Source
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

// randomBytes fills byte buffers with random data
func randomBytes(src rand.Source, out chan<- []byte) {
	var (
		r int64
		i = BSIZE
	)

	for buf, ok := <-pool; ok; buf, ok = <-pool {
		r = src.Int63()
		switch { // Go seems to eliminate impossible cases
		case BSIZEMOD == 6:
			buf[BSIZE-6] = byte(r >> 48)
			fallthrough
		case BSIZEMOD == 5:
			buf[BSIZE-5] = byte(r >> 32)
			fallthrough
		case BSIZEMOD == 4:
			buf[BSIZE-4] = byte(r >> 24)
			fallthrough
		case BSIZEMOD == 3:
			buf[BSIZE-3] = byte(r >> 16)
			fallthrough
		case BSIZEMOD == 2:
			buf[BSIZE-2] = byte(r >> 8)
			fallthrough
		case BSIZEMOD == 1:
			buf[BSIZE-1] = byte(r)
		}

		for i = 0; i < BSIZE7; i += 7 {
			r = src.Int63()
			buf[i] = byte(r)
			buf[i+1] = byte(r >> 8)
			buf[i+2] = byte(r >> 16)
			buf[i+3] = byte(r >> 24)
			buf[i+4] = byte(r >> 32)
			buf[i+5] = byte(r >> 40)
			buf[i+6] = byte(r >> 48)
		}

		out <- buf
	}
}

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	narg = flag.NArg()

	if narg <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}
}

func garble(index int, f *os.File, c chan []byte) {
	// fmt.Println("garble(", index, f, c, ")")
	<-c
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	channels := make([]chan []byte, narg)
	files := make([]*os.File, narg)

	for i, arg := range flag.Args() {
		f, err := os.OpenFile(arg, os.O_RDWR, 0666)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		files[i] = f
		channels[i] = make(chan []byte, 8)
		go garble(i, files[i], channels[i])
	}

	hash := sha512.New()
	seed := []byte(phrase)
	b := make([]byte, hash.Size())
	for i := 0; i < 2; i++ {
		hash.Write(seed)
		hash.Sum(b[:0])
		// fmt.Println(b)
	}

	src := rand.NewSource(int64(b[0]) + int64(b[1])<<8)

	buf := make([]byte, BSIZE*SOURCES*MULTI)

	for i := 0; i < BSIZE*SOURCES*MULTI; i += BSIZE {
		pool <- buf[i : i+BSIZE]
	}

	fmt.Println("All sent.", src.Int63())

	out := make(chan []byte, 1)

	go randomBytes(src, out)

	buf = <-out

	os.Stdout.Write(buf)
}
<commit_msg>seeding sources<commit_after>package main

import (
	"bytes"
	cryptorand "crypto/rand"
	"crypto/sha512"
	"encoding/binary"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"runtime"
	"time"
)

const (
	BSIZE    = 65536
	BSIZE7   = BSIZE - BSIZEMOD
	BSIZEMOD = BSIZE % 7
	MULTI    = 4
	SOURCES  = 8
)

var (
	narg    = int(0)
	phrase  = ""
	pool    = make(chan []byte, SOURCES*MULTI)
	sources [SOURCES]rand.Source
)

// randomSeed produces a int64 seed based on crypto/rand and time.
func randomSeed() int64 {
	var seed int64

	urandom := make([]byte, 8)
	cryptorand.Reader.Read(urandom)

	for key, value := range urandom {
		seed ^= (int64(value) ^ time.Now().UTC().UnixNano()) << (uint(key) * 8)
	}

	return seed
}

// randomBytes fills byte buffers with random data
func randomBytes(src rand.Source, out chan<- []byte) {
	var (
		r int64
		i = BSIZE
	)

	for buf, ok := <-pool; ok; buf, ok = <-pool {
		r = src.Int63()
		switch { // Go seems to eliminate impossible cases
		case BSIZEMOD == 6:
			buf[BSIZE-6] = byte(r >> 48)
			fallthrough
		case BSIZEMOD == 5:
			buf[BSIZE-5] = byte(r >> 32)
			fallthrough
		case BSIZEMOD == 4:
			buf[BSIZE-4] = byte(r >> 24)
			fallthrough
		case BSIZEMOD == 3:
			buf[BSIZE-3] = byte(r >> 16)
			fallthrough
		case BSIZEMOD == 2:
			buf[BSIZE-2] = byte(r >> 8)
			fallthrough
		case BSIZEMOD == 1:
			buf[BSIZE-1] = byte(r)
		}

		for i = 0; i < BSIZE7; i += 7 {
			r = src.Int63()
			buf[i] = byte(r)
			buf[i+1] = byte(r >> 8)
			buf[i+2] = byte(r >> 16)
			buf[i+3] = byte(r >> 24)
			buf[i+4] = byte(r >> 32)
			buf[i+5] = byte(r >> 40)
			buf[i+6] = byte(r >> 48)
		}

		out <- buf
	}
}

func garble(index int, f *os.File, c chan []byte) {
	// fmt.Println("garble(", index, f, c, ")")
	<-c
}

func init() {
	flag.StringVar(&phrase, "phrase", "", "the Garble phrase, by default random")
	flag.Parse()

	narg = flag.NArg()

	if narg <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if phrase == "" {
		phrase = fmt.Sprintf("%016x", uint64(randomSeed()))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	channels := make([]chan []byte, narg)
	files := make([]*os.File, narg)

	for i, arg := range flag.Args() {
		f, err := os.OpenFile(arg, os.O_RDWR, 0666)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		files[i] = f
		channels[i] = make(chan []byte, 8)
		go garble(i, files[i], channels[i])
	}

	hash := sha512.New()
	sum := make([]byte, hash.Size())
	phrase := []byte(phrase)

	var src rand.Source

	for i := 0; i < SOURCES; i++ {
		var seed, s int64
		var err error

		hash.Write([]byte(":garble:"))
		hash.Write(phrase)
		hash.Sum(sum[:0])

		buf := bytes.NewReader(sum)
		s = 0

		for err == nil {
			err = binary.Read(buf, binary.LittleEndian, &s)
			seed ^= s
		}

		src = rand.NewSource(seed)
		fmt.Println(seed, src.Int63())
	}

	buf := make([]byte, BSIZE*SOURCES*MULTI)

	for i := 0; i < BSIZE*SOURCES*MULTI; i += BSIZE {
		pool <- buf[i : i+BSIZE]
	}

	fmt.Println("All sent.")

	out := make(chan []byte, 1)

	go randomBytes(src, out)

	buf = <-out

	os.Stdout.Write(buf)
}
<|endoftext|>


<commit_before>package medtronic

import (
	"fmt"
	"time"
)

const (
	tempBasal            Command = 0x98
	setAbsoluteTempBasal Command = 0x4C
	setPercentTempBasal  Command = 0x69

	maxBasal = 34000 // milliUnits
)

// TempBasalType represents the temp basal type.
type TempBasalType byte

//go:generate stringer -type TempBasalType

const (
	// Absolute represents the pump's use of absolute rates for temporary basals.
	Absolute TempBasalType = 0
	// Percent represents the pump's use of percentage rates for temporary basals.
	Percent TempBasalType = 1
)

// TempBasalInfo represents a temporary basal setting.
type TempBasalInfo struct {
	Duration time.Duration
	Type     TempBasalType
	Rate     *Insulin `json:",omitempty"`
	Percent  *uint8   `json:",omitempty"`
}

func decodeTempBasal(data []byte) (TempBasalInfo, error) {
	if len(data) < 7 || data[0] != 6 {
		return TempBasalInfo{}, BadResponseError{Command: tempBasal, Data: data}
	}
	d := time.Duration(twoByteInt(data[5:7])) * time.Minute
	tempType := TempBasalType(data[1])
	info := TempBasalInfo{Duration: d, Type: tempType}
	switch tempType {
	case Absolute:
		rate := twoByteInsulin(data[3:5], true)
		info.Rate = &rate
	case Percent:
		percent := data[2]
		info.Percent = &percent
	default:
		return info, BadResponseError{Command: tempBasal, Data: data}
	}
	return info, nil
}

// TempBasal returns the pump's current temporary basal setting.
// If none is in effect, it will have a Duration of 0.
func (pump *Pump) TempBasal() TempBasalInfo {
	data := pump.Execute(tempBasal)
	if pump.Error() != nil {
		return TempBasalInfo{}
	}
	info, err := decodeTempBasal(data)
	pump.SetError(err)
	return info
}

// SetAbsoluteTempBasal sets a temporary basal with the given absolute rate and duration.
func (pump *Pump) SetAbsoluteTempBasal(duration time.Duration, rate Insulin) {
	d := pump.halfHours(duration)
	if rate < 0 {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is negative", rate))
	}
	if rate > maxBasal {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is too large", rate))
	}
	if rate%25 != 0 {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is not a multiple of 25 milliUnits per hour", rate))
	}
	v := int(rate / 25)
	pump.Execute(setAbsoluteTempBasal, byte(v>>8), byte(v&0xFF), d)
}

// SetPercentTempBasal sets a temporary basal with the given percent rate and duration.
func (pump *Pump) SetPercentTempBasal(duration time.Duration, percent int) {
	d := pump.halfHours(duration)
	if percent < 0 || 100 < percent {
		pump.SetError(fmt.Errorf("percent temporary basal rate (%d) is not between 0 and 100", percent))
	}
	pump.Execute(setPercentTempBasal, byte(percent), d)
}

func (pump *Pump) halfHours(duration time.Duration) uint8 {
	const halfHour = 30 * time.Minute
	if duration%halfHour != 0 {
		pump.SetError(fmt.Errorf("duration (%v) is not a multiple of 30 minutes", duration))
	}
	return uint8(duration / halfHour)
}
<commit_msg>Add stricter validation of duration values<commit_after>package medtronic

import (
	"fmt"
	"time"
)

const (
	tempBasal            Command = 0x98
	setAbsoluteTempBasal Command = 0x4C
	setPercentTempBasal  Command = 0x69

	maxBasal    = 34000 // milliUnits
	maxDuration = 24 * time.Hour
)

// TempBasalType represents the temp basal type.
type TempBasalType byte

//go:generate stringer -type TempBasalType

const (
	// Absolute represents the pump's use of absolute rates for temporary basals.
	Absolute TempBasalType = 0
	// Percent represents the pump's use of percentage rates for temporary basals.
	Percent TempBasalType = 1
)

// TempBasalInfo represents a temporary basal setting.
type TempBasalInfo struct {
	Duration time.Duration
	Type     TempBasalType
	Rate     *Insulin `json:",omitempty"`
	Percent  *uint8   `json:",omitempty"`
}

func decodeTempBasal(data []byte) (TempBasalInfo, error) {
	if len(data) < 7 || data[0] != 6 {
		return TempBasalInfo{}, BadResponseError{Command: tempBasal, Data: data}
	}
	d := time.Duration(twoByteInt(data[5:7])) * time.Minute
	tempType := TempBasalType(data[1])
	info := TempBasalInfo{Duration: d, Type: tempType}
	switch tempType {
	case Absolute:
		rate := twoByteInsulin(data[3:5], true)
		info.Rate = &rate
	case Percent:
		percent := data[2]
		info.Percent = &percent
	default:
		return info, BadResponseError{Command: tempBasal, Data: data}
	}
	return info, nil
}

// TempBasal returns the pump's current temporary basal setting.
// If none is in effect, it will have a Duration of 0.
func (pump *Pump) TempBasal() TempBasalInfo {
	data := pump.Execute(tempBasal)
	if pump.Error() != nil {
		return TempBasalInfo{}
	}
	info, err := decodeTempBasal(data)
	pump.SetError(err)
	return info
}

// SetAbsoluteTempBasal sets a temporary basal with the given absolute rate and duration.
func (pump *Pump) SetAbsoluteTempBasal(duration time.Duration, rate Insulin) {
	d := pump.halfHours(duration)
	if rate < 0 {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is negative", rate))
		return
	}
	if rate > maxBasal {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is too large", rate))
		return
	}
	if rate%25 != 0 {
		pump.SetError(fmt.Errorf("absolute temporary basal rate (%d) is not a multiple of 25 milliUnits per hour", rate))
		return
	}
	v := int(rate / 25)
	pump.Execute(setAbsoluteTempBasal, byte(v>>8), byte(v&0xFF), d)
}

// SetPercentTempBasal sets a temporary basal with the given percent rate and duration.
func (pump *Pump) SetPercentTempBasal(duration time.Duration, percent int) {
	d := pump.halfHours(duration)
	if percent < 0 || 100 < percent {
		pump.SetError(fmt.Errorf("percent temporary basal rate (%d) is not between 0 and 100", percent))
		return
	}
	pump.Execute(setPercentTempBasal, byte(percent), d)
}

func (pump *Pump) halfHours(duration time.Duration) uint8 {
	const halfHour = 30 * time.Minute
	if duration%halfHour != 0 {
		pump.SetError(fmt.Errorf("duration (%v) is not a multiple of 30 minutes", duration))
		return 0
	}
	if duration < 0 {
		pump.SetError(fmt.Errorf("duration (%v) is negative", duration))
		return 0
	}
	if duration > maxDuration {
		pump.SetError(fmt.Errorf("duration (%v) is too large", duration))
		return 0
	}
	return uint8(duration / halfHour)
}
<|endoftext|>


<commit_before>package sync

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/deckarep/golang-set"
	"github.com/dreampuf/evernote-sdk-golang/client"
	"github.com/dreampuf/evernote-sdk-golang/notestore"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/dreampuf/evernote-sdk-golang/userstore"
	"github.com/pkg/errors"
)

const minimumFetchIntervalSeconds = 15 * 60
const cacheExtension = ".yml"

// Sync local cache from evernote server
func Sync(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, clientKey string, clientSecret string, developerToken string, isSandbox bool, notebookName string) error {
	cacheRoot = path.Clean(cacheRoot)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	if err := os.MkdirAll(noteCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create note cache path %v", noteCacheDir)
	}

	if err := os.MkdirAll(resourceCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create resource cache path %v", resourceCacheDir)
	}

	cli := client.NewClient(clientKey, clientSecret, getEnvironment(isSandbox))

	us, err := getUserStore(cli, isSandbox)
	if err != nil {
		return err
	}

	ns, err := getNoteStore(cli, us, &developerToken)
	if err != nil {
		return err
	}

	notUpdated, err := checkUpdate(ns, &cacheRoot, &developerToken)
	if err != nil {
		return err
	}
	if notUpdated {
		return errors.Errorf("note is not updated")
	}

	bookGUID, err := findNotebookGUID(&notebookName, cli, ns, &developerToken)
	if err != nil {
		return err
	}

	metadatas, err := findMetadatas(ns, bookGUID, &developerToken)
	if err != nil {
		return err
	}

	cachedIds, err := createCachedNoteIDMap(&noteCacheDir)
	if err != nil {
		return err
	}

	for _, note := range metadatas.GetNotes() {
		fmt.Printf("processing %v\n", note.GUID)

		noteCachePath := path.Join(noteCacheDir, string(note.GUID)+cacheExtension)
		cachedNote, err := readCachedNote(&noteCachePath, &note.GUID)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note")
		}

		if cachedNote == nil || cachedNote.UpdateSequenceNum == nil || note.UpdateSequenceNum == nil || *cachedNote.UpdateSequenceNum != *note.UpdateSequenceNum {
			note, err := ns.GetNote(developerToken, note.GUID, true, false, false, false)
			if err != nil {
				return errors.Wrapf(err, "can't get note %v", *note.GUID)
			}

			tags, err := ns.GetNoteTagNames(developerToken, *note.GUID)
			if err != nil {
				return errors.Wrapf(err, "can't get note tags %v", *note.GUID)
			}
			note.TagNames = tags

			fmt.Printf("downloaded %v[%v]\n", *note.Title, *note.GUID)

			writeCachedNoteToFile(&noteCachePath, note)
			saveResources(&resourceCacheDir, cachedNote, note, ns, &developerToken)
		}

		cachedIds.Remove(note.GUID)
	}

	for _, id := range cachedIds.ToSlice() {
		guid := id.(string)
		os.Remove(noteCacheDir + guid + cacheExtension)
	}

	return nil
}

func getEnvironment(isSandbox bool) client.EnvironmentType {
	if isSandbox {
		return client.SANDBOX
	}
	return client.PRODUCTION
}

func getUserStore(cli *client.EvernoteClient, isSandbox bool) (*userstore.UserStoreClient, error) {
	us, err := cli.GetUserStore()
	if err != nil {
		return nil, errors.Wrapf(err, "can't get user store (environment: %v)", getEnvironment(isSandbox))
	}

	versionOk, err := us.CheckVersion("chienote", userstore.EDAM_VERSION_MAJOR, userstore.EDAM_VERSION_MINOR)
	if !versionOk {
		return nil, errors.New("user store isn't correct version")
	}
	if err != nil {
		return nil, errors.Wrap(err, "error occured on checking user store version")
	}

	return us, nil
}

func getNoteStore(cli *client.EvernoteClient, us *userstore.UserStoreClient, developerToken *string) (*notestore.NoteStoreClient, error) {
	url, err := us.GetNoteStoreUrl(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store url")
	}
	if url == "" {
		return nil, errors.New("empty notestore url received")
	}

	ns, err := cli.GetNoteStoreWithURL(url)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store")
	}

	return ns, nil
}

func findNotebookGUID(notebookName *string, cli *client.EvernoteClient, ns *notestore.NoteStoreClient, developerToken *string) (notebookGUID *types.GUID, err error) {
	notebooks, err := ns.ListNotebooks(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get notebook list")
	}

	for _, book := range notebooks {
		if *book.Name == *notebookName {
			notebookGUID = book.GUID
			break
		}
	}

	if notebookGUID == nil || *notebookGUID == "" {
		return nil, errors.Errorf("can't get notebook %v", *notebookName)
	}

	return notebookGUID, nil
}

func findMetadatas(ns *notestore.NoteStoreClient, bookGUID *types.GUID, developerToken *string) (metadatas *notestore.NotesMetadataList, err error) {
	ascending := false
	filter := &notestore.NoteFilter{NotebookGuid: bookGUID, Ascending: &ascending}

	var resultSpec notestore.NotesMetadataResultSpec
	metadatas, err = ns.FindNotesMetadata(*developerToken, filter, 0, 1000, &resultSpec)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't find notes")
	}
	return metadatas, err
}

func createCachedNoteIDMap(noteCachePath *string) (mapset.Set, error) {
	cachedIDs := mapset.NewSet()
	cacheFileInfos, err := ioutil.ReadDir(*noteCachePath)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cache directory %v", *noteCachePath)
	}

	for _, cacheFileInfo := range cacheFileInfos {
		if strings.HasSuffix(cacheFileInfo.Name(), cacheExtension) {
			cachedIDs.Add(strings.Replace(cacheFileInfo.Name(), cacheExtension, "", 1))
		}
	}

	return cachedIDs, nil
}

func readCachedNoteFile(noteCachePath *string) (*types.Note, error) {
	cacheReader, err := os.OpenFile(*noteCachePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return nil, errors.Wrapf(err, "can't open cached note file %v", *noteCachePath)
	}
	defer cacheReader.Close()

	cachedYAML, err := ioutil.ReadAll(cacheReader)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cached note file %v", *noteCachePath)
	}

	cachedNote := &types.Note{}
	if err := yaml.Unmarshal(cachedYAML, cachedNote); err != nil {
		return nil, errors.Wrapf(err, "can't parse cached yaml file %v", *noteCachePath)
	}

	return cachedNote, nil
}

func readCachedNote(noteCachePath *string, noteGUID *types.GUID) (*types.Note, error) {
	var cachedNote *types.Note
	if _, err := os.Stat(*noteCachePath); err == nil {
		cachedNote, err = readCachedNoteFile(noteCachePath)
		if err != nil {
			return nil, errors.Wrapf(err, "error occurs when reading cached note file %v", *noteCachePath)
		}
	}

	return cachedNote, nil
}

func writeCachedNoteToFile(cachePath *string, note *types.Note) error {
	yamlBytes, err := yaml.Marshal(note)
	if err != nil {
		return errors.Wrapf(err, "can't marshal note as YAML %v", *cachePath)
	}

	if err := ioutil.WriteFile(*cachePath, yamlBytes, os.ModePerm); err != nil {
		return errors.Wrapf(err, "can't write marshaled note to %v", *cachePath)
	}

	return nil
}

func saveResources(resourceCacheDir *string, cachedNote *types.Note, receivedNote *types.Note, ns *notestore.NoteStoreClient, developerToken *string) error {
	localResourceMap := map[types.GUID]int32{}
	if cachedNote != nil {
		for _, cachedResource := range cachedNote.Resources {
			localResourceMap[*cachedResource.GUID] = *cachedResource.UpdateSequenceNum
		}
	}

	for _, resource := range receivedNote.Resources {
		fetchingNeeded := false
		cachedUpdateNum, exists := localResourceMap[*resource.GUID]
		if exists {

			if cachedUpdateNum != *resource.UpdateSequenceNum {
				fetchingNeeded = true
			}
			delete(localResourceMap, *resource.GUID)
		} else {
			fetchingNeeded = true
		}

		if fetchingNeeded {
			resourceWithBytes, err := ns.GetResource(*developerToken, *resource.GUID, true, false, true, false)
			if err != nil {
				return errors.Wrapf(err, "can't get resource %v", *resource.GUID)
			}

			var resourceFileName string
			if resourceWithBytes.Attributes.FileName == nil {
				// https://dev.evernote.com/doc/articles/resources.php#downloading
				var extension string
				switch *resourceWithBytes.Mime {
				case "image/gif":
					extension = ".gif"
				case "image/jpeg":
					extension = ".jpg"
				case "image/png":
					extension = ".png"
				case "audio/wav":
					extension = ".wav"
				case "audio/mpeg":
					extension = ".mp3"
				case "audio/amr":
					extension = ".amr"
				case "audio/pdf":
					extension = ".pdf"
				}
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + extension
			} else {
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + "-" + *resourceWithBytes.Attributes.FileName
			}

			p := path.Join(*resourceCacheDir, resourceFileName)
			ioutil.WriteFile(p, resourceWithBytes.Data.Body, os.ModePerm)
			fmt.Println("write resource to " + p)
		} else {
			fmt.Printf("not updated %v\n", *resource.GUID)
		}
	}

	return nil
}

func checkUpdate(ns *notestore.NoteStoreClient, cacheDir *string, developerToken *string) (notUpdated bool, err error) {
	syncState, err := ns.GetSyncState(*developerToken)
	if err != nil {
		return false, errors.Wrap(err, "can't get sync state")
	}

	syncStatePath := path.Join(*cacheDir, "sync_state.yml")

	prevStateBytes, err := ioutil.ReadFile(syncStatePath)
	if err == nil {
		prevState := &notestore.SyncState{Uploaded: new(int64)}
		if err := yaml.Unmarshal(prevStateBytes, prevState); err == nil {
			if prevState.UpdateCount == syncState.UpdateCount {
				return true, nil
			}

			if syncState.CurrentTime-prevState.CurrentTime < minimumFetchIntervalSeconds*1000 {
				return false, errors.Errorf("you should wait at least %v seconds from last update", minimumFetchIntervalSeconds)
			}
		}
	}

	stateBytes, err := yaml.Marshal(syncState)
	if err != nil {
		return false, errors.Wrapf(err, "can't marshal sync state")
	}

	if err := ioutil.WriteFile(syncStatePath, stateBytes, os.ModePerm); err != nil {
		return false, errors.Wrapf(err, "can't write sync state %v", syncStatePath)
	}

	return false, nil
}
<commit_msg>Fix re-downloading non-updated note<commit_after>package sync

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/deckarep/golang-set"
	"github.com/dreampuf/evernote-sdk-golang/client"
	"github.com/dreampuf/evernote-sdk-golang/notestore"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/dreampuf/evernote-sdk-golang/userstore"
	"github.com/pkg/errors"
)

const minimumFetchIntervalSeconds = 15 * 60
const cacheExtension = ".yml"

// Sync local cache from evernote server
func Sync(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, clientKey string, clientSecret string, developerToken string, isSandbox bool, notebookName string) error {
	cacheRoot = path.Clean(cacheRoot)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	if err := os.MkdirAll(noteCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create note cache path %v", noteCacheDir)
	}

	if err := os.MkdirAll(resourceCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create resource cache path %v", resourceCacheDir)
	}

	cli := client.NewClient(clientKey, clientSecret, getEnvironment(isSandbox))

	us, err := getUserStore(cli, isSandbox)
	if err != nil {
		return err
	}

	ns, err := getNoteStore(cli, us, &developerToken)
	if err != nil {
		return err
	}

	notUpdated, err := checkUpdate(ns, &cacheRoot, &developerToken)
	if err != nil {
		return err
	}
	if notUpdated {
		return errors.Errorf("note is not updated")
	}

	bookGUID, err := findNotebookGUID(&notebookName, cli, ns, &developerToken)
	if err != nil {
		return err
	}

	metadatas, err := findMetadatas(ns, bookGUID, &developerToken)
	if err != nil {
		return err
	}

	cachedIds, err := createCachedNoteIDMap(&noteCacheDir)
	if err != nil {
		return err
	}

	for _, note := range metadatas.GetNotes() {
		fmt.Printf("processing %v\n", note.GUID)

		noteCachePath := path.Join(noteCacheDir, string(note.GUID)+cacheExtension)
		cachedNote, err := readCachedNote(&noteCachePath, &note.GUID)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note")
		}

		if cachedNote == nil || cachedNote.UpdateSequenceNum == nil || note.UpdateSequenceNum == nil || *cachedNote.UpdateSequenceNum != *note.UpdateSequenceNum {
			note, err := ns.GetNote(developerToken, note.GUID, true, false, false, false)
			if err != nil {
				return errors.Wrapf(err, "can't get note %v", *note.GUID)
			}

			tags, err := ns.GetNoteTagNames(developerToken, *note.GUID)
			if err != nil {
				return errors.Wrapf(err, "can't get note tags %v", *note.GUID)
			}
			note.TagNames = tags

			fmt.Printf("downloaded %v[%v]\n", *note.Title, *note.GUID)

			writeCachedNoteToFile(&noteCachePath, note)
			saveResources(&resourceCacheDir, cachedNote, note, ns, &developerToken)
		}

		cachedIds.Remove(note.GUID)
	}

	for _, id := range cachedIds.ToSlice() {
		guid := id.(string)
		os.Remove(noteCacheDir + guid + cacheExtension)
	}

	return nil
}

func getEnvironment(isSandbox bool) client.EnvironmentType {
	if isSandbox {
		return client.SANDBOX
	}
	return client.PRODUCTION
}

func getUserStore(cli *client.EvernoteClient, isSandbox bool) (*userstore.UserStoreClient, error) {
	us, err := cli.GetUserStore()
	if err != nil {
		return nil, errors.Wrapf(err, "can't get user store (environment: %v)", getEnvironment(isSandbox))
	}

	versionOk, err := us.CheckVersion("chienote", userstore.EDAM_VERSION_MAJOR, userstore.EDAM_VERSION_MINOR)
	if !versionOk {
		return nil, errors.New("user store isn't correct version")
	}
	if err != nil {
		return nil, errors.Wrap(err, "error occured on checking user store version")
	}

	return us, nil
}

func getNoteStore(cli *client.EvernoteClient, us *userstore.UserStoreClient, developerToken *string) (*notestore.NoteStoreClient, error) {
	url, err := us.GetNoteStoreUrl(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store url")
	}
	if url == "" {
		return nil, errors.New("empty notestore url received")
	}

	ns, err := cli.GetNoteStoreWithURL(url)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store")
	}

	return ns, nil
}

func findNotebookGUID(notebookName *string, cli *client.EvernoteClient, ns *notestore.NoteStoreClient, developerToken *string) (notebookGUID *types.GUID, err error) {
	notebooks, err := ns.ListNotebooks(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get notebook list")
	}

	for _, book := range notebooks {
		if *book.Name == *notebookName {
			notebookGUID = book.GUID
			break
		}
	}

	if notebookGUID == nil || *notebookGUID == "" {
		return nil, errors.Errorf("can't get notebook %v", *notebookName)
	}

	return notebookGUID, nil
}

func findMetadatas(ns *notestore.NoteStoreClient, bookGUID *types.GUID, developerToken *string) (metadatas *notestore.NotesMetadataList, err error) {
	ascending := false
	filter := &notestore.NoteFilter{NotebookGuid: bookGUID, Ascending: &ascending}

	var resultSpec notestore.NotesMetadataResultSpec
	includeUpdateSequenceNum := true
	resultSpec.IncludeUpdateSequenceNum = &includeUpdateSequenceNum
	metadatas, err = ns.FindNotesMetadata(*developerToken, filter, 0, 1000, &resultSpec)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't find notes")
	}
	return metadatas, err
}

func createCachedNoteIDMap(noteCachePath *string) (mapset.Set, error) {
	cachedIDs := mapset.NewSet()
	cacheFileInfos, err := ioutil.ReadDir(*noteCachePath)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cache directory %v", *noteCachePath)
	}

	for _, cacheFileInfo := range cacheFileInfos {
		if strings.HasSuffix(cacheFileInfo.Name(), cacheExtension) {
			cachedIDs.Add(strings.Replace(cacheFileInfo.Name(), cacheExtension, "", 1))
		}
	}

	return cachedIDs, nil
}

func readCachedNoteFile(noteCachePath *string) (*types.Note, error) {
	cacheReader, err := os.OpenFile(*noteCachePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return nil, errors.Wrapf(err, "can't open cached note file %v", *noteCachePath)
	}
	defer cacheReader.Close()

	cachedYAML, err := ioutil.ReadAll(cacheReader)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cached note file %v", *noteCachePath)
	}

	cachedNote := &types.Note{}
	if err := yaml.Unmarshal(cachedYAML, cachedNote); err != nil {
		return nil, errors.Wrapf(err, "can't parse cached yaml file %v", *noteCachePath)
	}

	return cachedNote, nil
}

func readCachedNote(noteCachePath *string, noteGUID *types.GUID) (*types.Note, error) {
	var cachedNote *types.Note
	if _, err := os.Stat(*noteCachePath); err == nil {
		cachedNote, err = readCachedNoteFile(noteCachePath)
		if err != nil {
			return nil, errors.Wrapf(err, "error occurs when reading cached note file %v", *noteCachePath)
		}
	}

	return cachedNote, nil
}

func writeCachedNoteToFile(cachePath *string, note *types.Note) error {
	yamlBytes, err := yaml.Marshal(note)
	if err != nil {
		return errors.Wrapf(err, "can't marshal note as YAML %v", *cachePath)
	}

	if err := ioutil.WriteFile(*cachePath, yamlBytes, os.ModePerm); err != nil {
		return errors.Wrapf(err, "can't write marshaled note to %v", *cachePath)
	}

	return nil
}

func saveResources(resourceCacheDir *string, cachedNote *types.Note, receivedNote *types.Note, ns *notestore.NoteStoreClient, developerToken *string) error {
	localResourceMap := map[types.GUID]int32{}
	if cachedNote != nil {
		for _, cachedResource := range cachedNote.Resources {
			localResourceMap[*cachedResource.GUID] = *cachedResource.UpdateSequenceNum
		}
	}

	for _, resource := range receivedNote.Resources {
		fetchingNeeded := false
		cachedUpdateNum, exists := localResourceMap[*resource.GUID]
		if exists {

			if cachedUpdateNum != *resource.UpdateSequenceNum {
				fetchingNeeded = true
			}
			delete(localResourceMap, *resource.GUID)
		} else {
			fetchingNeeded = true
		}

		if fetchingNeeded {
			resourceWithBytes, err := ns.GetResource(*developerToken, *resource.GUID, true, false, true, false)
			if err != nil {
				return errors.Wrapf(err, "can't get resource %v", *resource.GUID)
			}

			var resourceFileName string
			if resourceWithBytes.Attributes.FileName == nil {
				// https://dev.evernote.com/doc/articles/resources.php#downloading
				var extension string
				switch *resourceWithBytes.Mime {
				case "image/gif":
					extension = ".gif"
				case "image/jpeg":
					extension = ".jpg"
				case "image/png":
					extension = ".png"
				case "audio/wav":
					extension = ".wav"
				case "audio/mpeg":
					extension = ".mp3"
				case "audio/amr":
					extension = ".amr"
				case "audio/pdf":
					extension = ".pdf"
				}
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + extension
			} else {
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + "-" + *resourceWithBytes.Attributes.FileName
			}

			p := path.Join(*resourceCacheDir, resourceFileName)
			ioutil.WriteFile(p, resourceWithBytes.Data.Body, os.ModePerm)
			fmt.Println("write resource to " + p)
		} else {
			fmt.Printf("not updated %v\n", *resource.GUID)
		}
	}

	return nil
}

func checkUpdate(ns *notestore.NoteStoreClient, cacheDir *string, developerToken *string) (notUpdated bool, err error) {
	syncState, err := ns.GetSyncState(*developerToken)
	if err != nil {
		return false, errors.Wrap(err, "can't get sync state")
	}

	syncStatePath := path.Join(*cacheDir, "sync_state.yml")

	prevStateBytes, err := ioutil.ReadFile(syncStatePath)
	if err == nil {
		prevState := &notestore.SyncState{Uploaded: new(int64)}
		if err := yaml.Unmarshal(prevStateBytes, prevState); err == nil {
			if prevState.UpdateCount == syncState.UpdateCount {
				return true, nil
			}

			if syncState.CurrentTime-prevState.CurrentTime < minimumFetchIntervalSeconds*1000 {
				return false, errors.Errorf("you should wait at least %v seconds from last update", minimumFetchIntervalSeconds)
			}
		}
	}

	stateBytes, err := yaml.Marshal(syncState)
	if err != nil {
		return false, errors.Wrapf(err, "can't marshal sync state")
	}

	if err := ioutil.WriteFile(syncStatePath, stateBytes, os.ModePerm); err != nil {
		return false, errors.Wrapf(err, "can't write sync state %v", syncStatePath)
	}

	return false, nil
}
<|endoftext|>


<commit_before>package sync

import (
	"io"
	"sync"

	ds "github.com/ipfs/go-datastore"
	dsq "github.com/ipfs/go-datastore/query"
)

// MutexDatastore contains a child datastire and a mutex.
// used for coarse sync
type MutexDatastore struct {
	sync.RWMutex

	child ds.Datastore
}

// MutexWrap constructs a datastore with a coarse lock around
// the entire datastore, for every single operation
func MutexWrap(d ds.Datastore) *MutexDatastore {
	return &MutexDatastore{child: d}
}

// Children implements Shim
func (d *MutexDatastore) Children() []ds.Datastore {
	return []ds.Datastore{d.child}
}

// IsThreadSafe implements ThreadSafeDatastore
func (d *MutexDatastore) IsThreadSafe() {}

// Put implements Datastore.Put
func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) {
	d.Lock()
	defer d.Unlock()
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *MutexDatastore) Delete(key ds.Key) (err error) {
	d.Lock()
	defer d.Unlock()
	return d.child.Delete(key)
}

// KeyList implements Datastore.KeyList
func (d *MutexDatastore) Query(q dsq.Query) (dsq.Results, error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Query(q)
}

func (d *MutexDatastore) Batch() (ds.Batch, error) {
	d.RLock()
	defer d.RUnlock()
	bds, ok := d.child.(ds.Batching)
	if !ok {
		return nil, ds.ErrBatchUnsupported
	}

	b, err := bds.Batch()
	if err != nil {
		return nil, err
	}
	return &syncBatch{
		batch: b,
		mds:   d,
	}, nil
}

func (d *MutexDatastore) Close() error {
	d.RWMutex.Lock()
	defer d.RWMutex.Unlock()
	if c, ok := d.child.(io.Closer); ok {
		return c.Close()
	}
	return nil
}

type syncBatch struct {
	batch ds.Batch
	mds   *MutexDatastore
}

func (b *syncBatch) Put(key ds.Key, val interface{}) error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Put(key, val)
}

func (b *syncBatch) Delete(key ds.Key) error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Delete(key)
}

func (b *syncBatch) Commit() error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Commit()
}

func (d *MutexDatastore) Check() error {
	if c, ok := d.child.(ds.CheckedDatastore); ok {
		return c.Check()
	}
	return nil
}

func (d *MutexDatastore) Scrub() error {
	if c, ok := d.child.(ds.ScrubbedDatastore); ok {
		return c.Scrub()
	}
	return nil
}

func (d *MutexDatastore) CollectGarbage() error {
	if c, ok := d.child.(ds.GCDatastore); ok {
		return c.CollectGarbage()
	}
	return nil
}
<commit_msg>Add locks to maintenance methods is syncds<commit_after>package sync

import (
	"io"
	"sync"

	ds "github.com/ipfs/go-datastore"
	dsq "github.com/ipfs/go-datastore/query"
)

// MutexDatastore contains a child datastire and a mutex.
// used for coarse sync
type MutexDatastore struct {
	sync.RWMutex

	child ds.Datastore
}

// MutexWrap constructs a datastore with a coarse lock around
// the entire datastore, for every single operation
func MutexWrap(d ds.Datastore) *MutexDatastore {
	return &MutexDatastore{child: d}
}

// Children implements Shim
func (d *MutexDatastore) Children() []ds.Datastore {
	return []ds.Datastore{d.child}
}

// IsThreadSafe implements ThreadSafeDatastore
func (d *MutexDatastore) IsThreadSafe() {}

// Put implements Datastore.Put
func (d *MutexDatastore) Put(key ds.Key, value interface{}) (err error) {
	d.Lock()
	defer d.Unlock()
	return d.child.Put(key, value)
}

// Get implements Datastore.Get
func (d *MutexDatastore) Get(key ds.Key) (value interface{}, err error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Get(key)
}

// Has implements Datastore.Has
func (d *MutexDatastore) Has(key ds.Key) (exists bool, err error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Has(key)
}

// Delete implements Datastore.Delete
func (d *MutexDatastore) Delete(key ds.Key) (err error) {
	d.Lock()
	defer d.Unlock()
	return d.child.Delete(key)
}

// KeyList implements Datastore.KeyList
func (d *MutexDatastore) Query(q dsq.Query) (dsq.Results, error) {
	d.RLock()
	defer d.RUnlock()
	return d.child.Query(q)
}

func (d *MutexDatastore) Batch() (ds.Batch, error) {
	d.RLock()
	defer d.RUnlock()
	bds, ok := d.child.(ds.Batching)
	if !ok {
		return nil, ds.ErrBatchUnsupported
	}

	b, err := bds.Batch()
	if err != nil {
		return nil, err
	}
	return &syncBatch{
		batch: b,
		mds:   d,
	}, nil
}

func (d *MutexDatastore) Close() error {
	d.RWMutex.Lock()
	defer d.RWMutex.Unlock()
	if c, ok := d.child.(io.Closer); ok {
		return c.Close()
	}
	return nil
}

type syncBatch struct {
	batch ds.Batch
	mds   *MutexDatastore
}

func (b *syncBatch) Put(key ds.Key, val interface{}) error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Put(key, val)
}

func (b *syncBatch) Delete(key ds.Key) error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Delete(key)
}

func (b *syncBatch) Commit() error {
	b.mds.Lock()
	defer b.mds.Unlock()
	return b.batch.Commit()
}

func (d *MutexDatastore) Check() error {
	if c, ok := d.child.(ds.CheckedDatastore); ok {
		d.RWMutex.Lock()
		defer d.RWMutex.Unlock()
		return c.Check()
	}
	return nil
}

func (d *MutexDatastore) Scrub() error {
	if c, ok := d.child.(ds.ScrubbedDatastore); ok {
		d.RWMutex.Lock()
		defer d.RWMutex.Unlock()
		return c.Scrub()
	}
	return nil
}

func (d *MutexDatastore) CollectGarbage() error {
	if c, ok := d.child.(ds.GCDatastore); ok {
		d.RWMutex.Lock()
		defer d.RWMutex.Unlock()
		return c.CollectGarbage()
	}
	return nil
}
<|endoftext|>