<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		fmt.Println("Please specify a URL to check")
		os.Exit(-1)
	}
	seedUrl = os.Args[1]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<commit_msg>allow the use of an ENV var to check - makes life easier on the magic auto-builds<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		seedUrl = os.Getenv("CHECKURL")
		if seedUrl == "" {
			fmt.Println("Please specify a URL to check")
			os.Exit(-1)
		}
	} else {
		seedUrl = os.Args[1]
	}

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/html"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	//fmt.Println("\t crawled \"" + req.url + "\"")

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl = os.Args[1]

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	seedUrls := os.Args[1:]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	for _, url := range seedUrls {
		new := NewUrl{
			from: "",
			url:  url,
		}
		chUrls <- new
	}

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || len(chWork) > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f
			// fmt.Printf("(w%d, w%d)", len(chWork), len(chUrls))

		case ret := <-chFinished:
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	for code, count := range summary {
		fmt.Printf("\t\tStatus %d : %d\n", code, count)
	}

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(summary[404])
}
<commit_msg>Lets let the last 10 workers complete before we exit :)<commit_after>package main

import (
	"fmt"
	"golang.org/x/net/html"
	"net/http"
	"net/url"
	"os"
	"strings"
)

// Helper function to pull the href attribute from a Token
func getHref(t html.Token) (ok bool, href string) {
	// Iterate over all of the Token's attributes until we find an "href"
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	// "bare" return will return the variables (ok, href) as defined in
	// the function definition
	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	//fmt.Println("\t crawled \"" + req.url + "\"")

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			// End of the document, we're done
			return
		case tt == html.StartTagToken:
			t := z.Token()

			// Check if the token is an <a> tag
			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			// Extract the href value, if there is one
			ok, newUrl := getHref(t)
			if !ok {
				continue
			}
			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl = os.Args[1]

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	seedUrls := os.Args[1:]

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 1000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 10; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	for _, url := range seedUrls {
		new := NewUrl{
			from: "",
			url:  url,
		}
		chUrls <- new
	}

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	for code, count := range summary {
		fmt.Printf("\t\tStatus %d : %d\n", code, count)
	}

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(summary[404])
}
<|endoftext|>


<commit_before>// Copyright 2012 Adam Sadovsky. All rights reserved.

package tadue

import (
	"errors"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func updateAll(typeName string, makeFn func() interface{}, updateFn func(value interface{}) bool,
	c *Context) error {
	q := datastore.NewQuery(typeName).KeysOnly()
	keys, err := q.GetAll(c.Aec(), nil)
	CheckError(err)

	for _, key := range keys {
		err := datastore.RunInTransaction(c.Aec(), func(aec appengine.Context) error {
			value := makeFn()
			if err := datastore.Get(aec, key, value); err != nil {
				return err
			}
			if updateFn(value) {
				if _, err := datastore.Put(aec, key, value); err != nil {
					return err
				}
			}
			return nil
		}, nil)

		if err != nil {
			return err
		}
	}
	return nil
}

func fixUserRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &User{}
	}
	updateFn := func(value interface{}) bool {
		user, ok := value.(*User)
		Assert(ok, "%v", value)
		user.Email = ParseEmail(user.Email)
		user.PayPalEmail = ParseEmail(user.PayPalEmail)
		return true
	}
	CheckError(updateAll("User", makeFn, updateFn, c))
}

func fixPayRequestRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &PayRequest{}
	}
	updateFn := func(value interface{}) bool {
		req, ok := value.(*PayRequest)
		Assert(ok, "%v", value)
		req.PayeeEmail = ParseEmail(req.PayeeEmail)
		req.PayerEmail = ParseEmail(req.PayerEmail)
		return true
	}
	CheckError(updateAll("PayRequest", makeFn, updateFn, c))
}

func fixSessionRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &Session{}
	}
	updateFn := func(value interface{}) bool {
		session, ok := value.(*Session)
		Assert(ok, "%v", value)
		session.Email = ParseEmail(session.Email)
		return true
	}
	CheckError(updateAll("Session", makeFn, updateFn, c))
}

func fixUserIdRecordsOrDie(c *Context) {
	q := datastore.NewQuery("UserId").KeysOnly()
	keys, err := q.GetAll(c.Aec(), nil)
	CheckError(err)

	for _, key := range keys {
		oldEmail := key.StringID()
		newEmail := ParseEmail(oldEmail)
		if (oldEmail == newEmail) {
			continue
		}

		oldUserIdKey := ToUserIdKey(c.Aec(), oldEmail)
		newUserIdKey := ToUserIdKey(c.Aec(), newEmail)

		err := datastore.RunInTransaction(c.Aec(), func(aec appengine.Context) error {
			userIdStruct := &UserId{}

			// Check that there's no UserId record for newEmail.
			if err := datastore.Get(aec, newUserIdKey, userIdStruct); err != datastore.ErrNoSuchEntity {
				if err == nil {
					return errors.New(
						fmt.Sprintf("UserId already exists: %q, %q", oldEmail, newEmail))
				}
				return err
			}

			// Get the UserId record for oldEmail, and write one for newEmail.
			if err := datastore.Get(aec, oldUserIdKey, userIdStruct); err != nil {
				return err
			}
			if _, err := datastore.Put(aec, newUserIdKey, userIdStruct); err != nil {
				return err
			}

			// Delete the UserId record for oldEmail.
			if err := datastore.Delete(aec, oldUserIdKey); err != nil {
				return err
			}
			return nil
		}, makeXG())
		CheckError(err)
	}
}

func handleFix(w http.ResponseWriter, r *http.Request, c *Context) {
	fixPayRequestRecordsOrDie(c)
	fixSessionRecordsOrDie(c)
	fixUserRecordsOrDie(c)
	fixUserIdRecordsOrDie(c)
	ServeInfo(w, "Done")
}
<commit_msg>gofmt fix<commit_after>// Copyright 2012 Adam Sadovsky. All rights reserved.

package tadue

import (
	"errors"
	"fmt"
	"net/http"

	"appengine"
	"appengine/datastore"
)

func updateAll(typeName string, makeFn func() interface{}, updateFn func(value interface{}) bool,
	c *Context) error {
	q := datastore.NewQuery(typeName).KeysOnly()
	keys, err := q.GetAll(c.Aec(), nil)
	CheckError(err)

	for _, key := range keys {
		err := datastore.RunInTransaction(c.Aec(), func(aec appengine.Context) error {
			value := makeFn()
			if err := datastore.Get(aec, key, value); err != nil {
				return err
			}
			if updateFn(value) {
				if _, err := datastore.Put(aec, key, value); err != nil {
					return err
				}
			}
			return nil
		}, nil)

		if err != nil {
			return err
		}
	}
	return nil
}

func fixUserRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &User{}
	}
	updateFn := func(value interface{}) bool {
		user, ok := value.(*User)
		Assert(ok, "%v", value)
		user.Email = ParseEmail(user.Email)
		user.PayPalEmail = ParseEmail(user.PayPalEmail)
		return true
	}
	CheckError(updateAll("User", makeFn, updateFn, c))
}

func fixPayRequestRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &PayRequest{}
	}
	updateFn := func(value interface{}) bool {
		req, ok := value.(*PayRequest)
		Assert(ok, "%v", value)
		req.PayeeEmail = ParseEmail(req.PayeeEmail)
		req.PayerEmail = ParseEmail(req.PayerEmail)
		return true
	}
	CheckError(updateAll("PayRequest", makeFn, updateFn, c))
}

func fixSessionRecordsOrDie(c *Context) {
	makeFn := func() interface{} {
		return &Session{}
	}
	updateFn := func(value interface{}) bool {
		session, ok := value.(*Session)
		Assert(ok, "%v", value)
		session.Email = ParseEmail(session.Email)
		return true
	}
	CheckError(updateAll("Session", makeFn, updateFn, c))
}

func fixUserIdRecordsOrDie(c *Context) {
	q := datastore.NewQuery("UserId").KeysOnly()
	keys, err := q.GetAll(c.Aec(), nil)
	CheckError(err)

	for _, key := range keys {
		oldEmail := key.StringID()
		newEmail := ParseEmail(oldEmail)
		if oldEmail == newEmail {
			continue
		}

		oldUserIdKey := ToUserIdKey(c.Aec(), oldEmail)
		newUserIdKey := ToUserIdKey(c.Aec(), newEmail)

		err := datastore.RunInTransaction(c.Aec(), func(aec appengine.Context) error {
			userIdStruct := &UserId{}

			// Check that there's no UserId record for newEmail.
			if err := datastore.Get(aec, newUserIdKey, userIdStruct); err != datastore.ErrNoSuchEntity {
				if err == nil {
					return errors.New(
						fmt.Sprintf("UserId already exists: %q, %q", oldEmail, newEmail))
				}
				return err
			}

			// Get the UserId record for oldEmail, and write one for newEmail.
			if err := datastore.Get(aec, oldUserIdKey, userIdStruct); err != nil {
				return err
			}
			if _, err := datastore.Put(aec, newUserIdKey, userIdStruct); err != nil {
				return err
			}

			// Delete the UserId record for oldEmail.
			if err := datastore.Delete(aec, oldUserIdKey); err != nil {
				return err
			}
			return nil
		}, makeXG())
		CheckError(err)
	}
}

func handleFix(w http.ResponseWriter, r *http.Request, c *Context) {
	fixPayRequestRecordsOrDie(c)
	fixSessionRecordsOrDie(c)
	fixUserRecordsOrDie(c)
	fixUserIdRecordsOrDie(c)
	ServeInfo(w, "Done")
}
<|endoftext|>


<commit_before>package searchers

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	aw "github.com/deanishe/awgo"
	"github.com/rkoval/alfred-aws-console-services-workflow/core"
)

func SearchEC2SecurityGroups(wf *aw.Workflow, query string) error {
	sess, cfg := core.LoadAWSConfig()
	svc := ec2.New(sess, cfg)

	values := []*string{
		aws.String(strings.Join([]string{"*", query, "*"}, "")),
	}

	var name string
	if strings.HasPrefix(query, "sg-") {
		// assume we're querying by ID here
		name = "group-id"
	} else {
		name = "tag:Name"
	}
	params := &ec2.DescribeSecurityGroupsInput{
		Filters: []*ec2.Filter{
			{
				Name:   aws.String(name),
				Values: values,
			},
		},
	}

	resp, err := svc.DescribeSecurityGroups(params)
	if err != nil {
		wf.NewItem(err.Error()).
			Icon(aw.IconError)
		return err
	}
	// log.Printf("%+v\n", *resp)

	for _, securityGroup := range resp.SecurityGroups {
		var title string
		var subtitle string
		name := GetTagValue(securityGroup.Tags, "Name")
		if name != "" {
			title = name
			subtitle = *securityGroup.GroupId
		} else {
			title = *securityGroup.GroupId
		}

		if subtitle != "" {
			subtitle += " "
		}
		subtitle += *securityGroup.Description

		wf.NewItem(title).
			Subtitle(subtitle).
			Arg(fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/v2/home?region=%s#SecurityGroups:search=%s", *cfg.Region, *cfg.Region, *securityGroup.GroupId)).
			Icon(core.GetImageIcon("ec2")).
			Valid(true)
	}

	return nil
}
<commit_msg>fixed security group permalink<commit_after>package searchers

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	aw "github.com/deanishe/awgo"
	"github.com/rkoval/alfred-aws-console-services-workflow/core"
)

func SearchEC2SecurityGroups(wf *aw.Workflow, query string) error {
	sess, cfg := core.LoadAWSConfig()
	svc := ec2.New(sess, cfg)

	values := []*string{
		aws.String(strings.Join([]string{"*", query, "*"}, "")),
	}

	var name string
	if strings.HasPrefix(query, "sg-") {
		// assume we're querying by ID here
		name = "group-id"
	} else {
		name = "tag:Name"
	}
	params := &ec2.DescribeSecurityGroupsInput{
		Filters: []*ec2.Filter{
			{
				Name:   aws.String(name),
				Values: values,
			},
		},
	}

	resp, err := svc.DescribeSecurityGroups(params)
	if err != nil {
		wf.NewItem(err.Error()).
			Icon(aw.IconError)
		return err
	}
	// log.Printf("%+v\n", *resp)

	for _, securityGroup := range resp.SecurityGroups {
		var title string
		var subtitle string
		name := GetTagValue(securityGroup.Tags, "Name")
		if name != "" {
			title = name
			subtitle = *securityGroup.GroupId
		} else {
			title = *securityGroup.GroupId
		}

		if subtitle != "" {
			subtitle += " "
		}
		subtitle += *securityGroup.Description

		wf.NewItem(title).
			Subtitle(subtitle).
			Arg(fmt.Sprintf("https://%s.console.aws.amazon.com/ec2/v2/home?region=%s#SecurityGroups:group-id=%s", *cfg.Region, *cfg.Region, *securityGroup.GroupId)).
			Icon(core.GetImageIcon("ec2")).
			Valid(true)
	}

	return nil
}
<|endoftext|>


<commit_before>package task

import (
	"bufio"
	"errors"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/cenkalti/yap/datetime"
	"github.com/satori/go.uuid"
)

const taskExt = ".task"

// Task is a file stored in tasks dir.
type Task struct {
	ID          uint16
	UUID        uuid.UUID
	Title       string             `key:"title"`
	CreatedAt   time.Time          `key:"created_at"`
	CompletedAt *time.Time         `key:"completed_at"`
	DueDate     *datetime.DateTime `key:"due_date"`
	WaitDate    *datetime.DateTime `key:"wait_date"`
}

func readFile(filename string) (t Task, err error) {
	base := filepath.Base(filename)
	t.UUID, err = uuid.FromString(base[:len(base)-len(taskExt)])
	if err != nil {
		return
	}
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer checkClose(f)
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := strings.TrimSpace(scanner.Text())
		if text == "" {
			continue
		}
		parts := strings.SplitN(text, " ", 2)
		if len(parts) != 2 {
			err = errors.New("invalid task file")
			return
		}
		err = t.setKeyVal(parts[0], parts[1])
		if err != nil {
			return
		}
	}
	err = scanner.Err()
	return
}

var parsers = map[string]func(t *Task, value string) error{
	"title": func(t *Task, value string) (err error) {
		t.Title = value
		return
	},
	"created_at": func(t *Task, value string) (err error) {
		t.CreatedAt, err = time.Parse(time.RFC3339Nano, value)
		return
	},
	"completed_at": func(t *Task, value string) (err error) {
		var ctime time.Time
		ctime, err = time.Parse(time.RFC3339Nano, value)
		t.CompletedAt = &ctime
		return
	},
	"due_date": func(t *Task, value string) (err error) {
		var dt datetime.DateTime
		dt, err = datetime.Parse(value)
		t.DueDate = &dt
		return
	},
	"wait_date": func(t *Task, value string) (err error) {
		var dt datetime.DateTime
		dt, err = datetime.Parse(value)
		t.WaitDate = &dt
		return
	},
}

func (t *Task) setKeyVal(key, value string) error {
	f, ok := parsers[key]
	if !ok {
		return errors.New("invalid key")
	}
	return f(t, value)
}

// write the task to file at <dirTasks>/<UUID>.task
func (t Task) write() error {
	path := filepath.Join(dirTasks, t.UUID.String()) + taskExt
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(f)
	if err = t.writeFields(w); err != nil {
		return err
	}
	if err = w.Flush(); err != nil {
		return err
	}
	if err = f.Sync(); err != nil {
		return err
	}
	return f.Close()
}

func (t Task) writeFields(w io.Writer) error {
	val := reflect.ValueOf(t)
	for i := 0; i < val.NumField(); i++ {
		tag := val.Type().Field(i).Tag.Get("key")
		if tag == "" {
			continue
		}
		fval := val.Field(i)
		if fval.Kind() == reflect.Ptr {
			if fval.IsNil() {
				continue
			}
			fval = fval.Elem()
		}
		iface := fval.Interface()
		var s string
		switch fval.Type() {
		case reflect.TypeOf(""):
			s = iface.(string)
		case reflect.TypeOf(time.Time{}):
			s = iface.(time.Time).Format(time.RFC3339Nano)
		case reflect.TypeOf(datetime.DateTime{}):
			s = iface.(datetime.DateTime).String()
		default:
			return errors.New("unknown type")
		}
		_, err := w.Write([]byte(tag + " " + s + "\n"))
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>split func<commit_after>package task

import (
	"bufio"
	"errors"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/cenkalti/yap/datetime"
	"github.com/satori/go.uuid"
)

const taskExt = ".task"

// Task is a file stored in tasks dir.
type Task struct {
	ID          uint16
	UUID        uuid.UUID
	Title       string             `key:"title"`
	CreatedAt   time.Time          `key:"created_at"`
	CompletedAt *time.Time         `key:"completed_at"`
	DueDate     *datetime.DateTime `key:"due_date"`
	WaitDate    *datetime.DateTime `key:"wait_date"`
}

func readFile(filename string) (t Task, err error) {
	base := filepath.Base(filename)
	t.UUID, err = uuid.FromString(base[:len(base)-len(taskExt)])
	if err != nil {
		return
	}
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer checkClose(f)
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := strings.TrimSpace(scanner.Text())
		if text == "" {
			continue
		}
		parts := strings.SplitN(text, " ", 2)
		if len(parts) != 2 {
			err = errors.New("invalid task file")
			return
		}
		err = t.setKeyVal(parts[0], parts[1])
		if err != nil {
			return
		}
	}
	err = scanner.Err()
	return
}

var parsers = map[string]func(t *Task, value string) error{
	"title": func(t *Task, value string) (err error) {
		t.Title = value
		return
	},
	"created_at": func(t *Task, value string) (err error) {
		t.CreatedAt, err = time.Parse(time.RFC3339Nano, value)
		return
	},
	"completed_at": func(t *Task, value string) (err error) {
		var ctime time.Time
		ctime, err = time.Parse(time.RFC3339Nano, value)
		t.CompletedAt = &ctime
		return
	},
	"due_date": func(t *Task, value string) (err error) {
		var dt datetime.DateTime
		dt, err = datetime.Parse(value)
		t.DueDate = &dt
		return
	},
	"wait_date": func(t *Task, value string) (err error) {
		var dt datetime.DateTime
		dt, err = datetime.Parse(value)
		t.WaitDate = &dt
		return
	},
}

func (t *Task) setKeyVal(key, value string) error {
	f, ok := parsers[key]
	if !ok {
		return errors.New("invalid key")
	}
	return f(t, value)
}

// write the task to file at <dirTasks>/<UUID>.task
func (t Task) write() error {
	path := filepath.Join(dirTasks, t.UUID.String()) + taskExt
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(f)
	if err = t.writeFields(w); err != nil {
		return err
	}
	if err = w.Flush(); err != nil {
		return err
	}
	if err = f.Sync(); err != nil {
		return err
	}
	return f.Close()
}

func (t Task) writeFields(w io.Writer) error {
	val := reflect.ValueOf(t)
	for i := 0; i < val.NumField(); i++ {
		tag := val.Type().Field(i).Tag.Get("key")
		if tag == "" {
			continue
		}
		fval := val.Field(i)
		if fval.Kind() == reflect.Ptr {
			if fval.IsNil() {
				continue
			}
			fval = fval.Elem()
		}
		_, err := w.Write([]byte(tag + " " + stringValue(fval) + "\n"))
		if err != nil {
			return err
		}
	}
	return nil
}

func stringValue(v reflect.Value) string {
	i := v.Interface()
	switch v.Type() {
	case reflect.TypeOf(""):
		return i.(string)
	case reflect.TypeOf(time.Time{}):
		return i.(time.Time).Format(time.RFC3339Nano)
	case reflect.TypeOf(datetime.DateTime{}):
		return i.(datetime.DateTime).String()
	default:
		panic("unknown type")
	}
}
<|endoftext|>


<commit_before>package main

const (
	// bucketBytes is the number of bytes of the key used to determine the hash bucket.
	// If we wanted to dynamically scale through many different orders of magnitude,
	// we'd need to adjust bucketBytes over time. But this stays static for now,
	// so this hashtable is meant for holding 10s of millions of entries.
	bucketBytes = 3
	numBuckets  = 1 << (8 * bucketBytes)
	// initialBucketSize determines the number of slots to place at each bucket position initially
	// (this grows as the table is resized).
	initialBucketSize = 3
	minLoadFactor     = 0.35
	maxLoadFactor     = 0.65
)

type RefMapEntry struct {
	occ bool
	key [20]byte
	val RecordRef
}

// A RefMap maps SHA-1 key hashes ([20]byte) to RecordRefs.
// It is implemented as an open-addressed hash table with linear probing.
// It is resized when the the load factor grows too high.
// Using this instead of a Go map type avoids GC penalties from using very large maps.
type RefMap struct {
	slots      []RefMapEntry
	bucketSize int
	len        int
	cap        int
}

func newRefMap(bucketSize int) *RefMap {
	n := numBuckets * bucketSize
	return &RefMap{
		slots:      make([]RefMapEntry, n),
		bucketSize: bucketSize,
		cap:        n,
	}
}

func NewRefMap() *RefMap {
	return newRefMap(initialBucketSize)
}

func (m *RefMap) Put(key [20]byte, val RecordRef) (exists bool) {
	for i := m.slotIdx(key); ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			// We've found an empty slot to put this entry.
			m.slots[i] = RefMapEntry{
				occ: true,
				key: key,
				val: val,
			}
			m.len++
			if m.load() > maxLoadFactor {
				m.resize(m.bucketSize + 1)
			}
			return false
		}
		if key == m.slots[i].key {
			m.slots[i].val = val
			return true
		}
	}
}

func (m *RefMap) Get(key [20]byte) (val RecordRef, ok bool) {
	for i := m.slotIdx(key); ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			return RecordRef{}, false
		}
		if key == m.slots[i].key {
			return m.slots[i].val, true
		}
	}
}

func (m *RefMap) Delete(key [20]byte) (ok bool) {
	i := m.slotIdx(key)
	for ; ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			return false
		}
		if key == m.slots[i].key {
			break
		}
	}
	// We've found our entry: it is at i. When we remove it, there's going to be a gap in the table which could
	// break the probing invariant: for a key k in the hash table, there are no unoccupied slots between k's
	// initial slot index and the location where k resides.
	// So, we'll iterate forwards through the table, moving items into the unoccupied slot when we can.
	// See pseudocode at http://en.wikipedia.org/wiki/Open_addressing
	j := i
	for {
		m.slots[i].occ = false
	findSwappable:
		j = (j + 1) % m.cap
		if !m.slots[j].occ {
			break
		}
		k := m.slotIdx(m.slots[j].key)
		if i <= j && i < k && k <= j {
			goto findSwappable
		}
		if i > j && (i < k || k <= j) {
			goto findSwappable
		}
		// We found an entry that can be swapped into slot i without breaking the probing invariant.
		m.slots[i] = m.slots[j]
		i = j
	}
	m.len--
	if m.bucketSize > initialBucketSize && m.load() < minLoadFactor {
		m.resize(m.bucketSize - 1)
	}
	return true
}

func (m *RefMap) Len() int { return m.len }

// TODO: check this is inlined
func (m *RefMap) slotIdx(k [20]byte) int {
	return (int(k[0])<<16 + int(k[1])<<8 + int(k[2])) * m.bucketSize
}

func (m *RefMap) load() float64 {
	return float64(m.len) / float64(m.cap)
}

func (m *RefMap) resize(newBucketSize int) {
	m2 := newRefMap(newBucketSize)
	for i := range m.slots {
		if m.slots[i].occ {
			m2.Put(m.slots[i].key, m.slots[i].val)
		}
	}
	*m = *m2
}
<commit_msg>Remove TODO<commit_after>package main

const (
	// bucketBytes is the number of bytes of the key used to determine the hash bucket.
	// If we wanted to dynamically scale through many different orders of magnitude,
	// we'd need to adjust bucketBytes over time. But this stays static for now,
	// so this hashtable is meant for holding 10s of millions of entries.
	bucketBytes = 3
	numBuckets  = 1 << (8 * bucketBytes)
	// initialBucketSize determines the number of slots to place at each bucket position initially
	// (this grows as the table is resized).
	initialBucketSize = 3
	minLoadFactor     = 0.35
	maxLoadFactor     = 0.65
)

type RefMapEntry struct {
	occ bool
	key [20]byte
	val RecordRef
}

// A RefMap maps SHA-1 key hashes ([20]byte) to RecordRefs.
// It is implemented as an open-addressed hash table with linear probing.
// It is resized when the the load factor grows too high.
// Using this instead of a Go map type avoids GC penalties from using very large maps.
type RefMap struct {
	slots      []RefMapEntry
	bucketSize int
	len        int
	cap        int
}

func newRefMap(bucketSize int) *RefMap {
	n := numBuckets * bucketSize
	return &RefMap{
		slots:      make([]RefMapEntry, n),
		bucketSize: bucketSize,
		cap:        n,
	}
}

func NewRefMap() *RefMap {
	return newRefMap(initialBucketSize)
}

func (m *RefMap) Put(key [20]byte, val RecordRef) (exists bool) {
	for i := m.slotIdx(key); ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			// We've found an empty slot to put this entry.
			m.slots[i] = RefMapEntry{
				occ: true,
				key: key,
				val: val,
			}
			m.len++
			if m.load() > maxLoadFactor {
				m.resize(m.bucketSize + 1)
			}
			return false
		}
		if key == m.slots[i].key {
			m.slots[i].val = val
			return true
		}
	}
}

func (m *RefMap) Get(key [20]byte) (val RecordRef, ok bool) {
	for i := m.slotIdx(key); ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			return RecordRef{}, false
		}
		if key == m.slots[i].key {
			return m.slots[i].val, true
		}
	}
}

func (m *RefMap) Delete(key [20]byte) (ok bool) {
	i := m.slotIdx(key)
	for ; ; i = (i + 1) % m.cap {
		if !m.slots[i].occ {
			return false
		}
		if key == m.slots[i].key {
			break
		}
	}
	// We've found our entry: it is at i. When we remove it, there's going to be a gap in the table which could
	// break the probing invariant: for a key k in the hash table, there are no unoccupied slots between k's
	// initial slot index and the location where k resides.
	// So, we'll iterate forwards through the table, moving items into the unoccupied slot when we can.
	// See pseudocode at http://en.wikipedia.org/wiki/Open_addressing
	j := i
	for {
		m.slots[i].occ = false
	findSwappable:
		j = (j + 1) % m.cap
		if !m.slots[j].occ {
			break
		}
		k := m.slotIdx(m.slots[j].key)
		if i <= j && i < k && k <= j {
			goto findSwappable
		}
		if i > j && (i < k || k <= j) {
			goto findSwappable
		}
		// We found an entry that can be swapped into slot i without breaking the probing invariant.
		m.slots[i] = m.slots[j]
		i = j
	}
	m.len--
	if m.bucketSize > initialBucketSize && m.load() < minLoadFactor {
		m.resize(m.bucketSize - 1)
	}
	return true
}

func (m *RefMap) Len() int { return m.len }

func (m *RefMap) slotIdx(k [20]byte) int {
	return (int(k[0])<<16 + int(k[1])<<8 + int(k[2])) * m.bucketSize
}

func (m *RefMap) load() float64 {
	return float64(m.len) / float64(m.cap)
}

func (m *RefMap) resize(newBucketSize int) {
	m2 := newRefMap(newBucketSize)
	for i := range m.slots {
		if m.slots[i].occ {
			m2.Put(m.slots[i].key, m.slots[i].val)
		}
	}
	*m = *m2
}
<|endoftext|>


<commit_before>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
)

func (srv *server) handleWebcam(w http.ResponseWriter, r *http.Request) {
	img, err := http.Get("http://195.221.117.245:80/axis-cgi/jpg/image.cgi")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer img.Body.Close()
	buf, err := ioutil.ReadAll(img.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
	_, err = w.Write(buf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (srv *server) fetchWebcamImage() string {
	url := "http://" + srv.Addr + "/webcam"
	//url := "http://195.221.117.245/axis-cgi/jpg/image.cgi"
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("error fetching webcam image: %v\n", err)
		return ""
	}
	defer resp.Body.Close()
	buf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("error reading webcam image: %v\n", err)
		return ""
	}

	return base64.StdEncoding.EncodeToString(buf)
}
<commit_msg>webcam: cleanup<commit_after>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/base64"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
)

func (srv *server) handleWebcam(w http.ResponseWriter, r *http.Request) {
	img, err := http.Get("http://195.221.117.245:80/axis-cgi/jpg/image.cgi")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer img.Body.Close()
	buf, err := ioutil.ReadAll(img.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
	_, err = w.Write(buf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

func (srv *server) fetchWebcamImage() string {
	url := "http://" + srv.Addr + "/webcam"
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("error fetching webcam image: %v\n", err)
		return ""
	}
	defer resp.Body.Close()
	buf, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("error reading webcam image: %v\n", err)
		return ""
	}

	return base64.StdEncoding.EncodeToString(buf)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP sockets

package net

import (
	"os"
	"syscall"
)

// Should we try to use the IPv4 socket interface if we're
// only dealing with IPv4 sockets?  As long as the host system
// understands IPv6, it's okay to pass IPv4 addresses to the IPv6
// interface.  That simplifies our code and is most general.
// Unfortunately, we need to run on kernels built without IPv6 support too.
// So probe the kernel to figure it out.
func kernelSupportsIPv6() bool {
	// FreeBSD does not support this sort of interface.
	if syscall.OS == "freebsd" {
		return false
	}
	fd, e := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
	if fd >= 0 {
		syscall.Close(fd)
	}
	return e == 0
}

var preferIPv4 = !kernelSupportsIPv6()

// TODO(rsc): if syscall.OS == "linux", we're supposd to read
// /proc/sys/net/core/somaxconn,
// to take advantage of kernels that have raised the limit.
func listenBacklog() int { return syscall.SOMAXCONN }

// Internet sockets (TCP, UDP)

// A sockaddr represents a TCP or UDP network address that can
// be converted into a syscall.Sockaddr.
type sockaddr interface {
	Addr
	sockaddr(family int) (syscall.Sockaddr, os.Error)
	family() int
}

func internetSocket(net string, laddr, raddr sockaddr, socktype, proto int, mode string, toAddr func(syscall.Sockaddr) Addr) (fd *netFD, err os.Error) {
	// Figure out IP version.
	// If network has a suffix like "tcp4", obey it.
	var oserr os.Error
	family := syscall.AF_INET6
	switch net[len(net)-1] {
	case '4':
		family = syscall.AF_INET
	case '6':
		// nothing to do
	default:
		// Otherwise, guess.
		// If the addresses are IPv4 and we prefer IPv4, use 4; else 6.
		if preferIPv4 &&
			(laddr == nil || laddr.family() == syscall.AF_INET) &&
			(raddr == nil || raddr.family() == syscall.AF_INET) {
			family = syscall.AF_INET
		}
	}

	var la, ra syscall.Sockaddr
	if laddr != nil {
		if la, oserr = laddr.sockaddr(family); err != nil {
			goto Error
		}
	}
	if raddr != nil {
		if ra, oserr = raddr.sockaddr(family); err != nil {
			goto Error
		}
	}
	fd, oserr = socket(net, family, socktype, proto, la, ra, toAddr)
	if oserr != nil {
		goto Error
	}
	return fd, nil

Error:
	addr := raddr
	if mode == "listen" {
		addr = laddr
	}
	return nil, &OpError{mode, net, addr, oserr}
}

func getip(fd int, remote bool) (ip []byte, port int, ok bool) {
	// No attempt at error reporting because
	// there are no possible errors, and the
	// caller won't report them anyway.
	var sa syscall.Sockaddr
	if remote {
		sa, _ = syscall.Getpeername(fd)
	} else {
		sa, _ = syscall.Getsockname(fd)
	}
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		return sa.Addr[0:], sa.Port, true
	case *syscall.SockaddrInet6:
		return sa.Addr[0:], sa.Port, true
	}
	return
}

type InvalidAddrError string

func (e InvalidAddrError) String() string  { return string(e) }
func (e InvalidAddrError) Timeout() bool   { return false }
func (e InvalidAddrError) Temporary() bool { return false }

func ipToSockaddr(family int, ip IP, port int) (syscall.Sockaddr, os.Error) {
	switch family {
	case syscall.AF_INET:
		if len(ip) == 0 {
			ip = IPv4zero
		}
		if ip = ip.To4(); ip == nil {
			return nil, InvalidAddrError("non-IPv4 address")
		}
		s := new(syscall.SockaddrInet4)
		for i := 0; i < IPv4len; i++ {
			s.Addr[i] = ip[i]
		}
		s.Port = port
		return s, nil
	case syscall.AF_INET6:
		if len(ip) == 0 {
			ip = IPzero
		}
		// IPv4 callers use 0.0.0.0 to mean "announce on any available address".
		// In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0",
		// which it refuses to do.  Rewrite to the IPv6 all zeros.
		if p4 := ip.To4(); p4 != nil && p4[0] == 0 && p4[1] == 0 && p4[2] == 0 && p4[3] == 0 {
			ip = IPzero
		}
		if ip = ip.To16(); ip == nil {
			return nil, InvalidAddrError("non-IPv6 address")
		}
		s := new(syscall.SockaddrInet6)
		for i := 0; i < IPv6len; i++ {
			s.Addr[i] = ip[i]
		}
		s.Port = port
		return s, nil
	}
	return nil, InvalidAddrError("unexpected socket family")
}

// Split "host:port" into "host" and "port".
// Host cannot contain colons unless it is bracketed.
func splitHostPort(hostport string) (host, port string, err os.Error) {
	// The port starts after the last colon.
	i := last(hostport, ':')
	if i < 0 {
		err = &AddrError{"missing port in address", hostport}
		return
	}

	host, port = hostport[0:i], hostport[i+1:]

	// Can put brackets around host ...
	if len(host) > 0 && host[0] == '[' && host[len(host)-1] == ']' {
		host = host[1 : len(host)-1]
	} else {
		// ... but if there are no brackets, no colons.
		if byteIndex(host, ':') >= 0 {
			err = &AddrError{"too many colons in address", hostport}
			return
		}
	}
	return
}

// Join "host" and "port" into "host:port".
// If host contains colons, will join into "[host]:port".
func joinHostPort(host, port string) string {
	// If host has colons, have to bracket it.
	if byteIndex(host, ':') >= 0 {
		return "[" + host + "]:" + port
	}
	return host + ":" + port
}

// Convert "host:port" into IP address and port.
func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) {
	host, port, err := splitHostPort(hostport)
	if err != nil {
		goto Error
	}

	var addr IP
	if host != "" {
		// Try as an IP address.
		addr = ParseIP(host)
		if addr == nil {
			// Not an IP address.  Try as a DNS name.
			_, addrs, err1 := LookupHost(host)
			if err1 != nil {
				err = err1
				goto Error
			}
			addr = ParseIP(addrs[0])
			if addr == nil {
				// should not happen
				err = &AddrError{"LookupHost returned invalid address", addrs[0]}
				goto Error
			}
		}
	}

	p, i, ok := dtoi(port, 0)
	if !ok || i != len(port) {
		p, err = LookupPort(net, port)
		if err != nil {
			goto Error
		}
	}
	if p < 0 || p > 0xFFFF {
		err = &AddrError{"invalid port", port}
		goto Error
	}

	return addr, p, nil

Error:
	return nil, 0, err
}
<commit_msg>net/ipsock.go: fix error checking bug Check oserr, not err for non-nil (err is never changed)<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP sockets

package net

import (
	"os"
	"syscall"
)

// Should we try to use the IPv4 socket interface if we're
// only dealing with IPv4 sockets?  As long as the host system
// understands IPv6, it's okay to pass IPv4 addresses to the IPv6
// interface.  That simplifies our code and is most general.
// Unfortunately, we need to run on kernels built without IPv6 support too.
// So probe the kernel to figure it out.
func kernelSupportsIPv6() bool {
	// FreeBSD does not support this sort of interface.
	if syscall.OS == "freebsd" {
		return false
	}
	fd, e := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)
	if fd >= 0 {
		syscall.Close(fd)
	}
	return e == 0
}

var preferIPv4 = !kernelSupportsIPv6()

// TODO(rsc): if syscall.OS == "linux", we're supposd to read
// /proc/sys/net/core/somaxconn,
// to take advantage of kernels that have raised the limit.
func listenBacklog() int { return syscall.SOMAXCONN }

// Internet sockets (TCP, UDP)

// A sockaddr represents a TCP or UDP network address that can
// be converted into a syscall.Sockaddr.
type sockaddr interface {
	Addr
	sockaddr(family int) (syscall.Sockaddr, os.Error)
	family() int
}

func internetSocket(net string, laddr, raddr sockaddr, socktype, proto int, mode string, toAddr func(syscall.Sockaddr) Addr) (fd *netFD, err os.Error) {
	// Figure out IP version.
	// If network has a suffix like "tcp4", obey it.
	var oserr os.Error
	family := syscall.AF_INET6
	switch net[len(net)-1] {
	case '4':
		family = syscall.AF_INET
	case '6':
		// nothing to do
	default:
		// Otherwise, guess.
		// If the addresses are IPv4 and we prefer IPv4, use 4; else 6.
		if preferIPv4 &&
			(laddr == nil || laddr.family() == syscall.AF_INET) &&
			(raddr == nil || raddr.family() == syscall.AF_INET) {
			family = syscall.AF_INET
		}
	}

	var la, ra syscall.Sockaddr
	if laddr != nil {
		if la, oserr = laddr.sockaddr(family); oserr != nil {
			goto Error
		}
	}
	if raddr != nil {
		if ra, oserr = raddr.sockaddr(family); oserr != nil {
			goto Error
		}
	}
	fd, oserr = socket(net, family, socktype, proto, la, ra, toAddr)
	if oserr != nil {
		goto Error
	}
	return fd, nil

Error:
	addr := raddr
	if mode == "listen" {
		addr = laddr
	}
	return nil, &OpError{mode, net, addr, oserr}
}

func getip(fd int, remote bool) (ip []byte, port int, ok bool) {
	// No attempt at error reporting because
	// there are no possible errors, and the
	// caller won't report them anyway.
	var sa syscall.Sockaddr
	if remote {
		sa, _ = syscall.Getpeername(fd)
	} else {
		sa, _ = syscall.Getsockname(fd)
	}
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		return sa.Addr[0:], sa.Port, true
	case *syscall.SockaddrInet6:
		return sa.Addr[0:], sa.Port, true
	}
	return
}

type InvalidAddrError string

func (e InvalidAddrError) String() string  { return string(e) }
func (e InvalidAddrError) Timeout() bool   { return false }
func (e InvalidAddrError) Temporary() bool { return false }

func ipToSockaddr(family int, ip IP, port int) (syscall.Sockaddr, os.Error) {
	switch family {
	case syscall.AF_INET:
		if len(ip) == 0 {
			ip = IPv4zero
		}
		if ip = ip.To4(); ip == nil {
			return nil, InvalidAddrError("non-IPv4 address")
		}
		s := new(syscall.SockaddrInet4)
		for i := 0; i < IPv4len; i++ {
			s.Addr[i] = ip[i]
		}
		s.Port = port
		return s, nil
	case syscall.AF_INET6:
		if len(ip) == 0 {
			ip = IPzero
		}
		// IPv4 callers use 0.0.0.0 to mean "announce on any available address".
		// In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0",
		// which it refuses to do.  Rewrite to the IPv6 all zeros.
		if p4 := ip.To4(); p4 != nil && p4[0] == 0 && p4[1] == 0 && p4[2] == 0 && p4[3] == 0 {
			ip = IPzero
		}
		if ip = ip.To16(); ip == nil {
			return nil, InvalidAddrError("non-IPv6 address")
		}
		s := new(syscall.SockaddrInet6)
		for i := 0; i < IPv6len; i++ {
			s.Addr[i] = ip[i]
		}
		s.Port = port
		return s, nil
	}
	return nil, InvalidAddrError("unexpected socket family")
}

// Split "host:port" into "host" and "port".
// Host cannot contain colons unless it is bracketed.
func splitHostPort(hostport string) (host, port string, err os.Error) {
	// The port starts after the last colon.
	i := last(hostport, ':')
	if i < 0 {
		err = &AddrError{"missing port in address", hostport}
		return
	}

	host, port = hostport[0:i], hostport[i+1:]

	// Can put brackets around host ...
	if len(host) > 0 && host[0] == '[' && host[len(host)-1] == ']' {
		host = host[1 : len(host)-1]
	} else {
		// ... but if there are no brackets, no colons.
		if byteIndex(host, ':') >= 0 {
			err = &AddrError{"too many colons in address", hostport}
			return
		}
	}
	return
}

// Join "host" and "port" into "host:port".
// If host contains colons, will join into "[host]:port".
func joinHostPort(host, port string) string {
	// If host has colons, have to bracket it.
	if byteIndex(host, ':') >= 0 {
		return "[" + host + "]:" + port
	}
	return host + ":" + port
}

// Convert "host:port" into IP address and port.
func hostPortToIP(net, hostport string) (ip IP, iport int, err os.Error) {
	host, port, err := splitHostPort(hostport)
	if err != nil {
		goto Error
	}

	var addr IP
	if host != "" {
		// Try as an IP address.
		addr = ParseIP(host)
		if addr == nil {
			// Not an IP address.  Try as a DNS name.
			_, addrs, err1 := LookupHost(host)
			if err1 != nil {
				err = err1
				goto Error
			}
			addr = ParseIP(addrs[0])
			if addr == nil {
				// should not happen
				err = &AddrError{"LookupHost returned invalid address", addrs[0]}
				goto Error
			}
		}
	}

	p, i, ok := dtoi(port, 0)
	if !ok || i != len(port) {
		p, err = LookupPort(net, port)
		if err != nil {
			goto Error
		}
	}
	if p < 0 || p > 0xFFFF {
		err = &AddrError{"invalid port", port}
		goto Error
	}

	return addr, p, nil

Error:
	return nil, 0, err
}
<|endoftext|>


<commit_before>/*
 * This is the main core of the yaag package
 */
package yaag

import (
	"html/template"
	"io"
	"log"
	"os"
	"path/filepath"
)

const TEMPLATE = `<!DOCTYPE html>

    <title> API Documentation </title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

    <link href='http://fonts.googleapis.com/css?family=Roboto' rel='stylesheet' type='text/css'>
    <link rel="stylesheet" href="http://google-code-prettify.googlecode.com/svn/trunk/src/prettify.css">

    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap-theme.min.css">

            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse"
                    data-target="#bs-example-navbar-collapse-1">
                Toggle navigation

            </button>
            {{.Title}}

            <form class="navbar-form navbar-left" role="search">

                    <input type="text" class="form-control" placeholder="Search">

                <button type="submit" class="btn btn-default">Find</button>
            </form>

Base URL => {{.BaseLink}}

    {{ range $key, $value := .array }}

 {{$value.HttpVerb}}
        {{$value.Path}}

    {{ range $wrapperKey, $wrapperValue := $value.HtmlValues }}

 Example {{add $wrapperKey 1}}

            {{ if $wrapperValue.RequestHeader }}

 Request Headers

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.RequestHeader }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.PostForm }}

 Post Form

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.PostForm }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.RequestUrlParams }}

 URL Params

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.RequestUrlParams }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.RequestBody }}

 Request Body

{{ $wrapperValue.RequestBody }}

            {{ end }}

 Response Code

{{ $wrapperValue.ResponseCode }}

            {{ if $wrapperValue.ResponseHeader }}

 Response Headers

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.ResponseHeader }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.ResponseBody }}

 Response Body

{{ $wrapperValue.ResponseBody }}

            {{ end }}

    {{ end}}
    {{ end}}

    Developed by Gophers at betacraft Inc

`

var CommonHeaders = []string{
	"Accept",
	"Accept-Encoding",
	"Accept-Language",
	"Cache-Control",
	"Content-Length",
	"Content-Type",
	"Origin",
	"User-Agent",
	"X-Forwarded-For",
}

var count int

type APICall struct {
	Id int

	CurrentPath string
	MethodType  string

	PostForm map[string]string

	RequestHeader        map[string]string
	CommonRequestHeaders map[string]string
	ResponseHeader       map[string]string
	RequestUrlParams     map[string]string

	RequestBody  string
	ResponseBody string
	ResponseCode int
}

type PathSpec struct {
	HttpVerb   string
	Path       string
	HtmlValues []APICall
}

type ApiCallValue struct {
	BaseLink string
	Path     []PathSpec
}

type Config struct {
	On       bool
	DocTitle string
	DocPath  string
}

var ApiCallValueInstance = &ApiCallValue{}
var config *Config = &Config{On: false, DocPath: "apidoc.html", DocTitle: "YAAG"}

func IsOn() bool {
	return config.On
}

func Init(conf *Config) {
	config = conf
}

func add(x, y int) int {
	return x + y
}

func mult(x, y int) int {
	return (x + 1) * y
}

func GenerateHtml(htmlValue *APICall) {
	shouldAddPathSpec := true
	log.Printf("PathSpec : %v", ApiCallValueInstance.Path)
	for k, pathSpec := range ApiCallValueInstance.Path {
		if pathSpec.Path == htmlValue.CurrentPath && pathSpec.HttpVerb == htmlValue.MethodType {
			shouldAddPathSpec = false
			shouldAdd := true
			// for _, value := range pathSpec.HtmlValues {
			// 	if value.RequestBody == htmlValue.RequestBody {
			// 		shouldAdd = false
			// 	}
			// }
			if shouldAdd {
				htmlValue.Id = count
				count += 1
				deleteCommonHeaders(htmlValue)
				ApiCallValueInstance.Path[k].HtmlValues = append(pathSpec.HtmlValues, *htmlValue)
			}
		}
	}

	if shouldAddPathSpec {
		pathSpec := PathSpec{
			HttpVerb: htmlValue.MethodType,
			Path:     htmlValue.CurrentPath,
		}
		htmlValue.Id = count
		count += 1
		deleteCommonHeaders(htmlValue)
		pathSpec.HtmlValues = append(pathSpec.HtmlValues, *htmlValue)
		ApiCallValueInstance.Path = append(ApiCallValueInstance.Path, pathSpec)
	}
	funcs := template.FuncMap{"add": add, "mult": mult}
	t := template.New("API Documentation").Funcs(funcs)
	filePath, err := filepath.Abs(config.DocPath)
	htmlString := TEMPLATE

	t, err = t.Parse(htmlString)
	if err != nil {
		log.Println(err)
		return
	}
	homeHtmlFile, err := os.Create(filePath)
	defer homeHtmlFile.Close()

	if err != nil {
		log.Println(err)
		return
	}
	homeWriter := io.Writer(homeHtmlFile)
	t.Execute(homeWriter, map[string]interface{}{"array": ApiCallValueInstance.Path,
		"BaseLink": ApiCallValueInstance.BaseLink, "Title": config.DocTitle})
}

func deleteCommonHeaders(call *APICall) {
	delete(call.RequestHeader, "Accept")
	delete(call.RequestHeader, "Accept-Encoding")
	delete(call.RequestHeader, "Accept-Language")
	delete(call.RequestHeader, "Cache-Control")
	delete(call.RequestHeader, "Connection")
	delete(call.RequestHeader, "Cookie")
	delete(call.RequestHeader, "Origin")
	delete(call.RequestHeader, "User-Agent")
}
<commit_msg>Don't overwrite old doc file<commit_after>/*
 * This is the main core of the yaag package
 */
package yaag

import (
	"encoding/json"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

const TEMPLATE = `<!DOCTYPE html>

    <title> API Documentation </title>
    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css">

    <link href='http://fonts.googleapis.com/css?family=Roboto' rel='stylesheet' type='text/css'>
    <link rel="stylesheet" href="http://google-code-prettify.googlecode.com/svn/trunk/src/prettify.css">

    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap-theme.min.css">

            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse"
                    data-target="#bs-example-navbar-collapse-1">
                Toggle navigation

            </button>
            {{.Title}}

            <form class="navbar-form navbar-left" role="search">

                    <input type="text" class="form-control" placeholder="Search">

                <button type="submit" class="btn btn-default">Find</button>
            </form>

Base URL => {{.BaseLink}}

    {{ range $key, $value := .array }}

 {{$value.HttpVerb}}
        {{$value.Path}}

    {{ range $wrapperKey, $wrapperValue := $value.HtmlValues }}

 Example {{add $wrapperKey 1}}

            {{ if $wrapperValue.RequestHeader }}

 Request Headers

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.RequestHeader }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.PostForm }}

 Post Form

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.PostForm }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.RequestUrlParams }}

 URL Params

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.RequestUrlParams }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.RequestBody }}

 Request Body

{{ $wrapperValue.RequestBody }}

            {{ end }}

 Response Code

{{ $wrapperValue.ResponseCode }}

            {{ if $wrapperValue.ResponseHeader }}

 Response Headers

                    Key
                    Value

                {{ range $key, $value := $wrapperValue.ResponseHeader }}

                    {{ $key }}
                     {{ $value }}

                {{ end }}

            {{ end }}

            {{ if $wrapperValue.ResponseBody }}

 Response Body

{{ $wrapperValue.ResponseBody }}

            {{ end }}

    {{ end}}
    {{ end}}

    Developed by Gophers at betacraft Inc

`

var CommonHeaders = []string{
	"Accept",
	"Accept-Encoding",
	"Accept-Language",
	"Cache-Control",
	"Content-Length",
	"Content-Type",
	"Origin",
	"User-Agent",
	"X-Forwarded-For",
}

var count int

type APICall struct {
	Id int

	CurrentPath string
	MethodType  string

	PostForm map[string]string

	RequestHeader        map[string]string
	CommonRequestHeaders map[string]string
	ResponseHeader       map[string]string
	RequestUrlParams     map[string]string

	RequestBody  string
	ResponseBody string
	ResponseCode int
}

type PathSpec struct {
	HttpVerb   string
	Path       string
	HtmlValues []APICall
}

type ApiCallValue struct {
	BaseLink string
	Path     []PathSpec
}

type Config struct {
	On       bool
	DocTitle string
	DocPath  string
}

var ApiCallValueInstance = &ApiCallValue{}
var config *Config = &Config{On: false, DocPath: "apidoc.html", DocTitle: "YAAG"}

func IsOn() bool {
	return config.On
}

func Init(conf *Config) {
	filePath, err := filepath.Abs(conf.DocPath + ".json")
	dataFile, err := os.Open(filePath)
	defer dataFile.Close()

	if err == nil {
		json.NewDecoder(io.Reader(dataFile)).Decode(ApiCallValueInstance)
	}
	config = conf
}

func add(x, y int) int {
	return x + y
}

func mult(x, y int) int {
	return (x + 1) * y
}

func GenerateHtml(htmlValue *APICall) {
	shouldAddPathSpec := true
	log.Printf("PathSpec : %v", ApiCallValueInstance.Path)
	for k, pathSpec := range ApiCallValueInstance.Path {
		if pathSpec.Path == htmlValue.CurrentPath && pathSpec.HttpVerb == htmlValue.MethodType {
			shouldAddPathSpec = false
			shouldAdd := true
			// for _, value := range pathSpec.HtmlValues {
			// 	if value.RequestBody == htmlValue.RequestBody {
			// 		shouldAdd = false
			// 	}
			// }
			if shouldAdd {
				htmlValue.Id = count
				count += 1
				deleteCommonHeaders(htmlValue)
				ApiCallValueInstance.Path[k].HtmlValues = append(pathSpec.HtmlValues, *htmlValue)
			}
		}
	}

	if shouldAddPathSpec {
		pathSpec := PathSpec{
			HttpVerb: htmlValue.MethodType,
			Path:     htmlValue.CurrentPath,
		}
		htmlValue.Id = count
		count += 1
		deleteCommonHeaders(htmlValue)
		pathSpec.HtmlValues = append(pathSpec.HtmlValues, *htmlValue)
		ApiCallValueInstance.Path = append(ApiCallValueInstance.Path, pathSpec)
	}
	funcs := template.FuncMap{"add": add, "mult": mult}
	t := template.New("API Documentation").Funcs(funcs)
	filePath, err := filepath.Abs(config.DocPath)
	htmlString := TEMPLATE

	t, err = t.Parse(htmlString)
	if err != nil {
		log.Println(err)
		return
	}
	homeHtmlFile, err := os.Create(filePath)
	defer homeHtmlFile.Close()

	data, err := json.Marshal(ApiCallValueInstance)
	if err != nil {
		log.Println(err)
		return
	}

	if ioutil.WriteFile(filePath+".json", data, os.O_CREATE) != nil {
		log.Println(err)
		return
	}

	_, err := dataFile.Write(data)
	if err != nil {
		log.Println(err)
		return
	}
	homeWriter := io.Writer(homeHtmlFile)
	t.Execute(homeWriter, map[string]interface{}{"array": ApiCallValueInstance.Path,
		"BaseLink": ApiCallValueInstance.BaseLink, "Title": config.DocTitle})
}

func deleteCommonHeaders(call *APICall) {
	delete(call.RequestHeader, "Accept")
	delete(call.RequestHeader, "Accept-Encoding")
	delete(call.RequestHeader, "Accept-Language")
	delete(call.RequestHeader, "Cache-Control")
	delete(call.RequestHeader, "Connection")
	delete(call.RequestHeader, "Cookie")
	delete(call.RequestHeader, "Origin")
	delete(call.RequestHeader, "User-Agent")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"bufio";
	"gob";
	"http";
	"io";
	"log";
	"net";
	"os";
	"rpc";
	"strconv";
	"sync";
)

// Call represents an active RPC
type Call struct {
	ServiceMethod	string;	// The name of the service and method to call.
	Args	interface{};	// The argument to the function (*struct).
	Reply	interface{};	// The reply from the function (*struct).
	Error	os.Error;	// After completion, the error status.
	Done	chan *Call;	// Strobes when call is complete; value is the error status.
	seq	uint64;
}

// Client represents an RPC Client.
type Client struct {
	sync.Mutex;	// protects pending, seq
	shutdown	os.Error;	// non-nil if the client is shut down
	sending	sync.Mutex;
	seq	uint64;
	conn io.ReadWriteCloser;
	enc	*gob.Encoder;
	dec	*gob.Decoder;
	pending	map[uint64] *Call;
}

func (client *Client) send(c *Call) {
	// Register this call.
	client.Lock();
	if client.shutdown != nil {
		client.Unlock();
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return;
	}
	c.seq = client.seq;
	client.seq++;
	client.pending[c.seq] = c;
	client.Unlock();

	// Encode and send the request.
	request := new(Request);
	client.sending.Lock();
	request.Seq = c.seq;
	request.ServiceMethod = c.ServiceMethod;
	client.enc.Encode(request);
	err := client.enc.Encode(c.Args);
	if err != nil {
		panicln("rpc: client encode error:", err);
	}
	client.sending.Unlock();
}

func (client *Client) serve() {
	var err os.Error;
	for err == nil {
		response := new(Response);
		err = client.dec.Decode(response);
		if err != nil {
			break
		}
		seq := response.Seq;
		client.Lock();
		c := client.pending[seq];
		client.pending[seq] = c, false;
		client.Unlock();
		err = client.dec.Decode(c.Reply);
		c.Error = os.ErrorString(response.Error);
		// We don't want to block here.  It is the caller's responsibility to make
		// sure the channel has enough buffer space. See comment in Go().
		doNotBlock := c.Done <- c;
	}
	// Terminate pending calls.
	client.Lock();
	client.shutdown = err;
	for seq, call := range client.pending {
		call.Error = err;
		doNotBlock := call.Done <- call;
	}
	client.Unlock();
	log.Stderr("client protocol error:", err);
}

// NewClient returns a new Client to handle requests to the
// set of services at the other end of the connection.
func NewClient(conn io.ReadWriteCloser) *Client {
	client := new(Client);
	client.conn = conn;
	client.enc = gob.NewEncoder(conn);
	client.dec = gob.NewDecoder(conn);
	client.pending = make(map[uint64] *Call);
	go client.serve();
	return client;
}

// Dial connects to an HTTP RPC server at the specified network address.
func DialHTTP(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	io.WriteString(conn, "CONNECT " + rpcPath + " HTTP/1.0\n\n");

	// Require successful HTTP response
	// before switching to RPC protocol.
	resp, err := http.ReadResponse(bufio.NewReader(conn));
	if err == nil && resp.Status == connected {
		return NewClient(conn), nil;
	}
	if err == nil {
		err = os.ErrorString("unexpected HTTP response: " + resp.Status);
	}
	conn.Close();
	return nil, &net.OpError{"dial-http", network, address, err};
}

// Dial connects to an RPC server at the specified network address.
func Dial(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	return NewClient(conn), nil;
}

// Go invokes the function asynchronously.  It returns the Call structure representing
// the invocation.
func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call {
	c := new(Call);
	c.ServiceMethod = serviceMethod;
	c.Args = args;
	c.Reply = reply;
	if done == nil {
		done = make(chan *Call, 1);	// buffered.
	} else {
		// TODO(r): check cap > 0
		// If caller passes done != nil, it must arrange that
		// done has enough buffer for the number of simultaneous
		// RPCs that will be using that channel.
	}
	c.Done = done;
	if client.shutdown != nil {
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return c;
	}
	client.send(c);
	return c;
}

// Call invokes the named function, waits for it to complete, and returns its error status.
func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error {
	if client.shutdown != nil {
		return client.shutdown
	}
	call := <-client.Go(serviceMethod, args, reply, nil).Done;
	return call.Error;
}
<commit_msg>post-submit tweaks to previous cl<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"bufio";
	"gob";
	"http";
	"io";
	"log";
	"net";
	"os";
	"rpc";
	"strconv";
	"sync";
)

// Call represents an active RPC
type Call struct {
	ServiceMethod	string;	// The name of the service and method to call.
	Args	interface{};	// The argument to the function (*struct).
	Reply	interface{};	// The reply from the function (*struct).
	Error	os.Error;	// After completion, the error status.
	Done	chan *Call;	// Strobes when call is complete; value is the error status.
	seq	uint64;
}

// Client represents an RPC Client.
type Client struct {
	sync.Mutex;	// protects pending, seq
	shutdown	os.Error;	// non-nil if the client is shut down
	sending	sync.Mutex;
	seq	uint64;
	conn io.ReadWriteCloser;
	enc	*gob.Encoder;
	dec	*gob.Decoder;
	pending	map[uint64] *Call;
}

func (client *Client) send(c *Call) {
	// Register this call.
	client.Lock();
	if client.shutdown != nil {
		c.Error = client.shutdown;
		client.Unlock();
		doNotBlock := c.Done <- c;
		return;
	}
	c.seq = client.seq;
	client.seq++;
	client.pending[c.seq] = c;
	client.Unlock();

	// Encode and send the request.
	request := new(Request);
	client.sending.Lock();
	request.Seq = c.seq;
	request.ServiceMethod = c.ServiceMethod;
	client.enc.Encode(request);
	err := client.enc.Encode(c.Args);
	if err != nil {
		panicln("rpc: client encode error:", err);
	}
	client.sending.Unlock();
}

func (client *Client) serve() {
	var err os.Error;
	for err == nil {
		response := new(Response);
		err = client.dec.Decode(response);
		if err != nil {
			if err == os.EOF {
				err = io.ErrUnexpectedEOF;
			}
			break
		}
		seq := response.Seq;
		client.Lock();
		c := client.pending[seq];
		client.pending[seq] = c, false;
		client.Unlock();
		err = client.dec.Decode(c.Reply);
		c.Error = os.ErrorString(response.Error);
		// We don't want to block here.  It is the caller's responsibility to make
		// sure the channel has enough buffer space. See comment in Go().
		doNotBlock := c.Done <- c;
	}
	// Terminate pending calls.
	client.Lock();
	client.shutdown = err;
	for seq, call := range client.pending {
		call.Error = err;
		doNotBlock := call.Done <- call;
	}
	client.Unlock();
	log.Stderr("client protocol error:", err);
}

// NewClient returns a new Client to handle requests to the
// set of services at the other end of the connection.
func NewClient(conn io.ReadWriteCloser) *Client {
	client := new(Client);
	client.conn = conn;
	client.enc = gob.NewEncoder(conn);
	client.dec = gob.NewDecoder(conn);
	client.pending = make(map[uint64] *Call);
	go client.serve();
	return client;
}

// Dial connects to an HTTP RPC server at the specified network address.
func DialHTTP(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	io.WriteString(conn, "CONNECT " + rpcPath + " HTTP/1.0\n\n");

	// Require successful HTTP response
	// before switching to RPC protocol.
	resp, err := http.ReadResponse(bufio.NewReader(conn));
	if err == nil && resp.Status == connected {
		return NewClient(conn), nil;
	}
	if err == nil {
		err = os.ErrorString("unexpected HTTP response: " + resp.Status);
	}
	conn.Close();
	return nil, &net.OpError{"dial-http", network, address, err};
}

// Dial connects to an RPC server at the specified network address.
func Dial(network, address string) (*Client, os.Error) {
	conn, err := net.Dial(network, "", address);
	if err != nil {
		return nil, err
	}
	return NewClient(conn), nil;
}

// Go invokes the function asynchronously.  It returns the Call structure representing
// the invocation.
func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call {
	c := new(Call);
	c.ServiceMethod = serviceMethod;
	c.Args = args;
	c.Reply = reply;
	if done == nil {
		done = make(chan *Call, 1);	// buffered.
	} else {
		// TODO(r): check cap > 0
		// If caller passes done != nil, it must arrange that
		// done has enough buffer for the number of simultaneous
		// RPCs that will be using that channel.
	}
	c.Done = done;
	if client.shutdown != nil {
		c.Error = client.shutdown;
		doNotBlock := c.Done <- c;
		return c;
	}
	client.send(c);
	return c;
}

// Call invokes the named function, waits for it to complete, and returns its error status.
func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) os.Error {
	if client.shutdown != nil {
		return client.shutdown
	}
	call := <-client.Go(serviceMethod, args, reply, nil).Done;
	return call.Error;
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"

	host "github.com/libp2p/go-libp2p-host"
	mux "github.com/libp2p/go-stream-muxer"
	msmux "github.com/whyrusleeping/go-smux-multistream"
)

// MuxC is a stream multiplex transport constructor
type MuxC func(h host.Host) (mux.Transport, error)

// MsMuxC is a tuple containing a multiplex transport constructor and a protocol
// ID.
type MsMuxC struct {
	MuxC
	ID string
}

var muxArgTypes = newArgTypeSet(hostType, networkType, peerIDType, pstoreType)

// MuxerConstructor creates a multiplex constructor from the passed parameter
// using reflection.
func MuxerConstructor(m interface{}) (MuxC, error) {
	// Already constructed?
	if t, ok := m.(mux.Transport); ok {
		return func(_ host.Host) (mux.Transport, error) {
			return t, nil
		}, nil
	}

	ctor, err := makeConstructor(m, muxType, muxArgTypes)
	if err != nil {
		return nil, err
	}
	return func(h host.Host) (mux.Transport, error) {
		t, err := ctor(h, nil)
		if err != nil {
			return nil, err
		}
		return t.(mux.Transport), nil
	}, nil
}

func makeMuxer(h host.Host, tpts []MsMuxC) (mux.Transport, error) {
	muxMuxer := msmux.NewBlankTransport()
	transportSet := make(map[string]struct{}, len(tpts))
	for _, tptC := range tpts {
		if _, ok := transportSet[tptC.ID]; ok {
			return nil, fmt.Errorf("duplicate muxer transport: %s", tptC.ID)
		}
	}
	for _, tptC := range tpts {
		tpt, err := tptC.MuxC(h)
		if err != nil {
			return nil, err
		}
		muxMuxer.AddTransport(tptC.ID, tpt)
	}
	return muxMuxer, nil
}
<commit_msg>Ensure we filter duplicate transports in muxer<commit_after>package config

import (
	"fmt"

	host "github.com/libp2p/go-libp2p-host"
	mux "github.com/libp2p/go-stream-muxer"
	msmux "github.com/whyrusleeping/go-smux-multistream"
)

// MuxC is a stream multiplex transport constructor
type MuxC func(h host.Host) (mux.Transport, error)

// MsMuxC is a tuple containing a multiplex transport constructor and a protocol
// ID.
type MsMuxC struct {
	MuxC
	ID string
}

var muxArgTypes = newArgTypeSet(hostType, networkType, peerIDType, pstoreType)

// MuxerConstructor creates a multiplex constructor from the passed parameter
// using reflection.
func MuxerConstructor(m interface{}) (MuxC, error) {
	// Already constructed?
	if t, ok := m.(mux.Transport); ok {
		return func(_ host.Host) (mux.Transport, error) {
			return t, nil
		}, nil
	}

	ctor, err := makeConstructor(m, muxType, muxArgTypes)
	if err != nil {
		return nil, err
	}
	return func(h host.Host) (mux.Transport, error) {
		t, err := ctor(h, nil)
		if err != nil {
			return nil, err
		}
		return t.(mux.Transport), nil
	}, nil
}

func makeMuxer(h host.Host, tpts []MsMuxC) (mux.Transport, error) {
	muxMuxer := msmux.NewBlankTransport()
	transportSet := make(map[string]struct{}, len(tpts))
	for _, tptC := range tpts {
		if _, ok := transportSet[tptC.ID]; ok {
			return nil, fmt.Errorf("duplicate muxer transport: %s", tptC.ID)
		}
		transportSet[tptC.ID] = struct{}{}
	}
	for _, tptC := range tpts {
		tpt, err := tptC.MuxC(h)
		if err != nil {
			return nil, err
		}
		muxMuxer.AddTransport(tptC.ID, tpt)
	}
	return muxMuxer, nil
}
<|endoftext|>


<commit_before>package store

import (
	"database/sql"
	"reflect"
)

type SortBy struct {
	Column string
	Asc    bool
}

type Search struct {
	store  *Store
	i      interface{}
	Sort   []SortBy
	Filter Filter
}

func (s *Store) NewSearch(i interface{}) *Search {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if _, ok := s.types[typeName(i)]; !ok {
		return nil
	}
	return &Search{
		store: s,
		i:     i,
	}
}

type PreparedSearch struct {
	countStmt *sql.Stmt
	getStmt   *sql.Stmt
	vars      []interface{}
	store     *Store
}

func (s *Search) Prepare() (*PreparedSearch, error) {
	var (
		doneFirst    bool
		sql, sqlVars string
		vars         []interface{}
		name         = typeName(s.i)
	)
	if s.Filter != nil {
		sql += "WHERE " + s.Filter.SQL() + " "
		for _, i := range s.Filter.Vars() {
			if v := reflect.ValueOf(i); v.Kind() != reflect.Ptr {
				i = v.Addr().Interface()
			}
			if !isValidType(i) {
				return nil, ErrInvalidType
			}
			vars = append(vars, i)
		}
	}
	s.store.mutex.Lock()
	defer s.store.mutex.Unlock()
	count, err := s.store.db.Prepare("SELECT COUNT(1) FROM [" + name + "] " + sql)
	if err != nil {
		return nil, err
	}
	t := s.store.types[name]
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if doneFirst {
			sqlVars += ", "
		} else {
			doneFirst = true
		}
		sqlVars += "[" + f.name + "]"
	}
	if len(s.Sort) > 0 {
		sql += "SORT BY "
		for n, f := range s.Sort {
			if n > 0 {
				sql += ", "
			}
			sql += "[" + f.Column + "]"
			if f.Asc {
				sql += " ASC "
			} else {
				sql += " DESC "
			}
		}
	}
	get, err := s.store.db.Prepare("SELECT [" + t.fields[t.primary].name + "] FROM [" + name + "] " + sql + "LIMIT ? OFFSET ?;")
	if err != nil {
		return nil, err
	}
	return &PreparedSearch{
		count,
		get,
		vars,
		s.store,
	}, nil
}

func (p *PreparedSearch) Count() (int, error) {
	p.store.mutex.Lock()
	defer p.store.mutex.Unlock()
	row := p.countStmt.QueryRow(p.getVars()...)
	var count int
	err := row.Scan(&count)
	return count, err
}

func (p *PreparedSearch) GetPage(is []interface{}, offset int) (int, error) {
	if len(is) == 0 {
		return 0, nil
	}
	p.store.mutex.Lock()
	defer p.store.mutex.Unlock()
	rows, err := p.getStmt.Query(append(p.getVars(), len(is), offset)...)
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	return p.store.getPage(is, rows)
}

func (p *PreparedSearch) getVars() []interface{} {
	vars := make([]interface{}, len(p.vars), len(p.vars)+2)
	for n, v := range p.vars {
		vars[n] = reflect.ValueOf(v).Elem().Interface()
	}
	return vars
}
<commit_msg>Corrected SORT BY -> ORDER BY<commit_after>package store

import (
	"database/sql"
	"reflect"
)

type SortBy struct {
	Column string
	Asc    bool
}

type Search struct {
	store  *Store
	i      interface{}
	Sort   []SortBy
	Filter Filter
}

func (s *Store) NewSearch(i interface{}) *Search {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if _, ok := s.types[typeName(i)]; !ok {
		return nil
	}
	return &Search{
		store: s,
		i:     i,
	}
}

type PreparedSearch struct {
	countStmt *sql.Stmt
	getStmt   *sql.Stmt
	vars      []interface{}
	store     *Store
}

func (s *Search) Prepare() (*PreparedSearch, error) {
	var (
		doneFirst    bool
		sql, sqlVars string
		vars         []interface{}
		name         = typeName(s.i)
	)
	if s.Filter != nil {
		sql += "WHERE " + s.Filter.SQL() + " "
		for _, i := range s.Filter.Vars() {
			if v := reflect.ValueOf(i); v.Kind() != reflect.Ptr {
				i = v.Addr().Interface()
			}
			if !isValidType(i) {
				return nil, ErrInvalidType
			}
			vars = append(vars, i)
		}
	}
	s.store.mutex.Lock()
	defer s.store.mutex.Unlock()
	count, err := s.store.db.Prepare("SELECT COUNT(1) FROM [" + name + "] " + sql)
	if err != nil {
		return nil, err
	}
	t := s.store.types[name]
	for pos, f := range t.fields {
		if pos == t.primary {
			continue
		}
		if doneFirst {
			sqlVars += ", "
		} else {
			doneFirst = true
		}
		sqlVars += "[" + f.name + "]"
	}
	if len(s.Sort) > 0 {
		sql += "ORDER BY "
		for n, f := range s.Sort {
			if n > 0 {
				sql += ", "
			}
			sql += "[" + f.Column + "]"
			if f.Asc {
				sql += " ASC "
			} else {
				sql += " DESC "
			}
		}
	}
	get, err := s.store.db.Prepare("SELECT [" + t.fields[t.primary].name + "] FROM [" + name + "] " + sql + "LIMIT ? OFFSET ?;")
	if err != nil {
		return nil, err
	}
	return &PreparedSearch{
		count,
		get,
		vars,
		s.store,
	}, nil
}

func (p *PreparedSearch) Count() (int, error) {
	p.store.mutex.Lock()
	defer p.store.mutex.Unlock()
	row := p.countStmt.QueryRow(p.getVars()...)
	var count int
	err := row.Scan(&count)
	return count, err
}

func (p *PreparedSearch) GetPage(is []interface{}, offset int) (int, error) {
	if len(is) == 0 {
		return 0, nil
	}
	p.store.mutex.Lock()
	defer p.store.mutex.Unlock()
	rows, err := p.getStmt.Query(append(p.getVars(), len(is), offset)...)
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	return p.store.getPage(is, rows)
}

func (p *PreparedSearch) getVars() []interface{} {
	vars := make([]interface{}, len(p.vars), len(p.vars)+2)
	for n, v := range p.vars {
		vars[n] = reflect.ValueOf(v).Elem().Interface()
	}
	return vars
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"strings"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// SysctlBuilder set up our sysctls
type SysctlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &SysctlBuilder{}

func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
	var sysctls []string

	// Common settings
	{
		sysctls = append(sysctls,
			"# Kubernetes Settings",
			"")

		// A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users
		// See https://github.com/kubernetes/kops/issues/1340
		sysctls = append(sysctls, "vm.max_map_count = 262144",
			"")

		// See https://github.com/kubernetes/kubernetes/pull/38001
		sysctls = append(sysctls,
			"kernel.softlockup_panic = 1",
			"kernel.softlockup_all_cpu_backtrace = 1",
			"")

		// See https://github.com/kubernetes/kube-deploy/issues/261
		sysctls = append(sysctls,
			"# Increase the number of connections",
			"net.core.somaxconn = 32768",
			"",

			"# Maximum Socket Receive Buffer",
			"net.core.rmem_max = 16777216",
			"",

			"# Default Socket Send Buffer",
			"net.core.wmem_max = 16777216",
			"",

			"# Increase the maximum total buffer-space allocatable",
			"net.ipv4.tcp_wmem = 4096 12582912 16777216",
			"net.ipv4.tcp_rmem = 4096 12582912 16777216",
			"",

			"# Increase the number of outstanding syn requests allowed",
			"net.ipv4.tcp_max_syn_backlog = 8096",
			"",

			"# For persistent HTTP connections",
			"net.ipv4.tcp_slow_start_after_idle = 0",
			"",

			"# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks",
			"net.ipv4.tcp_tw_reuse = 1",
			"",

			// We can't change the local_port_range without changing the NodePort range
			//"# Allowed local port range",
			//"net.ipv4.ip_local_port_range = 10240 65535",
			//"",

			"# Max number of packets that can be queued on interface input",
			"# If kernel is receiving packets faster than can be processed",
			"# this queue increases",
			"net.core.netdev_max_backlog = 16384",
			"",

			"# Increase size of file handles and inode cache",
			"fs.file-max = 2097152",
			"",

			"# Increase size of conntrack table size to avoid poor iptables performance",
			"net.netfilter.nf_conntrack_max = 1000000",
			"",
		)
	}

	if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderAWS) {
		sysctls = append(sysctls,
			"# AWS settings",
			"",
			"# Issue #23395",
			"net.ipv4.neigh.default.gc_thresh1=0",
			"")
	}

	if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderGCE) {
		sysctls = append(sysctls,
			"# GCE settings",
			"",
			"net.ipv4.ip_forward=1",
			"")
	}

	t := &nodetasks.File{
		Path:            "/etc/sysctl.d/99-k8s-general.conf",
		Contents:        fi.NewStringResource(strings.Join(sysctls, "\n")),
		Type:            nodetasks.FileType_File,
		OnChangeExecute: [][]string{{"sysctl", "--system"}},
	}
	c.AddTask(t)

	return nil
}
<commit_msg>remove setting nf_conntrack_max in nodeup<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"strings"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
)

// SysctlBuilder set up our sysctls
type SysctlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &SysctlBuilder{}

func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error {
	var sysctls []string

	// Common settings
	{
		sysctls = append(sysctls,
			"# Kubernetes Settings",
			"")

		// A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users
		// See https://github.com/kubernetes/kops/issues/1340
		sysctls = append(sysctls, "vm.max_map_count = 262144",
			"")

		// See https://github.com/kubernetes/kubernetes/pull/38001
		sysctls = append(sysctls,
			"kernel.softlockup_panic = 1",
			"kernel.softlockup_all_cpu_backtrace = 1",
			"")

		// See https://github.com/kubernetes/kube-deploy/issues/261
		sysctls = append(sysctls,
			"# Increase the number of connections",
			"net.core.somaxconn = 32768",
			"",

			"# Maximum Socket Receive Buffer",
			"net.core.rmem_max = 16777216",
			"",

			"# Default Socket Send Buffer",
			"net.core.wmem_max = 16777216",
			"",

			"# Increase the maximum total buffer-space allocatable",
			"net.ipv4.tcp_wmem = 4096 12582912 16777216",
			"net.ipv4.tcp_rmem = 4096 12582912 16777216",
			"",

			"# Increase the number of outstanding syn requests allowed",
			"net.ipv4.tcp_max_syn_backlog = 8096",
			"",

			"# For persistent HTTP connections",
			"net.ipv4.tcp_slow_start_after_idle = 0",
			"",

			"# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks",
			"net.ipv4.tcp_tw_reuse = 1",
			"",

			// We can't change the local_port_range without changing the NodePort range
			//"# Allowed local port range",
			//"net.ipv4.ip_local_port_range = 10240 65535",
			//"",

			"# Max number of packets that can be queued on interface input",
			"# If kernel is receiving packets faster than can be processed",
			"# this queue increases",
			"net.core.netdev_max_backlog = 16384",
			"",

			"# Increase size of file handles and inode cache",
			"fs.file-max = 2097152",
			"",
		)
	}

	if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderAWS) {
		sysctls = append(sysctls,
			"# AWS settings",
			"",
			"# Issue #23395",
			"net.ipv4.neigh.default.gc_thresh1=0",
			"")
	}

	if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderGCE) {
		sysctls = append(sysctls,
			"# GCE settings",
			"",
			"net.ipv4.ip_forward=1",
			"")
	}

	t := &nodetasks.File{
		Path:            "/etc/sysctl.d/99-k8s-general.conf",
		Contents:        fi.NewStringResource(strings.Join(sysctls, "\n")),
		Type:            nodetasks.FileType_File,
		OnChangeExecute: [][]string{{"sysctl", "--system"}},
	}
	c.AddTask(t)

	return nil
}
<|endoftext|>