<commit_before>// Package speedtest interfaces for testing internet bandwidth through HTTP by speedtest.net
package speedtest

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"
)

type byDistance []Server
type ST struct {
	cfg     Settings
	servers []Server
}
type Client struct {
	IP  string  `xml:"ip,attr"`
	Lat float64 `xml:"lat,attr"`
	Lon float64 `xml:"lon,attr"`
	ISP string  `xml:"isp,attr"`
}
type Server struct {
	Name     string  `xml:"name,attr"`
	Id       string  `xml:"id,attr"`
	Sponsor  string  `xml:"sponsor,attr"`
	Country  string  `xml:"country,attr"`
	URL      string  `xml:"url,attr"`
	URL2     string  `xml:"url2,attr"`
	Lat      float64 `xml:"lat,attr"`
	Lon      float64 `xml:"lon,attr"`
	Distance float64
}

type Hosts struct {
	Server []Server `xml:"servers>server"`
}

type Settings struct {
	Download struct {
		TestLength    float64 `xml:"testlength,attr"`
		ThreadsPerURL int     `xml:"threadsperurl,attr"`
	} `xml:"download"`
	Upload struct {
		Ratio         int     `xml:"ratio,attr"`
		MaxChunkCount int     `xml:"maxchunkcount,attr"`
		Threads       int     `xml:"threads,attr"`
		TestLength    float64 `xml:"testlength,attr"`
	} `xml:"upload"`
	ServerCfg struct {
		IgnoreIds string `xml:"ignoreids,attr"`
	} `xml:"server-config"`
	Client struct {
		Client
	} `xml:"client"`
}

func Run() error {
	st := ST{}
	fmt.Printf("Powered by Ookla — http://www.speedtest.net/terms\n")
	fmt.Printf("Downloading the speedtest.net configuration")
	if err := st.getConfig(); err != nil {
		return err
	}
	fmt.Printf(" \u2713\nYour IP: %s, Service Provider: %s\n", st.cfg.Client.IP, st.cfg.Client.ISP)
	fmt.Printf("Retrieving speedtest.net servers list")
	if err := st.getServers(); err != nil {
		return err
	}
	fmt.Printf(" \u2713\nSelecting best server based on Geo and Latency")
	server, latency := st.bestServer()
	if server.Distance == 0 {
		return fmt.Errorf("could not find a server")
	}
	fmt.Printf(" \u2713\nHosted by %s (%s) %.2f ms %.0f miles\n",
		server.Sponsor,
		server.Name,
		latency*1000,
		server.Distance)
	down := st.download(server)
	fmt.Printf("Download: %.2f Mbps\n", down)
	return nil
}

func getData(url string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return []byte{}, err
	}

	req.Header.Set("User-Agent", "Mozilla/5.0")
	req.Header.Set("Cache-Control", "no-cache")

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}
	if resp.StatusCode != 200 {
		return []byte{}, fmt.Errorf("can not connect")
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return b, err
	}
	return b, nil
}

func (st *ST) getConfig() error {
	b, err := getData("http://www.speedtest.net/speedtest-config.php")
	if err != nil {
		return err
	}
	xml.Unmarshal(b, &st.cfg)
	return nil
}

func (st *ST) getServers() error {
	var (
		isIgnoreId = make(map[string]struct{})
		stServers  = []string{
			"http://www.speedtest.net/speedtest-servers-static.php",
			"http://c.speedtest.net/speedtest-servers-static.php",
		}
		hosts Hosts
	)

	for _, server := range stServers {
		b, err := getData(server)
		if err != nil {
			continue
		}

		xml.Unmarshal(b, &hosts)

		st.cfg.Client.Lon = st.cfg.Client.Lon * math.Pi / 180
		st.cfg.Client.Lat = st.cfg.Client.Lat * math.Pi / 180

		for _, ignoreId := range strings.Split(st.cfg.ServerCfg.IgnoreIds, ",") {
			isIgnoreId[ignoreId] = struct{}{}
		}

		for i, server := range hosts.Server {
			if _, ok := isIgnoreId[hosts.Server[i].Id]; !ok {
				hosts.Server[i].Distance = distance(st.cfg.Client.Lon, st.cfg.Client.Lat, server)
				st.servers = append(st.servers, hosts.Server[i])
			}
		}

		sort.Sort(byDistance(st.servers))
		break
	}
	return nil
}

func (st *ST) bestServer() (Server, float64) {
	var (
		latency float64
		sum     float64
		server  Server
	)
	latency = 1000
	for i := range []int{1, 2, 3, 4} {
		base := st.servers[i].URL[:strings.LastIndex(st.servers[i].URL, "/")]
		url := base + "/latency.txt"
		sum = 0
		for range []int{1, 2} {
			ts := time.Now()
			_, err := getData(url)
			if err != nil {
				sum = 100000.0
				break
			}
			elapsed := time.Since(ts)
			sum += elapsed.Seconds()
		}
		if sum/2 < latency {
			server = st.servers[i]
			latency = sum / 2
		}
	}
	return server, latency
}

func (st *ST) download(server Server) float64 {
	var (
		wg        sync.WaitGroup
		urls      []string
		totalRcvd float64
		sizeDld   = []int{350, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000}
	)

	base := server.URL[:strings.LastIndex(server.URL, "/")]

	for _, size := range sizeDld {
		for i := 0; i < st.cfg.Download.ThreadsPerURL; i++ {
			urls = append(urls, fmt.Sprintf("%s/random%dx%d.jpg", base, size, size))
		}
	}
	ts := time.Now()
	fmt.Printf("Testing download ")
	for _, url := range urls {
		wg.Add(1)
		go func(url string) {
			var (
				buf   = make([]byte, 10240)
				total int
			)

			defer wg.Done()
			timeout := time.Duration(st.cfg.Download.TestLength) * time.Second
			client := http.Client{
				Timeout: timeout,
			}
			resp, _ := client.Get(url)
			ts := time.Now()
			for {
				lr := io.LimitReader(resp.Body, 10240)
				n, err := io.ReadFull(lr, buf)
				total += n
				if n == 0 || err != nil {
					break
				}
				if st.cfg.Download.TestLength < time.Since(ts).Seconds() {
					break
				}
			}
			totalRcvd += float64(total)
			fmt.Printf(".")
		}(url)
	}
	wg.Wait()
	fmt.Printf("\n")
	return totalRcvd * 8 / time.Since(ts).Seconds() / 1000 / 1000
}

func (st *ST) upload(server Server) float64 {
	var (
		sizes       []int
		totalUpload int
		sizeUpl     = []int{32768, 65536, 131072, 262144, 524288, 1048576, 7340032}
		count       = st.cfg.Upload.MaxChunkCount * 2 / len(sizeUpl[st.cfg.Upload.Ratio-1:])
		data        = make(map[int]string, len(sizeUpl))
		jobCh       = make(chan int, len(sizeUpl)*count)
		resCh       = make(chan int, len(sizeUpl)*count)
		done        = make(chan struct{})
		base        = server.URL[:strings.LastIndex(server.URL, "/")]
	)

	for _, size := range sizeUpl[st.cfg.Upload.Ratio-1:] {
		token := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
		data[size] = strings.Repeat(token, size/36)
		for i := 0; i < count; i++ {
			sizes = append(sizes, size)
		}
	}
	for t := 0; t < st.cfg.Upload.MaxChunkCount; t++ {
		go func() {
		LOOP:
			for {
				size := <-jobCh
				testLength := int(st.cfg.Upload.TestLength)
				chunkNums := len(data[size]) / testLength
				ts := time.Now()
				c := 0
				for time.Since(ts).Seconds() < st.cfg.Upload.TestLength {
					buf := strings.NewReader(data[size][testLength*c : testLength*(c+1)])
					resp, err := http.Post(base+"/upload.php", "text/plain", buf)
					if err != nil || resp.StatusCode != 200 {
						continue
					}
					resCh <- testLength
					chunkNums--
					if chunkNums > 0 {
						c++
					} else {
						break LOOP
					}
					select {
					case <-done:
						break LOOP
					default:
						continue
					}
				}

			}
		}()
	}

	for _, size := range sizes {
		jobCh <- size
	}

	go func() {
		for {
			chunk, ok := <-resCh
			if !ok {
				break
			}
			totalUpload += chunk
		}
	}()

	<-time.After(time.Duration(st.cfg.Upload.TestLength) * time.Second)
	close(done)
	close(resCh)

	return float64(totalUpload) * 8 / st.cfg.Upload.TestLength / 1000 / 1000
}

func findExt() {
}

func distance(cLon, cLat float64, server Server) float64 {
	server.Lon = server.Lon * math.Pi / 180
	server.Lat = server.Lat * math.Pi / 180

	deltaLon := server.Lon - cLon
	deltaLat := server.Lat - cLat

	hsinLat := math.Pow(math.Sin(deltaLat/2), 2)
	hsinLon := math.Pow(math.Sin(deltaLon/2), 2)

	a := hsinLat + math.Cos(server.Lat)*math.Cos(cLat)*hsinLon
	c := 2 * 3961 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))

	return c
}

func (a byDistance) Len() int           { return len(a) }
func (a byDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }
<commit_msg>speedtest upload func<commit_after>// Package speedtest interfaces for testing internet bandwidth through HTTP by speedtest.net
package speedtest

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"
)

type byDistance []Server
type ST struct {
	cfg     Settings
	servers []Server
}
type Client struct {
	IP  string  `xml:"ip,attr"`
	Lat float64 `xml:"lat,attr"`
	Lon float64 `xml:"lon,attr"`
	ISP string  `xml:"isp,attr"`
}
type Server struct {
	Name     string  `xml:"name,attr"`
	Id       string  `xml:"id,attr"`
	Sponsor  string  `xml:"sponsor,attr"`
	Country  string  `xml:"country,attr"`
	URL      string  `xml:"url,attr"`
	URL2     string  `xml:"url2,attr"`
	Lat      float64 `xml:"lat,attr"`
	Lon      float64 `xml:"lon,attr"`
	Distance float64
}

type Hosts struct {
	Server []Server `xml:"servers>server"`
}

type Settings struct {
	Download struct {
		TestLength    float64 `xml:"testlength,attr"`
		ThreadsPerURL int     `xml:"threadsperurl,attr"`
	} `xml:"download"`
	Upload struct {
		Ratio         int     `xml:"ratio,attr"`
		MaxChunkCount int     `xml:"maxchunkcount,attr"`
		Threads       int     `xml:"threads,attr"`
		TestLength    float64 `xml:"testlength,attr"`
	} `xml:"upload"`
	ServerCfg struct {
		IgnoreIds string `xml:"ignoreids,attr"`
	} `xml:"server-config"`
	Client struct {
		Client
	} `xml:"client"`
}

func Run() error {
	st := ST{}
	fmt.Printf("Powered by Ookla — http://www.speedtest.net/terms\n")
	fmt.Printf("Downloading the speedtest.net configuration")
	if err := st.getConfig(); err != nil {
		return err
	}
	fmt.Printf(" \u2713\nYour IP: %s, Service Provider: %s\n", st.cfg.Client.IP, st.cfg.Client.ISP)
	fmt.Printf("Retrieving speedtest.net servers list")
	if err := st.getServers(); err != nil {
		return err
	}
	fmt.Printf(" \u2713\nSelecting best server based on Geo and Latency")
	server, latency := st.bestServer()
	if server.Distance == 0 {
		return fmt.Errorf("could not find a server")
	}
	fmt.Printf(" \u2713\nHosted by %s (%s) %.2f ms, %.0f miles\n",
		server.Sponsor,
		server.Name,
		latency*1000,
		server.Distance)
	down := st.download(server)
	fmt.Printf("Download: %s\n", down)
	up := st.upload(server)
	fmt.Printf("Upload: %s\n", up)
	return nil
}

func getData(url string) ([]byte, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return []byte{}, err
	}

	req.Header.Set("User-Agent", "Mozilla/5.0")
	req.Header.Set("Cache-Control", "no-cache")

	resp, err := client.Do(req)
	if err != nil {
		return []byte{}, err
	}
	if resp.StatusCode != 200 {
		return []byte{}, fmt.Errorf("can not connect")
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return b, err
	}
	return b, nil
}

func (st *ST) getConfig() error {
	b, err := getData("http://www.speedtest.net/speedtest-config.php")
	if err != nil {
		return err
	}
	xml.Unmarshal(b, &st.cfg)
	return nil
}

func (st *ST) getServers() error {
	var (
		isIgnoreId = make(map[string]struct{})
		stServers  = []string{
			"http://www.speedtest.net/speedtest-servers-static.php",
			"http://c.speedtest.net/speedtest-servers-static.php",
		}
		hosts Hosts
	)

	for _, server := range stServers {
		b, err := getData(server)
		if err != nil {
			continue
		}

		xml.Unmarshal(b, &hosts)

		st.cfg.Client.Lon = st.cfg.Client.Lon * math.Pi / 180
		st.cfg.Client.Lat = st.cfg.Client.Lat * math.Pi / 180

		for _, ignoreId := range strings.Split(st.cfg.ServerCfg.IgnoreIds, ",") {
			isIgnoreId[ignoreId] = struct{}{}
		}

		for i, server := range hosts.Server {
			if _, ok := isIgnoreId[hosts.Server[i].Id]; !ok {
				hosts.Server[i].Distance = distance(st.cfg.Client.Lon, st.cfg.Client.Lat, server)
				st.servers = append(st.servers, hosts.Server[i])
			}
		}

		sort.Sort(byDistance(st.servers))
		break
	}
	return nil
}

func (st *ST) bestServer() (Server, float64) {
	var (
		latency float64
		sum     float64
		server  Server
	)
	latency = 1000
	for i := range []int{1, 2, 3, 4} {
		base := st.servers[i].URL[:strings.LastIndex(st.servers[i].URL, "/")]
		url := base + "/latency.txt"
		sum = 0
		for range []int{1, 2} {
			ts := time.Now()
			_, err := getData(url)
			if err != nil {
				sum = 100000.0
				break
			}
			elapsed := time.Since(ts)
			sum += elapsed.Seconds()
		}
		if sum/2 < latency {
			server = st.servers[i]
			latency = sum / 2
		}
	}
	return server, latency
}

func (st *ST) download(server Server) string {
	var (
		wg        sync.WaitGroup
		urls      []string
		totalRcvd float64
		sizeDld   = []int{350, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000}
	)

	base := server.URL[:strings.LastIndex(server.URL, "/")]

	for _, size := range sizeDld {
		for i := 0; i < st.cfg.Download.ThreadsPerURL; i++ {
			urls = append(urls, fmt.Sprintf("%s/random%dx%d.jpg", base, size, size))
		}
	}
	ts := time.Now()
	fmt.Printf("Testing download ")
	for _, url := range urls {
		wg.Add(1)
		go func(url string) {
			var (
				buf   = make([]byte, 10240)
				total int
			)

			defer wg.Done()
			timeout := time.Duration(st.cfg.Download.TestLength) * time.Second
			client := http.Client{
				Timeout: timeout,
			}
			resp, _ := client.Get(url)
			ts := time.Now()
			for {
				lr := io.LimitReader(resp.Body, 10240)
				n, err := io.ReadFull(lr, buf)
				total += n
				if n == 0 || err != nil {
					break
				}
				if st.cfg.Download.TestLength < time.Since(ts).Seconds() {
					break
				}
			}
			totalRcvd += float64(total)
			fmt.Printf(".")
		}(url)
	}
	wg.Wait()
	fmt.Printf("\n")
	return fmtUnit(totalRcvd / time.Since(ts).Seconds())
}

func (st *ST) upload(server Server) string {
	var (
		sizes       []int
		totalUpload int
		sizeUpl     = []int{32768, 65536, 131072, 262144, 524288, 1048576, 7340032}
		count       = st.cfg.Upload.MaxChunkCount * 2 / len(sizeUpl[st.cfg.Upload.Ratio-1:])
		jobCh       = make(chan int, len(sizeUpl)*count)
		resCh       = make(chan int, len(sizeUpl)*count)
		done        = make(chan struct{})
		base        = server.URL[:strings.LastIndex(server.URL, "/")]
	)

	for _, size := range sizeUpl[st.cfg.Upload.Ratio-1:] {
		for i := 0; i < count; i++ {
			sizes = append(sizes, size)
		}
	}
	// TODO: needs optimization
	fmt.Printf("Testing upload ")
	for t := 0; t < st.cfg.Upload.MaxChunkCount; t++ {
		go func() {
			defer func() {
				recover()
			}()
			tr := &http.Transport{
				DisableKeepAlives:  false,
				DisableCompression: true,
			}
			client := &http.Client{
				Transport: tr,
			}
			req, err := http.NewRequest("POST", base+"/upload.php", nil)
			if err != nil {
				return
			}
			req.Header.Set("User-Agent", "Mozilla/5.0")
			testLength := 1024 * 16
			chunkData := strings.Repeat("0123456789ABCDEFGHIJKLMNOPQRSTUV", testLength/32)
		DONE:
			for {
				size := <-jobCh
				chunkNums := size / testLength
				ts := time.Now()
				c := 0
			LOOP:
				for time.Since(ts).Seconds() < st.cfg.Upload.TestLength {
					buf := strings.NewReader(chunkData)
					req.Body = ioutil.NopCloser(buf)
					resp, err := client.Do(req)
					if err != nil || resp.StatusCode != 200 {
						println(err.Error())
						continue
					}

					io.Copy(ioutil.Discard, resp.Body)
					resp.Body.Close()

					resCh <- testLength
					chunkNums--
					if chunkNums > 0 {
						c++
					} else {
						break LOOP
					}
					select {
					case <-done:
						break DONE
					default:
						continue
					}
				}
				fmt.Printf(".")
			}
		}()
	}

	ts := time.Now()
	for _, size := range sizes {
		jobCh <- size
	}

	go func() {
		for {
			chunk, ok := <-resCh
			if !ok {
				break
			}
			totalUpload += chunk
		}
	}()

	<-time.After(time.Duration(st.cfg.Upload.TestLength) * time.Second)
	close(done)
	close(resCh)

	fmt.Printf("\n")
	return fmtUnit(float64(totalUpload) / time.Since(ts).Seconds())
}

func findExt() {
}

func fmtUnit(n float64) string {
	n = n * 8
	if n > 1e9 {
		return fmt.Sprintf("%.2f Gbps", n/1e9)
	} else if n > 1e6 {
		return fmt.Sprintf("%.2f Mbps", n/1e6)
	}
	return fmt.Sprintf("%.2f Kbps", n/1e3)
}

func distance(cLon, cLat float64, server Server) float64 {
	server.Lon = server.Lon * math.Pi / 180
	server.Lat = server.Lat * math.Pi / 180

	deltaLon := server.Lon - cLon
	deltaLat := server.Lat - cLat

	hsinLat := math.Pow(math.Sin(deltaLat/2), 2)
	hsinLon := math.Pow(math.Sin(deltaLon/2), 2)

	a := hsinLat + math.Cos(server.Lat)*math.Cos(cLat)*hsinLon
	c := 2 * 3961 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))

	return c
}

func (a byDistance) Len() int           { return len(a) }
func (a byDistance) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byDistance) Less(i, j int) bool { return a[i].Distance < a[j].Distance }
<|endoftext|>


<commit_before>package ttml

import (
	"os"
	"fmt"
	"encoding/xml"
)

func WriteTtml (v *WTt, outpath string) {
	f, err := os.Create(outpath)
	if err != nil {
		fmt.Println(err)

	}
	out, err := xml.MarshalIndent(v, "", "  ")
	if err != nil {
		panic(err)
	}
	bytehead := []byte(xml.Header)
	out = append(bytehead, out ...)
	f, arr := os.Create(outpath)
	if arr != nil {
		panic(arr)
	}
	defer f.Close()
	trr := ioutil.WriteFile(outpath, []byte(out), 0666)
	if trr != nil {
		panic(trr)
	}
}
<commit_msg>updating ttml<commit_after>package ttml

import (
	"os"
	"fmt"
	"encoding/xml"
	"io/ioutil"
)

func WriteTtml (v *WTt, outpath string) {
	f, err := os.Create(outpath)
	if err != nil {
		fmt.Println(err)

	}
	out, err := xml.MarshalIndent(v, "", "  ")
	if err != nil {
		panic(err)
	}
	bytehead := []byte(xml.Header)
	out = append(bytehead, out ...)
	f, arr := os.Create(outpath)
	if arr != nil {
		panic(arr)
	}
	defer f.Close()
	trr := ioutil.WriteFile(outpath, []byte(out), 0666)
	if trr != nil {
		panic(trr)
	}
}
<|endoftext|>


<commit_before>package restapi

import (
	"net/http"
	"sync"
	"sync/atomic"

	"github.com/go-openapi/errors"
	httpkit "github.com/go-openapi/runtime"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/swag"

	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/models"
	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/restapi/operations"
	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/restapi/operations/todos"
)

// This file is safe to edit. Once it exists it will not be overwritten

func configureFlags(api *operations.TodoListAPI) {

}

var items = make(map[int64]*models.Item)
var lastID int64

var itemsLock = &sync.Mutex{}

func newItemID() int64 {
	return atomic.AddInt64(&lastID, 1)
}

func addItem(item *models.Item) error {
	if item == nil {
		return errors.New(500, "item must be present")
	}

	itemsLock.Lock()
	defer itemsLock.Unlock()

	newID := newItemID()
	item.ID = newID
	items[newID] = item

	return nil
}

func updateItem(id int64, item *models.Item) error {
	if item == nil {
		return errors.New(500, "item must be present")
	}

	itemsLock.Lock()
	defer itemsLock.Unlock()

	_, exists := items[id]
	if !exists {
		return errors.NotFound("not found: item %d", id)
	}

	item.ID = id
	items[id] = item
	return nil
}

func deleteItem(id int64) error {
	itemsLock.Lock()
	defer itemsLock.Unlock()

	_, exists := items[id]
	if !exists {
		return errors.NotFound("not found: item %d", id)
	}

	delete(items, id)
	return nil
}

func allItems(since int64, limit int32) (result []*models.Item) {
	result = make([]*models.Item, 0)
	for id, item := range items {
		if len(result) >= int(limit) {
			return
		}
		if since == 0 || id > since {
			result = append(result, item)
		}
	}
	return
}

func configureAPI(api *operations.TodoListAPI) http.Handler {
	// configure the api here
	api.ServeError = errors.ServeError

	api.JSONConsumer = httpkit.JSONConsumer()

	api.JSONProducer = httpkit.JSONProducer()

	api.TodosAddOneHandler = todos.AddOneHandlerFunc(func(params todos.AddOneParams) middleware.Responder {
		if err := addItem(params.Body); err != nil {
			return todos.NewAddOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewAddOneCreated().WithPayload(params.Body)
	})

	api.TodosDestroyOneHandler = todos.DestroyOneHandlerFunc(func(params todos.DestroyOneParams) middleware.Responder {
		if err := deleteItem(params.ID); err != nil {
			return todos.NewDestroyOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewDestroyOneNoContent()
	})

	api.TodosFindTodosHandler = todos.FindTodosHandlerFunc(func(params todos.FindTodosParams) middleware.Responder {
		mergedParams := todos.NewFindTodosParams()
		mergedParams.Since = swag.Int64(0)
		if params.Since != nil {
			mergedParams.Since = params.Since
		}
		if params.Limit != nil {
			mergedParams.Limit = params.Limit
		}
		return todos.NewFindTodosOK().WithPayload(allItems(*mergedParams.Since, *mergedParams.Limit))
	})

	api.TodosUpdateOneHandler = todos.UpdateOneHandlerFunc(func(params todos.UpdateOneParams) middleware.Responder {
		if err := updateItem(params.ID, params.Body); err != nil {
			return todos.NewUpdateOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewUpdateOneOK().WithPayload(params.Body)
	})

	api.ServerShutdown = func() {}

	return setupGlobalMiddleware(api.Serve(setupMiddlewares))
}

// The middleware configuration is for the handler executors. These do not apply to the swagger.json document.
// The middleware executes after routing but before authentication, binding and validation
func setupMiddlewares(handler http.Handler) http.Handler {
	return handler
}

// The middleware configuration happens before anything, this middleware also applies to serving the swagger.json document.
// So this is a good place to plug in a panic handling middleware, logging and metrics
func setupGlobalMiddleware(handler http.Handler) http.Handler {
	return handler
}
<commit_msg>Provide example for configure command-line flags<commit_after>package restapi

import (
	"net/http"
	"sync"
	"sync/atomic"

	"github.com/go-openapi/errors"
	httpkit "github.com/go-openapi/runtime"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/swag"

	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/models"
	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/restapi/operations"
	"github.com/go-swagger/go-swagger/examples/tutorials/todo-list/server-complete/restapi/operations/todos"
)

// This file is safe to edit. Once it exists it will not be overwritten

var exampleFlags = struct {
	Example1 string `long:"example1" description:"Sample for showing how to configure cmd-line flags"`
	Example2 string `long:"example2" description:"Further info at https://github.com/jessevdk/go-flags"`
}{}

func configureFlags(api *operations.TodoListAPI) {
	api.CommandLineOptionsGroups = []swag.CommandLineOptionsGroup{
		swag.CommandLineOptionsGroup{
			ShortDescription: "Example Flags",
			LongDescription:  "",
			Options:          &exampleFlags,
		},
	}
}

var items = make(map[int64]*models.Item)
var lastID int64

var itemsLock = &sync.Mutex{}

func newItemID() int64 {
	return atomic.AddInt64(&lastID, 1)
}

func addItem(item *models.Item) error {
	if item == nil {
		return errors.New(500, "item must be present")
	}

	itemsLock.Lock()
	defer itemsLock.Unlock()

	newID := newItemID()
	item.ID = newID
	items[newID] = item

	return nil
}

func updateItem(id int64, item *models.Item) error {
	if item == nil {
		return errors.New(500, "item must be present")
	}

	itemsLock.Lock()
	defer itemsLock.Unlock()

	_, exists := items[id]
	if !exists {
		return errors.NotFound("not found: item %d", id)
	}

	item.ID = id
	items[id] = item
	return nil
}

func deleteItem(id int64) error {
	itemsLock.Lock()
	defer itemsLock.Unlock()

	_, exists := items[id]
	if !exists {
		return errors.NotFound("not found: item %d", id)
	}

	delete(items, id)
	return nil
}

func allItems(since int64, limit int32) (result []*models.Item) {
	result = make([]*models.Item, 0)
	for id, item := range items {
		if len(result) >= int(limit) {
			return
		}
		if since == 0 || id > since {
			result = append(result, item)
		}
	}
	return
}

func configureAPI(api *operations.TodoListAPI) http.Handler {
	// configure the api here
	api.ServeError = errors.ServeError

	api.JSONConsumer = httpkit.JSONConsumer()

	api.JSONProducer = httpkit.JSONProducer()

	api.TodosAddOneHandler = todos.AddOneHandlerFunc(func(params todos.AddOneParams) middleware.Responder {
		if err := addItem(params.Body); err != nil {
			return todos.NewAddOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewAddOneCreated().WithPayload(params.Body)
	})

	api.TodosDestroyOneHandler = todos.DestroyOneHandlerFunc(func(params todos.DestroyOneParams) middleware.Responder {
		if err := deleteItem(params.ID); err != nil {
			return todos.NewDestroyOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewDestroyOneNoContent()
	})

	api.TodosFindTodosHandler = todos.FindTodosHandlerFunc(func(params todos.FindTodosParams) middleware.Responder {
		mergedParams := todos.NewFindTodosParams()
		mergedParams.Since = swag.Int64(0)
		if params.Since != nil {
			mergedParams.Since = params.Since
		}
		if params.Limit != nil {
			mergedParams.Limit = params.Limit
		}
		return todos.NewFindTodosOK().WithPayload(allItems(*mergedParams.Since, *mergedParams.Limit))
	})

	api.TodosUpdateOneHandler = todos.UpdateOneHandlerFunc(func(params todos.UpdateOneParams) middleware.Responder {
		if err := updateItem(params.ID, params.Body); err != nil {
			return todos.NewUpdateOneDefault(500).WithPayload(&models.Error{Code: 500, Message: swag.String(err.Error())})
		}
		return todos.NewUpdateOneOK().WithPayload(params.Body)
	})

	api.ServerShutdown = func() {}

	return setupGlobalMiddleware(api.Serve(setupMiddlewares))
}

// The middleware configuration is for the handler executors. These do not apply to the swagger.json document.
// The middleware executes after routing but before authentication, binding and validation
func setupMiddlewares(handler http.Handler) http.Handler {
	return handler
}

// The middleware configuration happens before anything, this middleware also applies to serving the swagger.json document.
// So this is a good place to plug in a panic handling middleware, logging and metrics
func setupGlobalMiddleware(handler http.Handler) http.Handler {
	return handler
}
<|endoftext|>


<commit_before>package prompt

import (
	"fmt"
	"math"

	"github.com/julienroland/copro"
	"github.com/julienroland/usg"
)

type Checkbox struct {
	Question      string
	Choices       []*Choice
	CheckedChoice []*Choice
	app           *copro.App
}

type CheckboxResult struct {
	ID    []int
	Label []string
}

func NewCheckbox() *Checkbox {
	CheckboxList := new(Checkbox)
	app := copro.NewApp()
	CheckboxList.app = app
	CheckboxList.app.Run()
	return CheckboxList
}

func (s *Checkbox) Run() (CheckboxResult, error) {
	s.app.EntryCount = len(getChoicesWithoutSeparator(s.Choices)) - 1
	s.setChoicePointer()
	s.manageDefaultPointer()
	s.app.Renderer(func() {
		s.RenderHeader()
		s.RenderChoices(s.app.Pointer)
	})

	ids := []int{}
	labels := []string{}
	for _, pointer := range s.app.SavedPointers {
		for _, choice := range s.Choices {
			if choice.pointer == pointer && !choice.IsSeparator {
				ids = append(ids, choice.ID)
				labels = append(labels, choice.Label)
			}
		}
	}
	result := CheckboxResult{ID: ids, Label: labels}
	return result, nil
}

func (s *Checkbox) manageDefaultPointer() {
	pointers := []int{}
	for _, choice := range s.Choices {
		if choice.Selected && !choice.IsSeparator {
			pointers = append(pointers, choice.pointer)
		}
	}
	s.app.SavedPointers = pointers

}

func (s *Checkbox) setChoicePointer() error {
	position := 0
	for _, choice := range s.Choices {
		if choice.IsSeparator {
			continue
		}
		choice.pointer = position
		position++
	}
	return nil
}

func (s *Checkbox) RenderHeader() {
	if len(s.Question) <= 0 {
		fmt.Errorf("You need to ask something")
	}

	copro.DisplayGreen(s.Question)
	copro.Display(fmt.Sprintf("Press <%s> key to select an item", s.app.KeyboardConfig.SelectKey[0]))
}

func (s *Checkbox) RenderChoices(currentPosition int) {

	headSize := 2
	perPage := s.app.Height - headSize - 1

	nbOfSeparator := 0
	for _, choice := range s.Choices {
		if choice.IsSeparator {
			nbOfSeparator++
		}
	}
	limit := len(s.Choices)
	pages := int(math.Ceil(float64(limit) / float64(perPage)))
	current := 0
	if currentPosition >= (perPage - nbOfSeparator) {
		current = currentPosition
	}
	currentMax := int(math.Min(float64(limit), float64(current+perPage)))
	for _, choice := range s.Choices[current:currentMax] {
		if choice.IsSeparator {
			copro.DisplayGrey(" --- " + choice.Label + " ---")
			continue
		}
		isSelected := false
		if choice.pointer == currentPosition {
			isSelected = true
		}
		isChecked := false
		for _, pointer := range s.app.SavedPointers {
			if choice.pointer == pointer {
				isChecked = true
			}
		}
		line := ""
		if isSelected {
			line += usg.Get.Pointer + " "
		} else {
			line += "  "
		}

		if isChecked {
			line += usg.Get.CircleFilled + " "
		} else {
			line += usg.Get.Circle + " "
		}

		line += " " + choice.Label

		if isSelected {
			copro.DisplayYellow(line)
			continue
		}

		copro.Display(line)
	}
	if pages > 1 && current != (limit-nbOfSeparator-1) {
		copro.DisplayGrey(" --- more available ---")
	}
}
<commit_msg>[Checkbox] Fix return values<commit_after>package prompt

import (
	"fmt"
	"math"

	"github.com/julienroland/copro"
	"github.com/julienroland/usg"
)

type Checkbox struct {
	Question      string
	Choices       []*Choice
	CheckedChoice []*Choice
	app           *copro.App
}

type CheckboxResult struct {
	ID    int
	Label string
}

func NewCheckbox() *Checkbox {
	CheckboxList := new(Checkbox)
	app := copro.NewApp()
	CheckboxList.app = app
	CheckboxList.app.Run()
	return CheckboxList
}

func (s *Checkbox) Run() ([]CheckboxResult, error) {
	s.app.EntryCount = len(getChoicesWithoutSeparator(s.Choices)) - 1
	s.setChoicePointer()
	s.manageDefaultPointer()
	s.app.Renderer(func() {
		s.RenderHeader()
		s.RenderChoices(s.app.Pointer)
	})

	result := []CheckboxResult{}
	for _, pointer := range s.app.SavedPointers {
		for _, choice := range s.Choices {
			if choice.pointer == pointer && !choice.IsSeparator {
				result = append(result, CheckboxResult{
					ID:    choice.ID,
					Label: choice.Label,
				})
			}
		}
	}
	return result, nil
}

func (s *Checkbox) manageDefaultPointer() {
	pointers := []int{}
	for _, choice := range s.Choices {
		if choice.Selected && !choice.IsSeparator {
			pointers = append(pointers, choice.pointer)
		}
	}
	s.app.SavedPointers = pointers

}

func (s *Checkbox) setChoicePointer() error {
	position := 0
	for _, choice := range s.Choices {
		if choice.IsSeparator {
			continue
		}
		choice.pointer = position
		position++
	}
	return nil
}

func (s *Checkbox) RenderHeader() {
	if len(s.Question) <= 0 {
		fmt.Errorf("You need to ask something")
	}

	copro.DisplayGreen(s.Question)
	copro.Display(fmt.Sprintf("Press <%s> key to select an item", s.app.KeyboardConfig.SelectKey[0]))
}

func (s *Checkbox) RenderChoices(currentPosition int) {

	headSize := 2
	perPage := s.app.Height - headSize - 1

	nbOfSeparator := 0
	for _, choice := range s.Choices {
		if choice.IsSeparator {
			nbOfSeparator++
		}
	}
	limit := len(s.Choices)
	pages := int(math.Ceil(float64(limit) / float64(perPage)))
	current := 0
	if currentPosition >= (perPage - nbOfSeparator) {
		current = currentPosition
	}
	currentMax := int(math.Min(float64(limit), float64(current+perPage)))
	for _, choice := range s.Choices[current:currentMax] {
		if choice.IsSeparator {
			copro.DisplayGrey(" --- " + choice.Label + " ---")
			continue
		}
		isSelected := false
		if choice.pointer == currentPosition {
			isSelected = true
		}
		isChecked := false
		for _, pointer := range s.app.SavedPointers {
			if choice.pointer == pointer {
				isChecked = true
			}
		}
		line := ""
		if isSelected {
			line += usg.Get.Pointer + " "
		} else {
			line += "  "
		}

		if isChecked {
			line += usg.Get.CircleFilled + " "
		} else {
			line += usg.Get.Circle + " "
		}

		line += " " + choice.Label

		if isSelected {
			copro.DisplayYellow(line)
			continue
		}

		copro.Display(line)
	}
	if pages > 1 && current != (limit-nbOfSeparator-1) {
		copro.DisplayGrey(" --- more available ---")
	}
}
<|endoftext|>


<commit_before>// Package vk implements the OAuth2 protocol for authenticating users through vk.com.
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package vk

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

var (
	authURL      = "https://oauth.vk.com/authorize"
	tokenURL     = "https://oauth.vk.com/access_token"
	endpointUser = "https://api.vk.com/method/users.get"
	apiVersion   = "5.71"
)

// New creates a new VK provider and sets up important connection details.
// You should always call `vk.New` to get a new provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "vk",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing Github.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
	version      string
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// BeginAuth asks VK for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}

	return session, nil
}

// FetchUser will go to VK and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken: sess.AccessToken,
		Provider:    p.Name(),
		ExpiresAt:   sess.ExpiresAt,
		Email:       sess.email,
	}

	if user.AccessToken == "" {
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	fields := "photo_200,nickname"
	requestURL := fmt.Sprintf("%s?user_ids=%d&fields=%s&access_token=%s&v=%s", endpointUser, sess.userID, fields, sess.AccessToken, apiVersion)
	response, err := p.Client().Get(requestURL)
	if err != nil {
		return user, err
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, response.StatusCode)
	}

	bits, err := ioutil.ReadAll(response.Body)

	err = json.NewDecoder(bytes.NewReader(bits)).Decode(&user.RawData)
	if err != nil {
		return user, err
	}

	err = userFromReader(bytes.NewReader(bits), &user)
	return user, err
}

func userFromReader(reader io.Reader, user *goth.User) error {
	response := struct {
		Response []struct {
			ID        int    `json:"id"`
			FirstName string `json:"first_name"`
			LastName  string `json:"last_name"`
			NickName  string `json:"nickname"`
			Photo200  string `json:"photo_200"`
		} `json:"response"`
	}{}

	err := json.NewDecoder(reader).Decode(&response)
	if err != nil {
		return err
	}

	if len(response.Response) == 0 {
		return fmt.Errorf("vk cannot get user information")
	}

	u := response.Response[0]

	user.UserID = string(u.ID)
	user.FirstName = u.FirstName
	user.LastName = u.LastName
	user.NickName = u.NickName
	user.AvatarURL = u.Photo200

	return err
}

// Debug is a no-op for the vk package.
func (p *Provider) Debug(debug bool) {}

//RefreshToken refresh token is not provided by vk
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	return nil, errors.New("Refresh token is not provided by vk")
}

//RefreshTokenAvailable refresh token is not provided by vk
func (p *Provider) RefreshTokenAvailable() bool {
	return false
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{
			"email",
		},
	}

	defaultScopes := map[string]struct{}{
		"email": {},
	}

	for _, scope := range scopes {
		if _, exists := defaultScopes[scope]; !exists {
			c.Scopes = append(c.Scopes, scope)
		}
	}

	return c
}
<commit_msg>Fix vk provider bug<commit_after>// Package vk implements the OAuth2 protocol for authenticating users through vk.com.
// This package can be used as a reference implementation of an OAuth2 provider for Goth.
package vk

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

var (
	authURL      = "https://oauth.vk.com/authorize"
	tokenURL     = "https://oauth.vk.com/access_token"
	endpointUser = "https://api.vk.com/method/users.get"
	apiVersion   = "5.71"
)

// New creates a new VK provider and sets up important connection details.
// You should always call `vk.New` to get a new provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "vk",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing Github.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
	version      string
}

// Name is the name used to retrieve this provider later.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// BeginAuth asks VK for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}

	return session, nil
}

// FetchUser will go to VK and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	sess := session.(*Session)
	user := goth.User{
		AccessToken: sess.AccessToken,
		Provider:    p.Name(),
		ExpiresAt:   sess.ExpiresAt,
		Email:       sess.email,
	}

	if user.AccessToken == "" {
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	fields := "photo_200,nickname"
	requestURL := fmt.Sprintf("%s?user_ids=%d&fields=%s&access_token=%s&v=%s", endpointUser, sess.userID, fields, sess.AccessToken, apiVersion)
	response, err := p.Client().Get(requestURL)
	if err != nil {
		return user, err
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, response.StatusCode)
	}

	bits, err := ioutil.ReadAll(response.Body)

	err = json.NewDecoder(bytes.NewReader(bits)).Decode(&user.RawData)
	if err != nil {
		return user, err
	}

	err = userFromReader(bytes.NewReader(bits), &user)
	return user, err
}

func userFromReader(reader io.Reader, user *goth.User) error {
	response := struct {
		Response []struct {
			ID        int64  `json:"id"`
			FirstName string `json:"first_name"`
			LastName  string `json:"last_name"`
			NickName  string `json:"nickname"`
			Photo200  string `json:"photo_200"`
		} `json:"response"`
	}{}

	err := json.NewDecoder(reader).Decode(&response)
	if err != nil {
		return err
	}

	if len(response.Response) == 0 {
		return fmt.Errorf("vk cannot get user information")
	}

	u := response.Response[0]

	user.UserID = strconv.FormatInt(u.ID, 10)
	user.FirstName = u.FirstName
	user.LastName = u.LastName
	user.NickName = u.NickName
	user.AvatarURL = u.Photo200

	return err
}

// Debug is a no-op for the vk package.
func (p *Provider) Debug(debug bool) {}

//RefreshToken refresh token is not provided by vk
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	return nil, errors.New("Refresh token is not provided by vk")
}

//RefreshTokenAvailable refresh token is not provided by vk
func (p *Provider) RefreshTokenAvailable() bool {
	return false
}

func newConfig(provider *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     provider.ClientKey,
		ClientSecret: provider.Secret,
		RedirectURL:  provider.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authURL,
			TokenURL: tokenURL,
		},
		Scopes: []string{
			"email",
		},
	}

	defaultScopes := map[string]struct{}{
		"email": {},
	}

	for _, scope := range scopes {
		if _, exists := defaultScopes[scope]; !exists {
			c.Scopes = append(c.Scopes, scope)
		}
	}

	return c
}
<|endoftext|>


<commit_before>package pubsub

type Subscriber interface {
	Subscribe(name, topic string) (<-chan Event, error)
}

func (p *Inmem) Subscribe(name, topic string) (<-chan Event, error) {
	events := make(chan Event)
	sub := subscription{
		name:      name,
		topic:     topic,
		eventChan: events,
	}
	p.mtx.Lock()
	p.subscriptions[topic] = append(p.subscriptions[topic], sub)
	p.mtx.Unlock()

	return events, nil
}

func (p *Inmem) dispatch() {
	for {
		select {
		case ev := <-p.publish:
			for _, sub := range p.subscriptions[ev.Topic] {
				go func(s subscription) { s.eventChan <- ev }(sub)
			}
		}
	}
}
<commit_msg>Fix race condition in pubsub package.<commit_after>package pubsub

type Subscriber interface {
	Subscribe(name, topic string) (<-chan Event, error)
}

func (p *Inmem) Subscribe(name, topic string) (<-chan Event, error) {
	events := make(chan Event)
	sub := subscription{
		name:      name,
		topic:     topic,
		eventChan: events,
	}
	p.mtx.Lock()
	p.subscriptions[topic] = append(p.subscriptions[topic], sub)
	p.mtx.Unlock()

	return events, nil
}

func (p *Inmem) dispatch() {
	for {
		select {
		case ev := <-p.publish:
			p.mtx.Lock()
			for _, sub := range p.subscriptions[ev.Topic] {
				go func(s subscription) { s.eventChan <- ev }(sub)
			}
			p.mtx.Unlock()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build freebsd || netbsd
// +build freebsd netbsd

package unix

import (
	"strings"
	"unsafe"
)

// Derive extattr namespace and attribute name

func xattrnamespace(fullattr string) (ns int, attr string, err error) {
	s := strings.IndexByte(fullattr, '.')
	if s == -1 {
		return -1, "", ENOATTR
	}

	namespace := fullattr[0:s]
	attr = fullattr[s+1:]

	switch namespace {
	case "user":
		return EXTATTR_NAMESPACE_USER, attr, nil
	case "system":
		return EXTATTR_NAMESPACE_SYSTEM, attr, nil
	default:
		return -1, "", ENOATTR
	}
}

func initxattrdest(dest []byte, idx int) (d unsafe.Pointer) {
	if len(dest) > idx {
		return unsafe.Pointer(&dest[idx])
	} else {
		return unsafe.Pointer(_zero)
	}
}

// FreeBSD and NetBSD implement their own syscalls to handle extended attributes

func Getxattr(file string, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetFile(file, nsid, a, uintptr(d), destsize)
}

func Fgetxattr(fd int, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetFd(fd, nsid, a, uintptr(d), destsize)
}

func Lgetxattr(link string, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetLink(link, nsid, a, uintptr(d), destsize)
}

// flags are unused on FreeBSD

func Fsetxattr(fd int, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetFd(fd, nsid, a, uintptr(d), datasiz)
	return
}

func Setxattr(file string, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetFile(file, nsid, a, uintptr(d), datasiz)
	return
}

func Lsetxattr(link string, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetLink(link, nsid, a, uintptr(d), datasiz)
	return
}

func Removexattr(file string, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteFile(file, nsid, a)
	return
}

func Fremovexattr(fd int, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteFd(fd, nsid, a)
	return
}

func Lremovexattr(link string, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteLink(link, nsid, a)
	return
}

func Listxattr(file string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	// FreeBSD won't allow you to list xattrs from multiple namespaces
	s := 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := ExtattrListFile(file, nsid, uintptr(d), destsiz)

		/* Errors accessing system attrs are ignored so that
		 * we can implement the Linux-like behavior of omitting errors that
		 * we don't have read permissions on
		 *
		 * Linux will still error if we ask for user attributes on a file that
		 * we don't have read permissions on, so don't ignore those errors
		 */
		if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
			continue
		} else if e != nil {
			return s, e
		}

		s += stmp
		destsiz -= s
		if destsiz < 0 {
			destsiz = 0
		}
		d = initxattrdest(dest, s)
	}

	return s, nil
}

func Flistxattr(fd int, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	s := 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := ExtattrListFd(fd, nsid, uintptr(d), destsiz)
		if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
			continue
		} else if e != nil {
			return s, e
		}

		s += stmp
		destsiz -= s
		if destsiz < 0 {
			destsiz = 0
		}
		d = initxattrdest(dest, s)
	}

	return s, nil
}

func Llistxattr(link string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	s := 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := ExtattrListLink(link, nsid, uintptr(d), destsiz)
		if e != nil && e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
			continue
		} else if e != nil {
			return s, e
		}

		s += stmp
		destsiz -= s
		if destsiz < 0 {
			destsiz = 0
		}
		d = initxattrdest(dest, s)
	}

	return s, nil
}
<commit_msg>unix: add namespaced versions of Listxattr/Flistxattr/Llistxattr on *BSD<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build freebsd || netbsd
// +build freebsd netbsd

package unix

import (
	"strings"
	"unsafe"
)

// Derive extattr namespace and attribute name

func xattrnamespace(fullattr string) (ns int, attr string, err error) {
	s := strings.IndexByte(fullattr, '.')
	if s == -1 {
		return -1, "", ENOATTR
	}

	namespace := fullattr[0:s]
	attr = fullattr[s+1:]

	switch namespace {
	case "user":
		return EXTATTR_NAMESPACE_USER, attr, nil
	case "system":
		return EXTATTR_NAMESPACE_SYSTEM, attr, nil
	default:
		return -1, "", ENOATTR
	}
}

func initxattrdest(dest []byte, idx int) (d unsafe.Pointer) {
	if len(dest) > idx {
		return unsafe.Pointer(&dest[idx])
	} else {
		return unsafe.Pointer(_zero)
	}
}

// FreeBSD and NetBSD implement their own syscalls to handle extended attributes

func Getxattr(file string, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetFile(file, nsid, a, uintptr(d), destsize)
}

func Fgetxattr(fd int, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetFd(fd, nsid, a, uintptr(d), destsize)
}

func Lgetxattr(link string, attr string, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsize := len(dest)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return -1, err
	}

	return ExtattrGetLink(link, nsid, a, uintptr(d), destsize)
}

// flags are unused on FreeBSD

func Fsetxattr(fd int, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetFd(fd, nsid, a, uintptr(d), datasiz)
	return
}

func Setxattr(file string, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetFile(file, nsid, a, uintptr(d), datasiz)
	return
}

func Lsetxattr(link string, attr string, data []byte, flags int) (err error) {
	var d unsafe.Pointer
	if len(data) > 0 {
		d = unsafe.Pointer(&data[0])
	}
	datasiz := len(data)

	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	_, err = ExtattrSetLink(link, nsid, a, uintptr(d), datasiz)
	return
}

func Removexattr(file string, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteFile(file, nsid, a)
	return
}

func Fremovexattr(fd int, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteFd(fd, nsid, a)
	return
}

func Lremovexattr(link string, attr string) (err error) {
	nsid, a, err := xattrnamespace(attr)
	if err != nil {
		return
	}

	err = ExtattrDeleteLink(link, nsid, a)
	return
}

func Listxattr(file string, dest []byte) (sz int, err error) {
	destsiz := len(dest)

	// FreeBSD won't allow you to list xattrs from multiple namespaces
	s, pos := 0, 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := ListxattrNS(file, nsid, dest[pos:])

		/* Errors accessing system attrs are ignored so that
		 * we can implement the Linux-like behavior of omitting errors that
		 * we don't have read permissions on
		 *
		 * Linux will still error if we ask for user attributes on a file that
		 * we don't have read permissions on, so don't ignore those errors
		 */
		if e != nil {
			if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
				continue
			}
			return s, e
		}

		s += stmp
		pos = s
		if pos > destsiz {
			pos = destsiz
		}
	}

	return s, nil
}

func ListxattrNS(file string, nsid int, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	s, e := ExtattrListFile(file, nsid, uintptr(d), destsiz)
	if e != nil {
		return 0, err
	}

	return s, nil
}

func Flistxattr(fd int, dest []byte) (sz int, err error) {
	destsiz := len(dest)

	s, pos := 0, 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := FlistxattrNS(fd, nsid, dest[pos:])

		if e != nil {
			if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
				continue
			}
			return s, e
		}

		s += stmp
		pos = s
		if pos > destsiz {
			pos = destsiz
		}
	}

	return s, nil
}

func FlistxattrNS(fd int, nsid int, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	s, e := ExtattrListFd(fd, nsid, uintptr(d), destsiz)
	if e != nil {
		return 0, err
	}

	return s, nil
}

func Llistxattr(link string, dest []byte) (sz int, err error) {
	destsiz := len(dest)

	s, pos := 0, 0
	for _, nsid := range [...]int{EXTATTR_NAMESPACE_USER, EXTATTR_NAMESPACE_SYSTEM} {
		stmp, e := LlistxattrNS(link, nsid, dest[pos:])

		if e != nil {
			if e == EPERM && nsid != EXTATTR_NAMESPACE_USER {
				continue
			}
			return s, e
		}

		s += stmp
		pos = s
		if pos > destsiz {
			pos = destsiz
		}
	}

	return s, nil
}

func LlistxattrNS(link string, nsid int, dest []byte) (sz int, err error) {
	d := initxattrdest(dest, 0)
	destsiz := len(dest)

	s, e := ExtattrListLink(link, nsid, uintptr(d), destsiz)
	if e != nil {
		return 0, err
	}

	return s, nil
}
<|endoftext|>


<commit_before>package dotquote

import (
	"sort"

	"testing"
)

func TestUnmarshalMapStringString(t *testing.T) {

	tests := []struct{
		Bytes []byte
		Expected map[string][]string
	}{
		{
			Bytes: []byte(`"apple"="one" "banana"="two" "cherry"="three" "kiwi"."watermelon"="45" "one"."two"."three"=[] "zero"=["alpha", "beta"] "do"."it"="ONE" "do"."it"="TWO" "do"."it"=["THREE", "FOUR"]`),
			Expected: map[string][]string{
				`"apple"`:             []string{`one`},
				`"banana"`:            []string{`two`},
				`"cherry"`:            []string{`three`},
				`"kiwi"."watermelon"`: []string{`45`},
				`"one"."two"."three"`: []string{},
				`"zero"`:              []string{`alpha`, `beta`},
				`"do"."it"`:           []string{`ONE`, `TWO`, `THREE`, `FOUR`},
			},
		},
	}

	for testNumber, test := range tests {

		actualMap := map[string][]string{}

		if err := Unmarshal(test.Bytes, actualMap); nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		if expected, actual := len(test.Expected), len(actualMap); expected != actual {
			t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)

			expectedKeys := []string{}
			for k, _ := range test.Expected {
				expectedKeys = append(expectedKeys, k)
			}
			sort.Strings(expectedKeys)
			for i,k := range expectedKeys {
				t.Errorf("EXPECTED [%d] %s => %q", i, k, test.Expected[k])
			}

			actualKeys := []string{}
			for k, _ := range actualMap {
				actualKeys = append(actualKeys, k)
			}
			sort.Strings(actualKeys)
			for i,k := range actualKeys {
				t.Errorf("ACTUAL   [%d] %s => %q", i, k, actualMap[k])
			}

			continue
		}
		for key, expectedValues := range test.Expected {
			actualValues := actualMap[key]

			for valueNumber, expectedValue := range expectedValues {
				actualValue := actualValues[valueNumber]

				if expected, actual := expectedValue, actualValue; expected != actual {
					t.Errorf("For test #%d and key %q and value #%d, expected %q, but actually got %q.", testNumber, key, valueNumber, expected, actual)
					continue
				}
			}
		}
	}
}
<commit_msg>improved tests<commit_after>package dotquote

import (
	"sort"

	"testing"
)

func TestUnmarshalMapStringString(t *testing.T) {

	tests := []struct{
		Bytes []byte
		Expected map[string][]string
	}{
		{
			Bytes: []byte(`"apple"="one" "banana"="two" "cherry"="three" "kiwi"."watermelon"="45" "one"."two"."three"=[] "zero"=["alpha", "beta"] "do"."it"="ONE" "do"."it"="TWO" "do"."it"=["THREE", "FOUR"]`),
			Expected: map[string][]string{
				`"apple"`:             []string{`one`},
				`"banana"`:            []string{`two`},
				`"cherry"`:            []string{`three`},
				`"kiwi"."watermelon"`: []string{`45`},
				`"one"."two"."three"`: []string{},
				`"zero"`:              []string{`alpha`, `beta`},
				`"do"."it"`:           []string{`ONE`, `TWO`, `THREE`, `FOUR`},
			},
		},
		{
			Bytes: []byte(` "app"."name"="myapi" "app"."deployenv"="PROD" "app"."build"."git"."branch"="master" "app"."build"."git"."revision"="2e3eef1dc1d1" "app"."build"."number"="5432" "app"."build"."when"="Mon Nov 21 10:05:14 PST 2016" "request"."client-address"="123.45.6.7:41325" "request"."method"="GET" "request"."uri"="/my/1.0/apple/banana/cherry" "request"."protocol"="HTTP/1.1" "request"."host"="localhost:8080" "request"."content-length"="0" "request"."transfer-encoding"=[] "response"."status-code"="200" "response"."content-length"="173" "trace"."begin-time"="2016-11-22 23:01:43.065121584 +0000 UTC" "trace"."end-time"="2016-11-22 23:01:43.065378496 +0000 UTC" "request"."header"."Upgrade-Insecure-Requests"=["1"] "request"."header"."User-Agent"=["Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0"] "request"."header"."Accept"=["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"] "request"."header"."Accept-Language"=["en-US,en;q=0.5"] "request"."header"."Accept-Encoding"=["gzip, deflate"] "request"."header"."Dnt"=["1"] "request"."header"."Connection"=["keep-alive"] `),
			Expected: map[string][]string{
				`"app"."name"`:                                   []string{`myapi`},
				`"app"."deployenv"`:                              []string{`PROD`},
				`"app"."build"."git"."branch"`:                   []string{`master`},
				`"app"."build"."git"."revision"`:                 []string{`2e3eef1dc1d1`},
				`"app"."build"."number"`:                         []string{`5432`},
				`"app"."build"."when"`:                           []string{`Mon Nov 21 10:05:14 PST 2016`},
				`"request"."client-address"`:                     []string{`123.45.6.7:41325`},
				`"request"."method"`:                             []string{`GET`},
				`"request"."uri"`:                                []string{`/my/1.0/apple/banana/cherry`},
				`"request"."protocol"`:                           []string{`HTTP/1.1`},
				`"request"."host"`:                               []string{`localhost:8080`},
				`"request"."content-length"`:                     []string{`0`},
				`"request"."transfer-encoding"`:                  []string{},
				`"response"."status-code"`:                       []string{`200`},
				`"response"."content-length"`:                    []string{`173`},
				`"trace"."begin-time"`:                           []string{`2016-11-22 23:01:43.065121584 +0000 UTC`},
				`"trace"."end-time"`:                             []string{`2016-11-22 23:01:43.065378496 +0000 UTC`},
				`"request"."header"."Upgrade-Insecure-Requests"`: []string{`1`},
				`"request"."header"."User-Agent"`:                []string{`Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0`},
				`"request"."header"."Accept"`:                    []string{`text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8`},
				`"request"."header"."Accept-Language"`:           []string{`en-US,en;q=0.5`},
				`"request"."header"."Accept-Encoding"`:           []string{`gzip, deflate`},
				`"request"."header"."Dnt"`:                       []string{`1`},
				`"request"."header"."Connection"`:                []string{`keep-alive`},
			},
		},
	}

	for testNumber, test := range tests {

		actualMap := map[string][]string{}

		if err := Unmarshal(test.Bytes, actualMap); nil != err {
			t.Errorf("For test #%d, did not expect an error, but actually got one: (%T) %v", testNumber, err, err)
			continue
		}

		if expected, actual := len(test.Expected), len(actualMap); expected != actual {
			t.Errorf("For test #%d, expected %d, but actually got %d.", testNumber, expected, actual)

			expectedKeys := []string{}
			for k, _ := range test.Expected {
				expectedKeys = append(expectedKeys, k)
			}
			sort.Strings(expectedKeys)
			for i,k := range expectedKeys {
				t.Errorf("EXPECTED [%d] %s => %q", i, k, test.Expected[k])
			}

			actualKeys := []string{}
			for k, _ := range actualMap {
				actualKeys = append(actualKeys, k)
			}
			sort.Strings(actualKeys)
			for i,k := range actualKeys {
				t.Errorf("ACTUAL   [%d] %s => %q", i, k, actualMap[k])
			}

			continue
		}
		for key, expectedValues := range test.Expected {
			actualValues, ok := actualMap[key]
			if !ok {
				t.Errorf("For test #%d and key |||||%s|||||, expected value(s) for key, but do not actually have that key.", testNumber, key)
				continue
			}

			if expected, actual := len(expectedValues), len(actualValues); expected != actual {
				t.Errorf("For test #%d and key |||||%s|||||, expected %d, but actually got %d.", testNumber, key, expected, actual)
				continue
			}
			for valueNumber, expectedValue := range expectedValues {
				actualValue := actualValues[valueNumber]

				if expected, actual := expectedValue, actualValue; expected != actual {
					t.Errorf("For test #%d and key |||||%s||||| and value #%d, expected %q, but actually got %q.", testNumber, key, valueNumber, expected, actual)
					continue
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"net/http"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	logging "github.com/op/go-logging"
	"github.com/spf13/cobra"
)

var (
	baseDir  string
	threadId = regexp.MustCompile(`thread-(\d*)-`)
	imageId  = regexp.MustCompile(`([^\/]+)\.(png|jpg)`)
	log      = logging.MustGetLogger("iloveck101")
)

func worker(destDir string, linkChan chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for target := range linkChan {
		resp, err := http.Get(target)
		if err != nil {
			log.Debug("Http.Get\nerror: %s\ntarget: %s", err, target)
			continue
		}
		defer resp.Body.Close()

		m, _, err := image.Decode(resp.Body)
		if err != nil {
			log.Debug("image.Decode\nerror: %s\ntarget: %s", err, target)
			continue
		}

		// Ignore small images
		bounds := m.Bounds()
		if bounds.Size().X > 300 && bounds.Size().Y > 300 {
			imgInfo := imageId.FindStringSubmatch(target)
			out, err := os.Create(destDir + "/" + imgInfo[1] + "." + imgInfo[2])
			if err != nil {
				log.Debug("os.Create\nerror: %s", err)
				continue
			}
			defer out.Close()
			switch imgInfo[2] {
			case "jpg":
				jpeg.Encode(out, m, nil)
			case "png":
				png.Encode(out, m)
			}
		}
	}
}

func crawler(target string, workerNum int) {
	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1#thread_subject").Text()
	dir := fmt.Sprintf("%v/%v - %v", baseDir, threadId.FindStringSubmatch(target)[1], title)

	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

// [todo] - Holy shit function, should refactor it!
func printGoogleResult(keyword string, page int) (hrefs []string) {
	client := &http.Client{}
	queryUrl := fmt.Sprintf("https://www.google.com.tw/search?espv=210&es_sm=119&q=%v+site:ck101.com&start=%v", keyword, page*10)
	req, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	hrefs = make([]string, 0)

	// Print result list
	doc.Find("li.g h3.r a").Each(func(i int, s *goquery.Selection) {
		title := s.Text()
		href, exist := s.Attr("href")
		if exist {
			hrefs = append(hrefs, href)
			fmt.Printf("[%v] %v\n", i, title)
		}
	})

	// Print pages
	fmt.Print("Pages: ")
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				fmt.Printf("[%v] ", i)
			} else {
				fmt.Printf("%v ", i)
			}
		}
	}
	fmt.Printf("(n:next, p:prev)\n")

	return hrefs
}

func main() {
	var format = logging.MustStringFormatter("%{level} %{message}")
	logging.SetFormatter(format)
	logging.SetLevel(logging.INFO, "iloveck101")

	usr, _ := user.Current()
	baseDir = fmt.Sprintf("%v/Pictures/iloveck101", usr.HomeDir)

	var postUrl string
	var workerNum int

	rootCmd := &cobra.Command{
		Use:   "iloveck101",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			crawler(postUrl, workerNum)
		},
	}
	rootCmd.Flags().StringVarP(&postUrl, "url", "u", "http://ck101.com/thread-2876990-1-1.html", "Url of post")
	rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 25, "Number of workers")

	searchCmd := &cobra.Command{
		Use:   "search",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			page := 0
			keyword := args[0]
			hrefs := printGoogleResult(keyword, page)

			scanner := bufio.NewScanner(os.Stdin)
			quit := false

			for !quit {
				fmt.Print("ck101> ")

				if !scanner.Scan() {
					break
				}

				line := scanner.Text()
				parts := strings.Split(line, " ")
				cmd := parts[0]
				args := parts[1:]

				switch cmd {
				case "quit":
					quit = true
				case "n":
					page = page + 1
					hrefs = printGoogleResult(keyword, page)
				case "p":
					if page > 0 {
						page = page - 1
					}
					hrefs = printGoogleResult(keyword, page)
				case "s":
					page = 0
					hrefs = printGoogleResult(args[0], page)
				case "d":
					index, err := strconv.ParseUint(args[0], 0, 0)
					if err != nil {
						fmt.Println(err)
						continue
					}
					if int(index) >= len(hrefs) {
						fmt.Println("Invalid index")
						continue
					}

					// Only support url with format ck101.com/thread-xxx
					if threadId.Match([]byte(hrefs[index])) {
						crawler(hrefs[index], 25)
						fmt.Println("Done!")
					} else {
						fmt.Println("Unsupport url:", hrefs[index])
					}
				default:
					fmt.Println("Unrecognized command:", cmd, args)
				}
			}
		},
	}

	rootCmd.AddCommand(searchCmd)
	rootCmd.Execute()
}
<commit_msg>Support Open ~/Pictures/iloveck101 in finder.<commit_after>package main

import (
	"bufio"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"net/http"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
	logging "github.com/op/go-logging"
	"github.com/skratchdot/open-golang/open"
	"github.com/spf13/cobra"
)

var (
	baseDir  string
	threadId = regexp.MustCompile(`thread-(\d*)-`)
	imageId  = regexp.MustCompile(`([^\/]+)\.(png|jpg)`)
	log      = logging.MustGetLogger("iloveck101")
)

func worker(destDir string, linkChan chan string, wg *sync.WaitGroup) {
	defer wg.Done()

	for target := range linkChan {
		resp, err := http.Get(target)
		if err != nil {
			log.Debug("Http.Get\nerror: %s\ntarget: %s", err, target)
			continue
		}
		defer resp.Body.Close()

		m, _, err := image.Decode(resp.Body)
		if err != nil {
			log.Debug("image.Decode\nerror: %s\ntarget: %s", err, target)
			continue
		}

		// Ignore small images
		bounds := m.Bounds()
		if bounds.Size().X > 300 && bounds.Size().Y > 300 {
			imgInfo := imageId.FindStringSubmatch(target)
			out, err := os.Create(destDir + "/" + imgInfo[1] + "." + imgInfo[2])
			if err != nil {
				log.Debug("os.Create\nerror: %s", err)
				continue
			}
			defer out.Close()
			switch imgInfo[2] {
			case "jpg":
				jpeg.Encode(out, m, nil)
			case "png":
				png.Encode(out, m)
			}
		}
	}
}

func crawler(target string, workerNum int) {
	doc, err := goquery.NewDocument(target)
	if err != nil {
		panic(err)
	}

	title := doc.Find("h1#thread_subject").Text()
	dir := fmt.Sprintf("%v/%v - %v", baseDir, threadId.FindStringSubmatch(target)[1], title)

	os.MkdirAll(dir, 0755)

	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go worker(dir, linkChan, wg)
	}

	doc.Find("div[itemprop=articleBody] img").Each(func(i int, img *goquery.Selection) {
		imgUrl, _ := img.Attr("file")
		linkChan <- imgUrl
	})

	close(linkChan)
	wg.Wait()
}

// [todo] - Holy shit function, should refactor it!
func printGoogleResult(keyword string, page int) (hrefs []string) {
	client := &http.Client{}
	queryUrl := fmt.Sprintf("https://www.google.com.tw/search?espv=210&es_sm=119&q=%v+site:ck101.com&start=%v", keyword, page*10)
	req, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		panic(err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		panic(err)
	}

	hrefs = make([]string, 0)

	// Print result list
	doc.Find("li.g h3.r a").Each(func(i int, s *goquery.Selection) {
		title := s.Text()
		href, exist := s.Attr("href")
		if exist {
			hrefs = append(hrefs, href)
			fmt.Printf("[%v] %v\n", i, title)
		}
	})

	// Print pages
	fmt.Print("Pages: ")
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				fmt.Printf("[%v] ", i)
			} else {
				fmt.Printf("%v ", i)
			}
		}
	}
	fmt.Printf("(n:next, p:prev)\n")

	return hrefs
}

func main() {
	var format = logging.MustStringFormatter("%{level} %{message}")
	logging.SetFormatter(format)
	logging.SetLevel(logging.INFO, "iloveck101")

	usr, _ := user.Current()
	baseDir = fmt.Sprintf("%v/Pictures/iloveck101", usr.HomeDir)

	var postUrl string
	var workerNum int

	rootCmd := &cobra.Command{
		Use:   "iloveck101",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			crawler(postUrl, workerNum)
		},
	}
	rootCmd.Flags().StringVarP(&postUrl, "url", "u", "http://ck101.com/thread-2876990-1-1.html", "Url of post")
	rootCmd.Flags().IntVarP(&workerNum, "worker", "w", 25, "Number of workers")

	searchCmd := &cobra.Command{
		Use:   "search",
		Short: "Download all the images in given post url",
		Run: func(cmd *cobra.Command, args []string) {
			page := 0
			keyword := args[0]
			hrefs := printGoogleResult(keyword, page)

			scanner := bufio.NewScanner(os.Stdin)
			quit := false

			for !quit {
				fmt.Print("ck101> ")

				if !scanner.Scan() {
					break
				}

				line := scanner.Text()
				parts := strings.Split(line, " ")
				cmd := parts[0]
				args := parts[1:]

				switch cmd {
				case "quit":
					quit = true
				case "n":
					page = page + 1
					hrefs = printGoogleResult(keyword, page)
				case "p":
					if page > 0 {
						page = page - 1
					}
					hrefs = printGoogleResult(keyword, page)
				case "s":
					page = 0
					hrefs = printGoogleResult(args[0], page)
				case "o":
					open.Run(baseDir)
				case "d":
					index, err := strconv.ParseUint(args[0], 0, 0)
					if err != nil {
						fmt.Println(err)
						continue
					}
					if int(index) >= len(hrefs) {
						fmt.Println("Invalid index")
						continue
					}

					// Only support url with format ck101.com/thread-xxx
					if threadId.Match([]byte(hrefs[index])) {
						crawler(hrefs[index], 25)
						fmt.Println("Done!")
					} else {
						fmt.Println("Unsupport url:", hrefs[index])
					}
				default:
					fmt.Println("Unrecognized command:", cmd, args)
				}
			}
		},
	}

	rootCmd.AddCommand(searchCmd)
	rootCmd.Execute()
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"fmt"
	"image"
	"math"

	"github.com/hajimehoshi/ebiten/internal/graphics"
)

// Deprecated (as of 1.1.0-alpha): Use ImageParts instead.
type ImagePart struct {
	Dst image.Rectangle
	Src image.Rectangle
}

// An ImageParts represents the parts of the destination image and the parts of the source image.
type ImageParts interface {
	Len() int
	Dst(i int) (x0, y0, x1, y1 int)
	Src(i int) (x0, y0, x1, y1 int)
}

// NOTE: Remove this in the future.
type imageParts []ImagePart

func (p imageParts) Len() int {
	return len(p)
}

func (p imageParts) Dst(i int) (x0, y0, x1, y1 int) {
	dst := &p[i].Dst
	return dst.Min.X, dst.Min.Y, dst.Max.X, dst.Max.Y
}

func (p imageParts) Src(i int) (x0, y0, x1, y1 int) {
	src := &p[i].Src
	return src.Min.X, src.Min.Y, src.Max.X, src.Max.Y
}

type wholeImage struct {
	width  int
	height int
}

func (w *wholeImage) Len() int {
	return 1
}

func (w *wholeImage) Dst(i int) (x0, y0, x1, y1 int) {
	return 0, 0, w.width, w.height
}

func (w *wholeImage) Src(i int) (x0, y0, x1, y1 int) {
	return 0, 0, w.width, w.height
}

func u(x int, width int) int16 {
	return int16(math.MaxInt16 * x / int(graphics.NextPowerOf2Int32(int32(width))))
}

func v(y int, height int) int16 {
	return int16(math.MaxInt16 * y / int(graphics.NextPowerOf2Int32(int32(height))))
}

type textureQuads struct {
	parts  ImageParts
	width  int
	height int
}

func (t *textureQuads) Len() int {
	return t.parts.Len()
}

func (t *textureQuads) SetVertices(vertices []int16) int {
	l := t.Len()
	if len(vertices) < l*16 {
		panic(fmt.Sprintf("graphics: vertices size must be greater than %d but %d", l*16, len(vertices)))
	}
	p := t.parts
	w, h := t.width, t.height
	n := 0
	for i := 0; i < l; i++ {
		dx0, dy0, dx1, dy1 := p.Dst(i)
		if dx0 == dx1 || dy0 == dy1 {
			continue
		}
		x0, y0, x1, y1 := int16(dx0), int16(dy0), int16(dx1), int16(dy1)
		sx0, sy0, sx1, sy1 := p.Src(i)
		if sx0 == sx1 || sy0 == sy1 {
			continue
		}
		u0, v0, u1, v1 := u(sx0, w), v(sy0, h), u(sx1, w), v(sy1, h)
		vertices[16*n] = x0
		vertices[16*n+1] = y0
		vertices[16*n+2] = u0
		vertices[16*n+3] = v0
		vertices[16*n+4] = x1
		vertices[16*n+5] = y0
		vertices[16*n+6] = u1
		vertices[16*n+7] = v0
		vertices[16*n+8] = x0
		vertices[16*n+9] = y1
		vertices[16*n+10] = u0
		vertices[16*n+11] = v1
		vertices[16*n+12] = x1
		vertices[16*n+13] = y1
		vertices[16*n+14] = u1
		vertices[16*n+15] = v1
		n++
	}
	return n
}
<commit_msg>graphics: Add uv (optimization)<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"fmt"
	"image"
	"math"

	"github.com/hajimehoshi/ebiten/internal/graphics"
)

// Deprecated (as of 1.1.0-alpha): Use ImageParts instead.
type ImagePart struct {
	Dst image.Rectangle
	Src image.Rectangle
}

// An ImageParts represents the parts of the destination image and the parts of the source image.
type ImageParts interface {
	Len() int
	Dst(i int) (x0, y0, x1, y1 int)
	Src(i int) (x0, y0, x1, y1 int)
}

// NOTE: Remove this in the future.
type imageParts []ImagePart

func (p imageParts) Len() int {
	return len(p)
}

func (p imageParts) Dst(i int) (x0, y0, x1, y1 int) {
	dst := &p[i].Dst
	return dst.Min.X, dst.Min.Y, dst.Max.X, dst.Max.Y
}

func (p imageParts) Src(i int) (x0, y0, x1, y1 int) {
	src := &p[i].Src
	return src.Min.X, src.Min.Y, src.Max.X, src.Max.Y
}

type wholeImage struct {
	width  int
	height int
}

func (w *wholeImage) Len() int {
	return 1
}

func (w *wholeImage) Dst(i int) (x0, y0, x1, y1 int) {
	return 0, 0, w.width, w.height
}

func (w *wholeImage) Src(i int) (x0, y0, x1, y1 int) {
	return 0, 0, w.width, w.height
}

type uv struct {
	width2p  int
	height2p int
}

func (c uv) u(x int) int16 {
	return int16(math.MaxInt16 * x / c.width2p)
}

func (c uv) v(y int) int16 {
	return int16(math.MaxInt16 * y / c.height2p)
}

type textureQuads struct {
	parts  ImageParts
	width  int
	height int
}

func (t *textureQuads) Len() int {
	return t.parts.Len()
}

func (t *textureQuads) SetVertices(vertices []int16) int {
	l := t.Len()
	if len(vertices) < l*16 {
		panic(fmt.Sprintf("graphics: vertices size must be greater than %d but %d", l*16, len(vertices)))
	}
	p := t.parts
	w, h := t.width, t.height
	n := 0
	c := uv{
		width2p:  int(graphics.NextPowerOf2Int32(int32(w))),
		height2p: int(graphics.NextPowerOf2Int32(int32(h))),
	}
	for i := 0; i < l; i++ {
		dx0, dy0, dx1, dy1 := p.Dst(i)
		if dx0 == dx1 || dy0 == dy1 {
			continue
		}
		x0, y0, x1, y1 := int16(dx0), int16(dy0), int16(dx1), int16(dy1)
		sx0, sy0, sx1, sy1 := p.Src(i)
		if sx0 == sx1 || sy0 == sy1 {
			continue
		}
		u0, v0, u1, v1 := c.u(sx0), c.v(sy0), c.u(sx1), c.v(sy1)
		vertices[16*n] = x0
		vertices[16*n+1] = y0
		vertices[16*n+2] = u0
		vertices[16*n+3] = v0
		vertices[16*n+4] = x1
		vertices[16*n+5] = y0
		vertices[16*n+6] = u1
		vertices[16*n+7] = v0
		vertices[16*n+8] = x0
		vertices[16*n+9] = y1
		vertices[16*n+10] = u0
		vertices[16*n+11] = v1
		vertices[16*n+12] = x1
		vertices[16*n+13] = y1
		vertices[16*n+14] = u1
		vertices[16*n+15] = v1
		n++
	}
	return n
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package imageproxy provides an image proxy server.  For typical use of
// creating and using a Proxy, see cmd/imageproxy/main.go.
package imageproxy

import (
	"bufio"
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/wojtekzw/httpcache"
	"strconv"
)

const (
	// MaxRespBodySize - maximum size of remote image to be proxied. If image is larger Get(url) will return error
	// It is safety feature to protect memory
	MaxRespBodySize = 10 * 1024 * 1024
)

var concurrencyGuard = make(chan struct{}, 20)

// Proxy serves image requests.
type Proxy struct {
	Client *http.Client // client used to fetch remote URLs
	Cache  Cache        // cache used to cache responses

	// Whitelist specifies a list of remote hosts that images can be
	// proxied from.  An empty list means all hosts are allowed.
	Whitelist []string

	// Referrers, when given, requires that requests to the image
	// proxy come from a referring host. An empty list means all
	// hosts are allowed.
	Referrers []string

	// DefaultBaseURL is the URL that relative remote URLs are resolved in
	// reference to.  If nil, all remote URLs specified in requests must be
	// absolute.
	DefaultBaseURL *url.URL

	// SignatureKey is the HMAC key used to verify signed requests.
	SignatureKey []byte

	// Allow images to scale beyond their original dimensions.
	ScaleUp bool
}

// NewProxy constructs a new proxy.  The provided http RoundTripper will be
// used to fetch remote URLs.  If nil is provided, http.DefaultTransport will
// be used.
func NewProxy(transport http.RoundTripper, cache Cache, maxResponseSize uint64) *Proxy {
	if transport == nil {
		transport = http.DefaultTransport
	}
	if cache == nil {
		cache = NopCache
	}

	proxy := Proxy{
		Cache: cache,
	}

	client := new(http.Client)
	client.Transport = &httpcache.Transport{
		Transport:           &TransformingTransport{transport, client, maxResponseSize},
		Cache:               cache,
		MarkCachedResponses: true,
	}

	proxy.Client = client

	return &proxy
}

// ServeHTTP handles image requests.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/favicon.ico" {
		return // ignore favicon requests
	}

	if r.URL.Path == "/health-check" {
		fmt.Fprint(w, "OK")
		return
	}

	concurrencyGuard <- struct{}{}
	defer func() { <-concurrencyGuard }()
	glog.Infof("concurrency: %d",len(concurrencyGuard))

	glog.Infof("pre-request: %v", r.URL.String())

	req, err := NewRequest(r, p.DefaultBaseURL)
	if err != nil {
		msg := fmt.Sprintf("invalid request URL: %v", err)
		glog.Error(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	// assign static settings from proxy to req.Options
	req.Options.ScaleUp = p.ScaleUp

	if err = p.allowed(req); err != nil {
		glog.Error(err)
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	resp, err := p.Client.Get(req.String())
	if err != nil {
		msg := fmt.Sprintf("error fetching remote image: %v", err)
		glog.Error(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	cached := resp.Header.Get(httpcache.XFromCache)
	glog.Infof("request: %v (served from cache: %v)", *req, cached == "1")

	copyHeader(w, resp, "Cache-Control")
	copyHeader(w, resp, "Last-Modified")
	copyHeader(w, resp, "Expires")
	copyHeader(w, resp, "Etag")
	copyHeader(w, resp, "Link")

	if is304 := check304(r, resp); is304 {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	copyHeader(w, resp, "Content-Length")
	copyHeader(w, resp, "Content-Type")
	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
}

func copyHeader(w http.ResponseWriter, r *http.Response, header string) {
	key := http.CanonicalHeaderKey(header)
	if value, ok := r.Header[key]; ok {
		w.Header()[key] = value
	}
}

// allowed determines whether the specified request contains an allowed
// referrer, host, and signature.  It returns an error if the request is not
// allowed.
func (p *Proxy) allowed(r *Request) error {
	if len(p.Referrers) > 0 && !validReferrer(p.Referrers, r.Original) {
		return fmt.Errorf("request does not contain an allowed referrer: %v", r)
	}

	if len(p.Whitelist) == 0 && len(p.SignatureKey) == 0 {
		return nil // no whitelist or signature key, all requests accepted
	}

	if len(p.Whitelist) > 0 && validHost(p.Whitelist, r.URL) {
		return nil
	}

	if len(p.SignatureKey) > 0 && validSignature(p.SignatureKey, r) {
		return nil
	}

	return fmt.Errorf("request does not contain an allowed host or valid signature: %v", r)
}

// validHost returns whether the host in u matches one of hosts.
func validHost(hosts []string, u *url.URL) bool {
	for _, host := range hosts {
		if u.Host == host {
			return true
		}
		if strings.HasPrefix(host, "*.") && strings.HasSuffix(u.Host, host[2:]) {
			return true
		}
	}

	return false
}

// returns whether the referrer from the request is in the host list.
func validReferrer(hosts []string, r *http.Request) bool {
	u, err := url.Parse(r.Header.Get("Referer"))
	if err != nil { // malformed or blank header, just deny
		return false
	}

	return validHost(hosts, u)
}

// validSignature returns whether the request signature is valid.
func validSignature(key []byte, r *Request) bool {
	sig := r.Options.Signature
	if m := len(sig) % 4; m != 0 { // add padding if missing
		sig += strings.Repeat("=", 4-m)
	}

	got, err := base64.URLEncoding.DecodeString(sig)
	if err != nil {
		glog.Errorf("error base64 decoding signature %q", r.Options.Signature)
		return false
	}

	mac := hmac.New(sha256.New, key)
	mac.Write([]byte(r.URL.String()))
	want := mac.Sum(nil)

	return hmac.Equal(got, want)
}

// check304 checks whether we should send a 304 Not Modified in response to
// req, based on the response resp.  This is determined using the last modified
// time and the entity tag of resp.
func check304(req *http.Request, resp *http.Response) bool {
	// TODO(willnorris): if-none-match header can be a comma separated list
	// of multiple tags to be matched, or the special value "*" which
	// matches all etags
	etag := resp.Header.Get("Etag")
	if etag != "" && etag == req.Header.Get("If-None-Match") {
		return true
	}

	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return false
	}
	ifModSince, err := time.Parse(time.RFC1123, req.Header.Get("If-Modified-Since"))
	if err != nil {
		return false
	}
	if lastModified.Before(ifModSince) {
		return true
	}

	return false
}

// TransformingTransport is an implementation of http.RoundTripper that
// optionally transforms images using the options specified in the request URL
// fragment.
type TransformingTransport struct {
	// Transport is the underlying http.RoundTripper used to satisfy
	// non-transform requests (those that do not include a URL fragment).
	Transport http.RoundTripper

	// CachingClient is used to fetch images to be resized.  This client is
	// used rather than Transport directly in order to ensure that
	// responses are properly cached.
	CachingClient *http.Client

	// ResponseSize - maximum size of remote image to be be fetched by Client. If image is larger Get(url) will return error
	ResponseSize uint64
}

// RoundTrip implements the http.RoundTripper interface.
func (t *TransformingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if req.URL.Fragment == "" {
		// normal requests pass through
		glog.Infof("fetching remote URL: %v", req.URL)
		return t.Transport.RoundTrip(req)
	}

	u := *req.URL
	u.Fragment = ""
	resp, err := t.CachingClient.Get(u.String())
	if err != nil {
		return nil, err
	}

	contentLength, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
	// no data reading - check first Content-Length
	if uint64(contentLength) > t.ResponseSize {
		return nil, fmt.Errorf("size too large: max size: %d, content-length: %d",t.ResponseSize, contentLength)
	}

	//read data with limiter if there is no Content-Length header or it is fake
	resp.Body = NewLimitedReadCloser(resp.Body, int64(t.ResponseSize))
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	opt := ParseOptions(req.URL.Fragment)

	img, err := Transform(b, opt,u.String())
	if err != nil {
		glog.Errorf("error transforming image: %v, Content-Type: %v, URL: %v", err, resp.Header.Get("Content-Type"),req.URL)
		img = b
	}

	// replay response with transformed image and updated content length
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "%s %s\n", resp.Proto, resp.Status)
	resp.Header.WriteSubset(buf, map[string]bool{"Content-Length": true})
	fmt.Fprintf(buf, "Content-Length: %d\n\n", len(img))
	buf.Write(img)

	return http.ReadResponse(bufio.NewReader(buf), req)
}
<commit_msg>concurrency limit 15<commit_after>// Copyright 2013 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package imageproxy provides an image proxy server.  For typical use of
// creating and using a Proxy, see cmd/imageproxy/main.go.
package imageproxy

import (
	"bufio"
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/wojtekzw/httpcache"
	"strconv"
)

const (
	// MaxRespBodySize - maximum size of remote image to be proxied. If image is larger Get(url) will return error
	// It is safety feature to protect memory
	MaxRespBodySize = 10 * 1024 * 1024
)

var concurrencyGuard = make(chan struct{}, 15)

// Proxy serves image requests.
type Proxy struct {
	Client *http.Client // client used to fetch remote URLs
	Cache  Cache        // cache used to cache responses

	// Whitelist specifies a list of remote hosts that images can be
	// proxied from.  An empty list means all hosts are allowed.
	Whitelist []string

	// Referrers, when given, requires that requests to the image
	// proxy come from a referring host. An empty list means all
	// hosts are allowed.
	Referrers []string

	// DefaultBaseURL is the URL that relative remote URLs are resolved in
	// reference to.  If nil, all remote URLs specified in requests must be
	// absolute.
	DefaultBaseURL *url.URL

	// SignatureKey is the HMAC key used to verify signed requests.
	SignatureKey []byte

	// Allow images to scale beyond their original dimensions.
	ScaleUp bool
}

// NewProxy constructs a new proxy.  The provided http RoundTripper will be
// used to fetch remote URLs.  If nil is provided, http.DefaultTransport will
// be used.
func NewProxy(transport http.RoundTripper, cache Cache, maxResponseSize uint64) *Proxy {
	if transport == nil {
		transport = http.DefaultTransport
	}
	if cache == nil {
		cache = NopCache
	}

	proxy := Proxy{
		Cache: cache,
	}

	client := new(http.Client)
	client.Transport = &httpcache.Transport{
		Transport:           &TransformingTransport{transport, client, maxResponseSize},
		Cache:               cache,
		MarkCachedResponses: true,
	}

	proxy.Client = client

	return &proxy
}

// ServeHTTP handles image requests.
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/favicon.ico" {
		return // ignore favicon requests
	}

	if r.URL.Path == "/health-check" {
		fmt.Fprint(w, "OK")
		return
	}

	concurrencyGuard <- struct{}{}
	defer func() { <-concurrencyGuard }()
	glog.Infof("concurrency: %d",len(concurrencyGuard))

	glog.Infof("pre-request: %v", r.URL.String())

	req, err := NewRequest(r, p.DefaultBaseURL)
	if err != nil {
		msg := fmt.Sprintf("invalid request URL: %v", err)
		glog.Error(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	// assign static settings from proxy to req.Options
	req.Options.ScaleUp = p.ScaleUp

	if err = p.allowed(req); err != nil {
		glog.Error(err)
		http.Error(w, err.Error(), http.StatusForbidden)
		return
	}

	resp, err := p.Client.Get(req.String())
	if err != nil {
		msg := fmt.Sprintf("error fetching remote image: %v", err)
		glog.Error(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	cached := resp.Header.Get(httpcache.XFromCache)
	glog.Infof("request: %v (served from cache: %v)", *req, cached == "1")

	copyHeader(w, resp, "Cache-Control")
	copyHeader(w, resp, "Last-Modified")
	copyHeader(w, resp, "Expires")
	copyHeader(w, resp, "Etag")
	copyHeader(w, resp, "Link")

	if is304 := check304(r, resp); is304 {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	copyHeader(w, resp, "Content-Length")
	copyHeader(w, resp, "Content-Type")
	w.WriteHeader(resp.StatusCode)
	io.Copy(w, resp.Body)
}

func copyHeader(w http.ResponseWriter, r *http.Response, header string) {
	key := http.CanonicalHeaderKey(header)
	if value, ok := r.Header[key]; ok {
		w.Header()[key] = value
	}
}

// allowed determines whether the specified request contains an allowed
// referrer, host, and signature.  It returns an error if the request is not
// allowed.
func (p *Proxy) allowed(r *Request) error {
	if len(p.Referrers) > 0 && !validReferrer(p.Referrers, r.Original) {
		return fmt.Errorf("request does not contain an allowed referrer: %v", r)
	}

	if len(p.Whitelist) == 0 && len(p.SignatureKey) == 0 {
		return nil // no whitelist or signature key, all requests accepted
	}

	if len(p.Whitelist) > 0 && validHost(p.Whitelist, r.URL) {
		return nil
	}

	if len(p.SignatureKey) > 0 && validSignature(p.SignatureKey, r) {
		return nil
	}

	return fmt.Errorf("request does not contain an allowed host or valid signature: %v", r)
}

// validHost returns whether the host in u matches one of hosts.
func validHost(hosts []string, u *url.URL) bool {
	for _, host := range hosts {
		if u.Host == host {
			return true
		}
		if strings.HasPrefix(host, "*.") && strings.HasSuffix(u.Host, host[2:]) {
			return true
		}
	}

	return false
}

// returns whether the referrer from the request is in the host list.
func validReferrer(hosts []string, r *http.Request) bool {
	u, err := url.Parse(r.Header.Get("Referer"))
	if err != nil { // malformed or blank header, just deny
		return false
	}

	return validHost(hosts, u)
}

// validSignature returns whether the request signature is valid.
func validSignature(key []byte, r *Request) bool {
	sig := r.Options.Signature
	if m := len(sig) % 4; m != 0 { // add padding if missing
		sig += strings.Repeat("=", 4-m)
	}

	got, err := base64.URLEncoding.DecodeString(sig)
	if err != nil {
		glog.Errorf("error base64 decoding signature %q", r.Options.Signature)
		return false
	}

	mac := hmac.New(sha256.New, key)
	mac.Write([]byte(r.URL.String()))
	want := mac.Sum(nil)

	return hmac.Equal(got, want)
}

// check304 checks whether we should send a 304 Not Modified in response to
// req, based on the response resp.  This is determined using the last modified
// time and the entity tag of resp.
func check304(req *http.Request, resp *http.Response) bool {
	// TODO(willnorris): if-none-match header can be a comma separated list
	// of multiple tags to be matched, or the special value "*" which
	// matches all etags
	etag := resp.Header.Get("Etag")
	if etag != "" && etag == req.Header.Get("If-None-Match") {
		return true
	}

	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return false
	}
	ifModSince, err := time.Parse(time.RFC1123, req.Header.Get("If-Modified-Since"))
	if err != nil {
		return false
	}
	if lastModified.Before(ifModSince) {
		return true
	}

	return false
}

// TransformingTransport is an implementation of http.RoundTripper that
// optionally transforms images using the options specified in the request URL
// fragment.
type TransformingTransport struct {
	// Transport is the underlying http.RoundTripper used to satisfy
	// non-transform requests (those that do not include a URL fragment).
	Transport http.RoundTripper

	// CachingClient is used to fetch images to be resized.  This client is
	// used rather than Transport directly in order to ensure that
	// responses are properly cached.
	CachingClient *http.Client

	// ResponseSize - maximum size of remote image to be be fetched by Client. If image is larger Get(url) will return error
	ResponseSize uint64
}

// RoundTrip implements the http.RoundTripper interface.
func (t *TransformingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if req.URL.Fragment == "" {
		// normal requests pass through
		glog.Infof("fetching remote URL: %v", req.URL)
		return t.Transport.RoundTrip(req)
	}

	u := *req.URL
	u.Fragment = ""
	resp, err := t.CachingClient.Get(u.String())
	if err != nil {
		return nil, err
	}

	contentLength, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
	// no data reading - check first Content-Length
	if uint64(contentLength) > t.ResponseSize {
		return nil, fmt.Errorf("size too large: max size: %d, content-length: %d",t.ResponseSize, contentLength)
	}

	//read data with limiter if there is no Content-Length header or it is fake
	resp.Body = NewLimitedReadCloser(resp.Body, int64(t.ResponseSize))
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	opt := ParseOptions(req.URL.Fragment)

	img, err := Transform(b, opt,u.String())
	if err != nil {
		glog.Errorf("error transforming image: %v, Content-Type: %v, URL: %v", err, resp.Header.Get("Content-Type"),req.URL)
		img = b
	}

	// replay response with transformed image and updated content length
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "%s %s\n", resp.Proto, resp.Status)
	resp.Header.WriteSubset(buf, map[string]bool{"Content-Length": true})
	fmt.Fprintf(buf, "Content-Length: %d\n\n", len(img))
	buf.Write(img)

	return http.ReadResponse(bufio.NewReader(buf), req)
}
<|endoftext|>


<commit_before>package uf

// WQUPC M union-find ops on a N objects
// worst time complexity is o((N+M)*lgN)
type WQUPC struct {
    store []int
    sz    []int
}

func NewWQUPC(n int) *WQUPC {
    st := make([]int, n)
    sz := make([]int, n)
    for i := range st {
        st[i] = i
        sz[i] = 1
    }
    return &WQUPC{store: st, sz: sz}
}

// Find time complexity proportional to depth of i/j, depth is at most lgN
// worst case O(lgN)
func (qu *WQUPC) Find(i, j int) bool {
    return qu.Root(i) == qu.Root(j)
}

// Union just like Find, time complexity Worst Case O(lgN)
func (qu *WQUPC) Union(i, j int) {
    ri, rj := qu.Root(i), qu.Root(j)
    if ri == rj {
        return
    }
    if qu.sz[ri] < qu.sz[rj] {
        qu.store[ri] = rj
        qu.sz[rj] += qu.sz[ri]
    } else {
        qu.store[rj] = ri
        qu.sz[ri] += qu.sz[rj]
    }
}

func (qu *WQUPC) Root(i int) int {
    for qu.store[i] != i {
        // make every other nodes in path points to its grandparent
        qu.store[i] = qu.store[qu.store[i]]
        i = qu.store[i]
    }
    return i
}
<commit_msg>gofmt wqupc.go<commit_after>package uf

// WQUPC M union-find ops on a N objects
// worst time complexity is o((N+M)*lgN)
type WQUPC struct {
	store []int
	sz    []int
}

func NewWQUPC(n int) *WQUPC {
	st := make([]int, n)
	sz := make([]int, n)
	for i := range st {
		st[i] = i
		sz[i] = 1
	}
	return &WQUPC{store: st, sz: sz}
}

// Find time complexity proportional to depth of i/j, depth is at most lgN
// worst case O(lgN)
func (qu *WQUPC) Find(i, j int) bool {
	return qu.Root(i) == qu.Root(j)
}

// Union just like Find, time complexity Worst Case O(lgN)
func (qu *WQUPC) Union(i, j int) {
	ri, rj := qu.Root(i), qu.Root(j)
	if ri == rj {
		return
	}
	if qu.sz[ri] < qu.sz[rj] {
		qu.store[ri] = rj
		qu.sz[rj] += qu.sz[ri]
	} else {
		qu.store[rj] = ri
		qu.sz[ri] += qu.sz[rj]
	}
}

func (qu *WQUPC) Root(i int) int {
	for qu.store[i] != i {
		// make every other nodes in path points to its grandparent
		qu.store[i] = qu.store[qu.store[i]]
		i = qu.store[i]
	}
	return i
}
<|endoftext|>