<commit_before>package couchbase

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"unsafe"
)

// The HTTP Client To Use
var HttpClient = http.DefaultClient

// Size of the connection pools (per host).
var PoolSize = 4

// Number of overflow connections allowed in a pool.
var PoolOverflow = PoolSize

// Auth callback gets the auth username and password for the given
// bucket.
type AuthHandler interface {
	GetCredentials() (string, string)
}

type RestPool struct {
	Name         string `json:"name"`
	StreamingURI string `json:"streamingUri"`
	URI          string `json:"uri"`
}

type Pools struct {
	ComponentsVersion     map[string]string `json:"componentsVersion,omitempty"`
	ImplementationVersion string            `json:"implementationVersion"`
	IsAdmin               bool              `json:"isAdminCreds"`
	UUID                  string            `json:"uuid"`
	Pools                 []RestPool        `json:"pools"`
}

// A computer in a cluster running the couchbase software.
type Node struct {
	ClusterCompatibility int                `json:"clusterCompatibility"`
	ClusterMembership    string             `json:"clusterMembership"`
	CouchAPIBase         string             `json:"couchApiBase"`
	Hostname             string             `json:"hostname"`
	InterestingStats     map[string]float64 `json:"interestingStats,omitempty"`
	MCDMemoryAllocated   float64            `json:"mcdMemoryAllocated"`
	MCDMemoryReserved    float64            `json:"mcdMemoryReserved"`
	MemoryFree           float64            `json:"memoryFree"`
	MemoryTotal          float64            `json:"memoryTotal"`
	OS                   string             `json:"os"`
	Ports                map[string]int     `json:"ports"`
	Status               string             `json:"status"`
	Uptime               int                `json:"uptime,string"`
	Version              string             `json:"version"`
	ThisNode             bool               `json:"thisNode,omitempty"`
}

// A pool of nodes and buckets.
type Pool struct {
	BucketMap map[string]Bucket
	Nodes     []Node

	BucketURL map[string]string `json:"buckets"`

	client Client
}

type VBucketServerMap struct {
	HashAlgorithm string   `json:"hashAlgorithm"`
	NumReplicas   int      `json:"numReplicas"`
	ServerList    []string `json:"serverList"`
	VBucketMap    [][]int  `json:"vBucketMap"`
}

// An individual bucket.  Herein lives the most useful stuff.
type Bucket struct {
	AuthType            string             `json:"authType"`
	Capabilities        []string           `json:"bucketCapabilities"`
	CapabilitiesVersion string             `json:"bucketCapabilitiesVer"`
	Type                string             `json:"bucketType"`
	Name                string             `json:"name"`
	NodeLocator         string             `json:"nodeLocator"`
	Quota               map[string]float64 `json:"quota,omitempty"`
	Replicas            int                `json:"replicaNumber"`
	Password            string             `json:"saslPassword"`
	URI                 string             `json:"uri"`
	StreamingURI        string             `json:"streamingUri"`
	LocalRandomKeyURI   string             `json:"localRandomKeyUri,omitempty"`
	UUID                string             `json:"uuid"`
	DDocs               struct {
		URI string `json:"uri"`
	} `json:"ddocs,omitempty"`
	BasicStats  map[string]interface{} `json:"basicStats,omitempty"`
	Controllers map[string]interface{} `json:"controllers,omitempty"`

	// These are used for JSON IO, but isn't used for processing
	// since it needs to be swapped out safely.
	VBSMJson  VBucketServerMap `json:"vBucketServerMap"`
	NodesJson []Node           `json:"nodes"`

	pool             *Pool
	connPools        unsafe.Pointer // *[]*connectionPool
	vBucketServerMap unsafe.Pointer // *VBucketServerMap
	nodeList         unsafe.Pointer // *[]Node
	commonSufix      string
}

// Get the current vbucket server map
func (b Bucket) VBServerMap() *VBucketServerMap {
	return (*VBucketServerMap)(atomic.LoadPointer(&b.vBucketServerMap))
}

func (b Bucket) Nodes() []Node {
	return *(*[]Node)(atomic.LoadPointer(&b.nodeList))
}

func (b Bucket) getConnPools() []*connectionPool {
	return *(*[]*connectionPool)(atomic.LoadPointer(&b.connPools))
}

func (b *Bucket) replaceConnPools(with []*connectionPool) {
	for {
		old := atomic.LoadPointer(&b.connPools)
		if atomic.CompareAndSwapPointer(&b.connPools, old, unsafe.Pointer(&with)) {
			if old != nil {
				for _, pool := range *(*[]*connectionPool)(old) {
					pool.Close()
				}
			}
			return
		}
	}
}

func (b Bucket) getConnPool(i int) *connectionPool {
	p := b.getConnPools()
	if len(p) > i {
		return p[i]
	}
	return nil
}

func (b Bucket) authHandler() (ah AuthHandler) {
	if b.pool != nil {
		ah = b.pool.client.ah
	}
	if ah == nil {
		ah = &basicAuth{b.Name, ""}
	}
	return
}

// Get the (sorted) list of memcached node addresses (hostname:port).
func (b Bucket) NodeAddresses() []string {
	vsm := b.VBServerMap()
	rv := make([]string, len(vsm.ServerList))
	copy(rv, vsm.ServerList)
	sort.Strings(rv)
	return rv
}

// Get the longest common suffix of all host:port strings in the node list.
func (b Bucket) CommonAddressSuffix() string {
	input := []string{}
	for _, n := range b.Nodes() {
		input = append(input, n.Hostname)
	}
	return FindCommonSuffix(input)
}

// The couchbase client gives access to all the things.
type Client struct {
	BaseURL  *url.URL
	ah       AuthHandler
	Info     Pools
	Statuses [256]uint64
}

func maybeAddAuth(req *http.Request, ah AuthHandler) {
	if ah != nil {
		user, pass := ah.GetCredentials()
		req.Header.Set("Authorization", "Basic "+
			base64.StdEncoding.EncodeToString([]byte(user+":"+pass)))
	}
}

func (c *Client) parseURLResponse(path string, out interface{}) error {
	u := *c.BaseURL
	u.User = nil
	if q := strings.Index(path, "?"); q > 0 {
		u.Path = path[:q]
		u.RawQuery = path[q+1:]
	} else {
		u.Path = path
	}

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}
	maybeAddAuth(req, c.ah)

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		bod, _ := ioutil.ReadAll(io.LimitReader(res.Body, 512))
		return fmt.Errorf("HTTP error %v getting %q: %s",
			res.Status, u.String(), bod)
	}

	d := json.NewDecoder(res.Body)
	if err = d.Decode(&out); err != nil {
		return err
	}
	return nil
}

type basicAuth struct {
	u, p string
}

func (b basicAuth) GetCredentials() (string, string) {
	return b.u, b.p
}

func basicAuthFromURL(us string) (ah AuthHandler) {
	u, err := ParseURL(us)
	if err != nil {
		return
	}
	if user := u.User; user != nil {
		pw, _ := user.Password()
		ah = basicAuth{user.Username(), pw}
	}
	return
}

// ConnectWithAuth connects to a couchbase cluster with the given
// authentication handler.
func ConnectWithAuth(baseU string, ah AuthHandler) (c Client, err error) {
	c.BaseURL, err = ParseURL(baseU)
	if err != nil {
		return
	}
	c.ah = ah

	return c, c.parseURLResponse("/pools", &c.Info)
}

// Connect to a couchbase cluster.  An authentication handler will be
// created from the userinfo in the URL if provided.
func Connect(baseU string) (Client, error) {
	return ConnectWithAuth(baseU, basicAuthFromURL(baseU))
}

func (b *Bucket) refresh() error {
	pool := b.pool
	tmpb := &Bucket{}
	err := pool.client.parseURLResponse(b.URI, tmpb)
	if err != nil {
		return err
	}
	newcps := make([]*connectionPool, len(tmpb.VBSMJson.ServerList))
	for i := range newcps {
		newcps[i] = newConnectionPool(
			tmpb.VBSMJson.ServerList[i],
			b.authHandler(), PoolSize, PoolOverflow)
	}
	b.replaceConnPools(newcps)
	atomic.StorePointer(&b.vBucketServerMap, unsafe.Pointer(&b.VBSMJson))
	atomic.StorePointer(&b.nodeList, unsafe.Pointer(&b.NodesJson))
	return nil
}

func (p *Pool) refresh() (err error) {
	p.BucketMap = make(map[string]Bucket)

	buckets := []Bucket{}
	err = p.client.parseURLResponse(p.BucketURL["uri"], &buckets)
	if err != nil {
		return err
	}
	for _, b := range buckets {
		b.pool = p
		b.nodeList = unsafe.Pointer(&b.NodesJson)
		b.replaceConnPools(make([]*connectionPool, len(b.VBSMJson.ServerList)))

		p.BucketMap[b.Name] = b
	}
	return nil
}

// Get a pool from within the couchbase cluster (usually "default").
func (c *Client) GetPool(name string) (p Pool, err error) {
	var poolURI string
	for _, p := range c.Info.Pools {
		if p.Name == name {
			poolURI = p.URI
		}
	}
	if poolURI == "" {
		return p, errors.New("No pool named " + name)
	}

	err = c.parseURLResponse(poolURI, &p)

	p.client = *c

	err = p.refresh()
	return
}

// Mark this bucket as no longer needed, closing connections it may have open.
func (b *Bucket) Close() {
	if b.connPools != nil {
		for _, c := range b.getConnPools() {
			if c != nil {
				c.Close()
			}
		}
		b.connPools = nil
	}
}

func bucket_finalizer(b *Bucket) {
	if b.connPools != nil {
		log.Printf("Warning: Finalizing a bucket with active connections.")
	}
}

// Get a bucket from within this pool.
func (p *Pool) GetBucket(name string) (*Bucket, error) {
	rv, ok := p.BucketMap[name]
	if !ok {
		return nil, errors.New("No bucket named " + name)
	}
	runtime.SetFinalizer(&rv, bucket_finalizer)
	err := rv.refresh()
	if err != nil {
		return nil, err
	}
	return &rv, nil
}

// Get the pool to which this bucket belongs.
func (b *Bucket) GetPool() *Pool {
	return b.pool
}

// Get the client from which we got this pool.
func (p *Pool) GetClient() *Client {
	return &p.client
}

// Convenience function for getting a named bucket from a URL
func GetBucket(endpoint, poolname, bucketname string) (*Bucket, error) {
	var err error
	client, err := Connect(endpoint)
	if err != nil {
		return nil, err
	}

	pool, err := client.GetPool(poolname)
	if err != nil {
		return nil, err
	}

	return pool.GetBucket(bucketname)
}
<commit_msg>Also use the newly parsed bucket's data for nodelist and vbm<commit_after>package couchbase

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"sort"
	"strings"
	"sync/atomic"
	"unsafe"
)

// The HTTP Client To Use
var HttpClient = http.DefaultClient

// Size of the connection pools (per host).
var PoolSize = 4

// Number of overflow connections allowed in a pool.
var PoolOverflow = PoolSize

// Auth callback gets the auth username and password for the given
// bucket.
type AuthHandler interface {
	GetCredentials() (string, string)
}

type RestPool struct {
	Name         string `json:"name"`
	StreamingURI string `json:"streamingUri"`
	URI          string `json:"uri"`
}

type Pools struct {
	ComponentsVersion     map[string]string `json:"componentsVersion,omitempty"`
	ImplementationVersion string            `json:"implementationVersion"`
	IsAdmin               bool              `json:"isAdminCreds"`
	UUID                  string            `json:"uuid"`
	Pools                 []RestPool        `json:"pools"`
}

// A computer in a cluster running the couchbase software.
type Node struct {
	ClusterCompatibility int                `json:"clusterCompatibility"`
	ClusterMembership    string             `json:"clusterMembership"`
	CouchAPIBase         string             `json:"couchApiBase"`
	Hostname             string             `json:"hostname"`
	InterestingStats     map[string]float64 `json:"interestingStats,omitempty"`
	MCDMemoryAllocated   float64            `json:"mcdMemoryAllocated"`
	MCDMemoryReserved    float64            `json:"mcdMemoryReserved"`
	MemoryFree           float64            `json:"memoryFree"`
	MemoryTotal          float64            `json:"memoryTotal"`
	OS                   string             `json:"os"`
	Ports                map[string]int     `json:"ports"`
	Status               string             `json:"status"`
	Uptime               int                `json:"uptime,string"`
	Version              string             `json:"version"`
	ThisNode             bool               `json:"thisNode,omitempty"`
}

// A pool of nodes and buckets.
type Pool struct {
	BucketMap map[string]Bucket
	Nodes     []Node

	BucketURL map[string]string `json:"buckets"`

	client Client
}

type VBucketServerMap struct {
	HashAlgorithm string   `json:"hashAlgorithm"`
	NumReplicas   int      `json:"numReplicas"`
	ServerList    []string `json:"serverList"`
	VBucketMap    [][]int  `json:"vBucketMap"`
}

// An individual bucket.  Herein lives the most useful stuff.
type Bucket struct {
	AuthType            string             `json:"authType"`
	Capabilities        []string           `json:"bucketCapabilities"`
	CapabilitiesVersion string             `json:"bucketCapabilitiesVer"`
	Type                string             `json:"bucketType"`
	Name                string             `json:"name"`
	NodeLocator         string             `json:"nodeLocator"`
	Quota               map[string]float64 `json:"quota,omitempty"`
	Replicas            int                `json:"replicaNumber"`
	Password            string             `json:"saslPassword"`
	URI                 string             `json:"uri"`
	StreamingURI        string             `json:"streamingUri"`
	LocalRandomKeyURI   string             `json:"localRandomKeyUri,omitempty"`
	UUID                string             `json:"uuid"`
	DDocs               struct {
		URI string `json:"uri"`
	} `json:"ddocs,omitempty"`
	BasicStats  map[string]interface{} `json:"basicStats,omitempty"`
	Controllers map[string]interface{} `json:"controllers,omitempty"`

	// These are used for JSON IO, but isn't used for processing
	// since it needs to be swapped out safely.
	VBSMJson  VBucketServerMap `json:"vBucketServerMap"`
	NodesJson []Node           `json:"nodes"`

	pool             *Pool
	connPools        unsafe.Pointer // *[]*connectionPool
	vBucketServerMap unsafe.Pointer // *VBucketServerMap
	nodeList         unsafe.Pointer // *[]Node
	commonSufix      string
}

// Get the current vbucket server map
func (b Bucket) VBServerMap() *VBucketServerMap {
	return (*VBucketServerMap)(atomic.LoadPointer(&b.vBucketServerMap))
}

func (b Bucket) Nodes() []Node {
	return *(*[]Node)(atomic.LoadPointer(&b.nodeList))
}

func (b Bucket) getConnPools() []*connectionPool {
	return *(*[]*connectionPool)(atomic.LoadPointer(&b.connPools))
}

func (b *Bucket) replaceConnPools(with []*connectionPool) {
	for {
		old := atomic.LoadPointer(&b.connPools)
		if atomic.CompareAndSwapPointer(&b.connPools, old, unsafe.Pointer(&with)) {
			if old != nil {
				for _, pool := range *(*[]*connectionPool)(old) {
					pool.Close()
				}
			}
			return
		}
	}
}

func (b Bucket) getConnPool(i int) *connectionPool {
	p := b.getConnPools()
	if len(p) > i {
		return p[i]
	}
	return nil
}

func (b Bucket) authHandler() (ah AuthHandler) {
	if b.pool != nil {
		ah = b.pool.client.ah
	}
	if ah == nil {
		ah = &basicAuth{b.Name, ""}
	}
	return
}

// Get the (sorted) list of memcached node addresses (hostname:port).
func (b Bucket) NodeAddresses() []string {
	vsm := b.VBServerMap()
	rv := make([]string, len(vsm.ServerList))
	copy(rv, vsm.ServerList)
	sort.Strings(rv)
	return rv
}

// Get the longest common suffix of all host:port strings in the node list.
func (b Bucket) CommonAddressSuffix() string {
	input := []string{}
	for _, n := range b.Nodes() {
		input = append(input, n.Hostname)
	}
	return FindCommonSuffix(input)
}

// The couchbase client gives access to all the things.
type Client struct {
	BaseURL  *url.URL
	ah       AuthHandler
	Info     Pools
	Statuses [256]uint64
}

func maybeAddAuth(req *http.Request, ah AuthHandler) {
	if ah != nil {
		user, pass := ah.GetCredentials()
		req.Header.Set("Authorization", "Basic "+
			base64.StdEncoding.EncodeToString([]byte(user+":"+pass)))
	}
}

func (c *Client) parseURLResponse(path string, out interface{}) error {
	u := *c.BaseURL
	u.User = nil
	if q := strings.Index(path, "?"); q > 0 {
		u.Path = path[:q]
		u.RawQuery = path[q+1:]
	} else {
		u.Path = path
	}

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}
	maybeAddAuth(req, c.ah)

	res, err := HttpClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		bod, _ := ioutil.ReadAll(io.LimitReader(res.Body, 512))
		return fmt.Errorf("HTTP error %v getting %q: %s",
			res.Status, u.String(), bod)
	}

	d := json.NewDecoder(res.Body)
	if err = d.Decode(&out); err != nil {
		return err
	}
	return nil
}

type basicAuth struct {
	u, p string
}

func (b basicAuth) GetCredentials() (string, string) {
	return b.u, b.p
}

func basicAuthFromURL(us string) (ah AuthHandler) {
	u, err := ParseURL(us)
	if err != nil {
		return
	}
	if user := u.User; user != nil {
		pw, _ := user.Password()
		ah = basicAuth{user.Username(), pw}
	}
	return
}

// ConnectWithAuth connects to a couchbase cluster with the given
// authentication handler.
func ConnectWithAuth(baseU string, ah AuthHandler) (c Client, err error) {
	c.BaseURL, err = ParseURL(baseU)
	if err != nil {
		return
	}
	c.ah = ah

	return c, c.parseURLResponse("/pools", &c.Info)
}

// Connect to a couchbase cluster.  An authentication handler will be
// created from the userinfo in the URL if provided.
func Connect(baseU string) (Client, error) {
	return ConnectWithAuth(baseU, basicAuthFromURL(baseU))
}

func (b *Bucket) refresh() error {
	pool := b.pool
	tmpb := &Bucket{}
	err := pool.client.parseURLResponse(b.URI, tmpb)
	if err != nil {
		return err
	}
	newcps := make([]*connectionPool, len(tmpb.VBSMJson.ServerList))
	for i := range newcps {
		newcps[i] = newConnectionPool(
			tmpb.VBSMJson.ServerList[i],
			b.authHandler(), PoolSize, PoolOverflow)
	}
	b.replaceConnPools(newcps)
	atomic.StorePointer(&b.vBucketServerMap, unsafe.Pointer(&tmpb.VBSMJson))
	atomic.StorePointer(&b.nodeList, unsafe.Pointer(&tmpb.NodesJson))
	return nil
}

func (p *Pool) refresh() (err error) {
	p.BucketMap = make(map[string]Bucket)

	buckets := []Bucket{}
	err = p.client.parseURLResponse(p.BucketURL["uri"], &buckets)
	if err != nil {
		return err
	}
	for _, b := range buckets {
		b.pool = p
		b.nodeList = unsafe.Pointer(&b.NodesJson)
		b.replaceConnPools(make([]*connectionPool, len(b.VBSMJson.ServerList)))

		p.BucketMap[b.Name] = b
	}
	return nil
}

// Get a pool from within the couchbase cluster (usually "default").
func (c *Client) GetPool(name string) (p Pool, err error) {
	var poolURI string
	for _, p := range c.Info.Pools {
		if p.Name == name {
			poolURI = p.URI
		}
	}
	if poolURI == "" {
		return p, errors.New("No pool named " + name)
	}

	err = c.parseURLResponse(poolURI, &p)

	p.client = *c

	err = p.refresh()
	return
}

// Mark this bucket as no longer needed, closing connections it may have open.
func (b *Bucket) Close() {
	if b.connPools != nil {
		for _, c := range b.getConnPools() {
			if c != nil {
				c.Close()
			}
		}
		b.connPools = nil
	}
}

func bucket_finalizer(b *Bucket) {
	if b.connPools != nil {
		log.Printf("Warning: Finalizing a bucket with active connections.")
	}
}

// Get a bucket from within this pool.
func (p *Pool) GetBucket(name string) (*Bucket, error) {
	rv, ok := p.BucketMap[name]
	if !ok {
		return nil, errors.New("No bucket named " + name)
	}
	runtime.SetFinalizer(&rv, bucket_finalizer)
	err := rv.refresh()
	if err != nil {
		return nil, err
	}
	return &rv, nil
}

// Get the pool to which this bucket belongs.
func (b *Bucket) GetPool() *Pool {
	return b.pool
}

// Get the client from which we got this pool.
func (p *Pool) GetClient() *Client {
	return &p.client
}

// Convenience function for getting a named bucket from a URL
func GetBucket(endpoint, poolname, bucketname string) (*Bucket, error) {
	var err error
	client, err := Connect(endpoint)
	if err != nil {
		return nil, err
	}

	pool, err := client.GetPool(poolname)
	if err != nil {
		return nil, err
	}

	return pool.GetBucket(bucketname)
}
<|endoftext|>


<commit_before>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PostsService is the service for accessing Post-related calls from the
// Pinboard API.
type PostsService struct {
	client *Client
}

const (
	timeLayoutFull  = "2006-01-02T15:04:05Z"
	timeLayoutShort = "2006-01-02"
)

// Post represents a post stored in Pinboard. Fields are transformed from the
// actual response to be a bit more sane. For example, description from the
// response is renamed to Title and the extended field is renamed to
// Description.
type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
	Time        *time.Time
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	dt, _ := time.Parse(timeLayoutFull, presp.Time)

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
		Time:        &dt,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
	Time        string `xml:"time,attr"`
}

//
type Date struct {
	Count int
	Date  *time.Time
}

func newDateFromPostResp(dresp *dateResp) (*Date, error) {
	dt, err := time.Parse(timeLayoutShort, dresp.Date)
	if err != nil {
		return nil, err
	}
	c, err := strconv.Atoi(dresp.Count)
	if err != nil {
		return nil, err
	}

	return &Date{
		Count: c,
		Date:  &dt,
	}, nil
}

type dateResp struct {
	Count string `xml:"count,attr"`
	Date  string `xml:"date,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.Format(timeLayoutFull)
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account where
// urlStr is the URL of the Post to delete.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Get returns one or more posts on a single day matching the arguments.
// If no date or url is given, date of most recent bookmark will be used.
//
// https://pinboard.in/api#posts_get
func (s *PostsService) Get(tags []string, creationTime *time.Time, urlStr string) ([]*Post, *http.Response, error) {

	params := &url.Values{}

	if creationTime != nil {
		params.Add("dt", creationTime.Format(timeLayoutFull))
	}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	if len(urlStr) > 0 {
		params.Add("url", urlStr)
	}

	req, err := s.client.NewRequest("posts/get", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// Returns the most recent time a bookmark was added, updated or deleted.
// Use this before calling posts/all to see if the data has changed since the last fetch.
//
// https://pinboard.in/api#posts_update
func (s *PostsService) LastTimeUpdated() (*time.Time, *http.Response, error) {
	req, err := s.client.NewRequest("posts/update", &url.Values{})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Time string `xml:"time,attr"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	updated, err := time.Parse(timeLayoutFull, result.Time)
	if err != nil {
		return nil, resp, err
	}

	return &updated, resp, nil
}

// Returns a list of dates with the number of posts at each date.
//
// https://pinboard.in/api#posts_dates
func (s *PostsService) Dates(tags []string) ([]*Date, *http.Response, error) {
	params := &url.Values{}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	req, err := s.client.NewRequest("posts/dates", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Dates []*dateResp `xml:"date"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	dates := make([]*Date, len(result.Dates))
	for i, v := range result.Dates {
		d, err := newDateFromPostResp(v)
		if err != nil {
			return nil, nil, err
		}
		dates[i] = d
	}

	return dates, resp, nil
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// All fetches all bookmarks in the user's account.
//
// https://pinboard.in/api#posts_all
func (s *PostsService) All(tags []string, start int, results int, fromdt, todt *time.Time) ([]*Post,
	*http.Response, error) {

	params := &url.Values{}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	if start > 0 {
		params.Add("start", strconv.Itoa(start))
	}

	if results > 0 {
		params.Add("results", strconv.Itoa(results))
	}

	if fromdt != nil {
		params.Add("fromdt", fromdt.Format(timeLayoutFull))
	}

	if todt != nil {
		params.Add("todt", todt.Format(timeLayoutFull))
	}

	req, err := s.client.NewRequest("posts/all", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil

}

// TODO
//
// https://pinboard.in/api#posts_suggest
func (s *PostsService) Suggest() ([]string, []string, *http.Response, error) {
	req, err := s.client.NewRequest("posts/suggest", nil)
	if err != nil {
		return nil, nil, nil, err
	}

	var result struct {
		Popular     []string `xml:"popular"`
		Recommended []string `xml:"recommended"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, nil, resp, err
	}
	return result.Popular, result.Recommended, resp, nil
}
<commit_msg>Replaced comment<commit_after>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PostsService is the service for accessing Post-related calls from the
// Pinboard API.
type PostsService struct {
	client *Client
}

const (
	timeLayoutFull  = "2006-01-02T15:04:05Z"
	timeLayoutShort = "2006-01-02"
)

// Post represents a post stored in Pinboard. Fields are transformed from the
// actual response to be a bit more sane. For example, description from the
// response is renamed to Title and the extended field is renamed to
// Description.
type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
	Time        *time.Time
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	dt, _ := time.Parse(timeLayoutFull, presp.Time)

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
		Time:        &dt,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
	Time        string `xml:"time,attr"`
}

//
type Date struct {
	Count int
	Date  *time.Time
}

func newDateFromPostResp(dresp *dateResp) (*Date, error) {
	dt, err := time.Parse(timeLayoutShort, dresp.Date)
	if err != nil {
		return nil, err
	}
	c, err := strconv.Atoi(dresp.Count)
	if err != nil {
		return nil, err
	}

	return &Date{
		Count: c,
		Date:  &dt,
	}, nil
}

type dateResp struct {
	Count string `xml:"count,attr"`
	Date  string `xml:"date,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.Format(timeLayoutFull)
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account where
// urlStr is the URL of the Post to delete.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Get returns one or more posts on a single day matching the arguments.
// If no date or url is given, date of most recent bookmark will be used.
//
// https://pinboard.in/api#posts_get
func (s *PostsService) Get(tags []string, creationTime *time.Time, urlStr string) ([]*Post, *http.Response, error) {

	params := &url.Values{}

	if creationTime != nil {
		params.Add("dt", creationTime.Format(timeLayoutFull))
	}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	if len(urlStr) > 0 {
		params.Add("url", urlStr)
	}

	req, err := s.client.NewRequest("posts/get", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// Returns the most recent time a bookmark was added, updated or deleted.
// Use this before calling posts/all to see if the data has changed since the last fetch.
//
// https://pinboard.in/api#posts_update
func (s *PostsService) LastTimeUpdated() (*time.Time, *http.Response, error) {
	req, err := s.client.NewRequest("posts/update", &url.Values{})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Time string `xml:"time,attr"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	updated, err := time.Parse(timeLayoutFull, result.Time)
	if err != nil {
		return nil, resp, err
	}

	return &updated, resp, nil
}

// Returns a list of dates with the number of posts at each date.
//
// https://pinboard.in/api#posts_dates
func (s *PostsService) Dates(tags []string) ([]*Date, *http.Response, error) {
	params := &url.Values{}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	req, err := s.client.NewRequest("posts/dates", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Dates []*dateResp `xml:"date"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	dates := make([]*Date, len(result.Dates))
	for i, v := range result.Dates {
		d, err := newDateFromPostResp(v)
		if err != nil {
			return nil, nil, err
		}
		dates[i] = d
	}

	return dates, resp, nil
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// All fetches all bookmarks in the user's account.
//
// https://pinboard.in/api#posts_all
func (s *PostsService) All(tags []string, start int, results int, fromdt, todt *time.Time) ([]*Post,
	*http.Response, error) {

	params := &url.Values{}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil && len(tags) > 0 {
		params.Add("tags", strings.Join(tags, " "))
	}

	if start > 0 {
		params.Add("start", strconv.Itoa(start))
	}

	if results > 0 {
		params.Add("results", strconv.Itoa(results))
	}

	if fromdt != nil {
		params.Add("fromdt", fromdt.Format(timeLayoutFull))
	}

	if todt != nil {
		params.Add("todt", todt.Format(timeLayoutFull))
	}

	req, err := s.client.NewRequest("posts/all", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil

}

// Returns a list of popular tags and recommended tags for a given URL.
// Popular tags are tags used site-wide for the url; recommended tags are drawn from the user's own tags.
//
// https://pinboard.in/api#posts_suggest
func (s *PostsService) Suggest() ([]string, []string, *http.Response, error) {
	req, err := s.client.NewRequest("posts/suggest", nil)
	if err != nil {
		return nil, nil, nil, err
	}

	var result struct {
		Popular     []string `xml:"popular"`
		Recommended []string `xml:"recommended"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, nil, resp, err
	}
	return result.Popular, result.Recommended, resp, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"fmt"
	"bytes"
	"encoding/binary"
	"compress/zlib"
)

const (
	protocolVersion = 2

	// Packet type IDs
	packetIDLogin              = 0x1
	packetIDHandshake          = 0x2
	packetIDPlayerInventory    = 0x5
	packetIDSpawnPosition      = 0x6
	packetIDPlayerPositionLook = 0xd
	packetIDPreChunk           = 0x32
	packetIDMapChunk           = 0x33

	// Inventory types
	inventoryTypeMain     = -1
	inventoryTypeArmor    = -2
	inventoryTypeCrafting = -3
)

func ReadByte(reader io.Reader) (b byte, err os.Error) {
	err = binary.Read(reader, binary.BigEndian, &b)
	return
}

func WriteByte(writer io.Writer, b byte) (err os.Error) {
	return binary.Write(writer, binary.BigEndian, b)
}

func WriteBool(writer io.Writer, b bool) (err os.Error) {
	var val byte

	if b {
		val = 1
	} else {
		val = 0
	}

	return WriteByte(writer, val)
}

func ReadInt16(reader io.Reader) (i int16, err os.Error) {
	err = binary.Read(reader, binary.BigEndian, &i)
	return
}

func WriteInt16(writer io.Writer, i int16) (err os.Error) {
	return binary.Write(writer, binary.BigEndian, i)
}

func ReadInt32(reader io.Reader) (i int32, err os.Error) {
	err = binary.Read(reader, binary.BigEndian, &i)
	return
}

func WriteInt32(writer io.Writer, i int32) (err os.Error) {
	return binary.Write(writer, binary.BigEndian, i)
}

func WriteFloat32(writer io.Writer, f float32) (err os.Error) {
	return binary.Write(writer, binary.BigEndian, f)
}

func WriteFloat64(writer io.Writer, f float64) (err os.Error) {
	return binary.Write(writer, binary.BigEndian, f)
}

func ReadString(reader io.Reader) (s string, err os.Error) {
	n, e := ReadInt16(reader)
	if e != nil {
		return "", e
	}

	bs := make([]byte, uint16(n))
	_, err = io.ReadFull(reader, bs)
	return string(bs), err
}

func WriteString(writer io.Writer, s string) (err os.Error) {
	bs := []byte(s)

	err = WriteInt16(writer, int16(len(bs)))
	if err != nil {
		return err
	}

	_, err = writer.Write(bs)
	return err
}

func ReadHandshake(reader io.Reader) (username string, err os.Error) {
	packetID, e := ReadByte(reader)
	if e != nil {
		return "", e
	}
	if packetID != packetIDHandshake {
		panic(fmt.Sprintf("ReadHandshake: invalid packet ID %#x", packetID))
	}

	return ReadString(reader)
}

func WriteHandshake(writer io.Writer, reply string) (err os.Error) {
	err = WriteByte(writer, packetIDHandshake)
	if err != nil {
		return
	}

	return WriteString(writer, reply)
}

func ReadLogin(reader io.Reader) (username, password string, err os.Error) {
	packetID, e := ReadByte(reader)
	if e != nil {
		return "", "", e
	}
	if packetID != packetIDLogin {
		panic(fmt.Sprintf("ReadLogin: invalid packet ID %#x", packetID))
	}

	version, e2 := ReadInt32(reader)
	if e2 != nil {
		return "", "", e2
	}
	if version != protocolVersion {
		panic(fmt.Sprintf("ReadLogin: unsupported protocol version %#x", version))
	}

	username, e3 := ReadString(reader)
	if e3 != nil {
		return "", "", e3
	}

	password, e4 := ReadString(reader)
	if e4 != nil {
		return "", "", e4
	}

	return username, password, nil
}

func WriteLogin(writer io.Writer) (err os.Error) {
	_, err = writer.Write([]byte{packetIDLogin, 0, 0, 0, 0, 0, 0, 0, 0})
	return err
}

func WriteSpawnPosition(writer io.Writer, position *XYZ) (err os.Error) {
	err = WriteByte(writer, packetIDSpawnPosition)
	if err != nil {
		return
	}

	err = WriteInt32(writer, int32(position.x))
	if err != nil {
		return
	}

	err = WriteInt32(writer, int32(position.y))
	if err != nil {
		return
	}

	err = WriteInt32(writer, int32(position.z))
	return
}

func WritePlayerInventory(writer io.Writer) (err os.Error) {
	type InventoryType struct {
		inventoryType int32
		count         int16
	}
	var inventories = []InventoryType{
		InventoryType{inventoryTypeMain, 36},
		InventoryType{inventoryTypeArmor, 4},
		InventoryType{inventoryTypeCrafting, 4},
	}

	for _, inventory := range inventories {
		err = WriteByte(writer, packetIDPlayerInventory)
		if err != nil {
			return
		}

		err = WriteInt32(writer, inventory.inventoryType)
		if err != nil {
			return
		}

		err = WriteInt16(writer, inventory.count)
		if err != nil {
			return
		}

		for i := int16(0); i < inventory.count; i++ {
			err = WriteInt16(writer, -1)
			if err != nil {
				return
			}
		}
	}
	return
}

func WritePlayerPositionLook(writer io.Writer, position *XYZ, orientation *Orientation, stance float64, flying bool) (err os.Error) {
	err = WriteByte(writer, packetIDPlayerPositionLook)
	if err != nil {
		return
	}

	err = WriteFloat64(writer, position.x)
	if err != nil {
		return
	}

	err = WriteFloat64(writer, position.y)
	if err != nil {
		return
	}

	err = WriteFloat64(writer, stance)
	if err != nil {
		return
	}

	err = WriteFloat64(writer, position.z)
	if err != nil {
		return
	}

	err = WriteFloat32(writer, orientation.rotation)
	if err != nil {
		return
	}

	err = WriteFloat32(writer, orientation.pitch)
	if err != nil {
		return
	}

	err = WriteBool(writer, flying)
	return
}

func WritePreChunk(writer io.Writer, x int32, z int32, willSend bool) (err os.Error) {
	err = WriteByte(writer, packetIDPreChunk)
	if err != nil {
		return
	}

	err = WriteInt32(writer, x)
	if err != nil {
		return
	}

	err = WriteInt32(writer, z)
	if err != nil {
		return
	}

	err = WriteBool(writer, willSend)
	return
}

func WriteMapChunk(writer io.Writer, chunk *Chunk) (err os.Error) {
	err = WriteByte(writer, packetIDMapChunk)
	if err != nil {
		return
	}

	err = WriteInt32(writer, chunk.x)
	if err != nil {
		return
	}

	err = WriteInt16(writer, 0)
	if err != nil {
		return
	}

	err = WriteInt32(writer, chunk.z)
	if err != nil {
		return
	}

	err = WriteByte(writer, byte(ChunkSizeX - 1))
	if err != nil {
		return
	}

	err = WriteByte(writer, byte(ChunkSizeY - 1))
	if err != nil {
		return
	}

	err = WriteByte(writer, byte(ChunkSizeZ - 1))
	if err != nil {
		return
	}

	buf := &bytes.Buffer{}
	compressed, err := zlib.NewWriter(buf)
	if err != nil {
		return
	}

	compressed.Write(chunk.blocks)
	compressed.Write(chunk.blockData)
	compressed.Write(chunk.blockLight)
	compressed.Write(chunk.skyLight)
	compressed.Close()
	bs := buf.Bytes()

	err = WriteInt32(writer, int32(len(bs)))
	if err != nil {
		return
	}

	_, err = writer.Write(bs)
	return
}
<commit_msg>proto: Use encoding/binary package directly<commit_after>package main

import (
	"io"
	"os"
	"fmt"
	"bytes"
	"encoding/binary"
	"compress/zlib"
)

const (
	protocolVersion = 2

	// Packet type IDs
	packetIDLogin              = 0x1
	packetIDHandshake          = 0x2
	packetIDPlayerInventory    = 0x5
	packetIDSpawnPosition      = 0x6
	packetIDPlayerPositionLook = 0xd
	packetIDPreChunk           = 0x32
	packetIDMapChunk           = 0x33

	// Inventory types
	inventoryTypeMain     = -1
	inventoryTypeArmor    = -2
	inventoryTypeCrafting = -3
)

func boolToByte(b bool) byte {
	if b {
		return 1
	}
	return 0
}

func byteToBool(b byte) bool {
	return b != 0
}

func ReadString(reader io.Reader) (s string, err os.Error) {
	var length int16
	err = binary.Read(reader, binary.BigEndian, &length)
	if err != nil {
		return
	}

	bs := make([]byte, uint16(length))
	_, err = io.ReadFull(reader, bs)
	return string(bs), err
}

func WriteString(writer io.Writer, s string) (err os.Error) {
	bs := []byte(s)

	err = binary.Write(writer, binary.BigEndian, int16(len(bs)))
	if err != nil {
		return
	}

	_, err = writer.Write(bs)
	return
}

func ReadHandshake(reader io.Reader) (username string, err os.Error) {
	var packetID byte
	err = binary.Read(reader, binary.BigEndian, &packetID)
	if err != nil {
		return
	}
	if packetID != packetIDHandshake {
		panic(fmt.Sprintf("ReadHandshake: invalid packet ID %#x", packetID))
	}

	return ReadString(reader)
}

func WriteHandshake(writer io.Writer, reply string) (err os.Error) {
	err = binary.Write(writer, binary.BigEndian, byte(packetIDHandshake))
	if err != nil {
		return
	}

	return WriteString(writer, reply)
}

func ReadLogin(reader io.Reader) (username, password string, err os.Error) {
	var packet struct {
		PacketID byte
		Version int32
	}

	err = binary.Read(reader, binary.BigEndian, &packet)
	if err != nil {
		return
	}
	if packet.PacketID != packetIDLogin {
		panic(fmt.Sprintf("ReadLogin: invalid packet ID %#x", packet.PacketID))
	}
	if packet.Version != protocolVersion {
		panic(fmt.Sprintf("ReadLogin: unsupported protocol version %#x", packet.Version))
	}

	username, err = ReadString(reader)
	if err != nil {
		return
	}

	password, err = ReadString(reader)
	return
}

func WriteLogin(writer io.Writer) (err os.Error) {
	_, err = writer.Write([]byte{packetIDLogin, 0, 0, 0, 0, 0, 0, 0, 0})
	return err
}

func WriteSpawnPosition(writer io.Writer, position *XYZ) (err os.Error) {
	var packet = struct {
		PacketID byte
		X int32
		Y int32
		Z int32
	}{
		packetIDSpawnPosition,
		int32(position.x),
		int32(position.y),
		int32(position.z),
	}
	err = binary.Write(writer, binary.BigEndian, &packet)
	return
}

func WritePlayerInventory(writer io.Writer) (err os.Error) {
	type InventoryType struct {
		inventoryType int32
		count         int16
	}
	var inventories = []InventoryType{
		InventoryType{inventoryTypeMain, 36},
		InventoryType{inventoryTypeArmor, 4},
		InventoryType{inventoryTypeCrafting, 4},
	}

	for _, inventory := range inventories {
		var packet = struct {
			PacketID byte
			InventoryType int32
			Count int16
		}{
			packetIDPlayerInventory,
			inventory.inventoryType,
			inventory.count,
		}
		err = binary.Write(writer, binary.BigEndian, &packet)
		if err != nil {
			return
		}

		for i := int16(0); i < inventory.count; i++ {
			err = binary.Write(writer, binary.BigEndian, int16(-1))
			if err != nil {
				return
			}
		}
	}
	return
}

func WritePlayerPositionLook(writer io.Writer, position *XYZ, orientation *Orientation, stance float64, flying bool) (err os.Error) {
	var packet = struct {
		PacketID byte
		X float64
		Y float64
		Stance float64
		Z float64
		Rotation float32
		Pitch float32
		Flying byte
	}{
		packetIDPlayerPositionLook,
		position.x,
		position.y,
		stance,
		position.z,
		orientation.rotation,
		orientation.pitch,
		boolToByte(flying),
	}
	err = binary.Write(writer, binary.BigEndian, &packet)
	return
}

func WritePreChunk(writer io.Writer, x int32, z int32, willSend bool) (err os.Error) {
	var packet = struct {
		PacketID byte
		X int32
		Z int32
		WillSend byte
	}{
		packetIDPreChunk,
		x,
		z,
		boolToByte(willSend),
	}
	err = binary.Write(writer, binary.BigEndian, &packet)
	return
}

func WriteMapChunk(writer io.Writer, chunk *Chunk) (err os.Error) {
	buf := &bytes.Buffer{}
	compressed, err := zlib.NewWriter(buf)
	if err != nil {
		return
	}

	compressed.Write(chunk.blocks)
	compressed.Write(chunk.blockData)
	compressed.Write(chunk.blockLight)
	compressed.Write(chunk.skyLight)
	compressed.Close()
	bs := buf.Bytes()

	var packet = struct {
		PacketID byte
		X int32
		Y int16
		Z int32
		SizeX byte
		SizeY byte
		SizeZ byte
		CompressedLength int32
		Compressed []byte
	}{
		packetIDMapChunk,
		chunk.x,
		0,
		chunk.z,
		ChunkSizeX - 1,
		ChunkSizeY - 1,
		ChunkSizeZ - 1,
		int32(len(bs)),
		bs,
	}

	err = binary.Write(writer, binary.BigEndian, &packet)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"net/http"
	"net/http/httputil"
	"path"
	"stathat.com/c/consistent"
	"strings"
)

// Builds the X-Backends header with a chain of nearest 3 backends on the
// consistent hash ring.
func BackendHeader(c *consistent.Consistent, name string) string {
	backends, _ := c.GetN(name, 3)
	// Need to shuffle these for reliable distribution of cache objects
	return strings.Join(backends, ",")
}

// Reverse proxy that selects the backend by nearest match to the request URL
// on the consistent hash ring.
func CacheProxy(c *consistent.Consistent) *httputil.ReverseProxy {
	return &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			backends, _ := c.GetN(req.URL.Path, 3)
			req.URL.Host = backends[0]
			req.Header.Set("X-Backends", strings.Join(backends, ","))
		},
		Transport: &http.Transport{},
	}
}

func CacheProxyManager(c *consistent.Consistent) *http.ServeMux {
	manager := http.NewServeMux()

	// Returns a list of all current consistent hash ring members
	manager.HandleFunc("/members", func(w http.ResponseWriter, req *http.Request) {
		io.WriteString(w, strings.Join(c.Members(), ","))
	})

	// Add a consistent hash ring member
	manager.HandleFunc("/add/", func(w http.ResponseWriter, req *http.Request) {
		// Path handling could be a bit more robust
		c.Add(path.Base(req.URL.Path))
		http.Redirect(w, req, "/members", 302)
	})

	// Add a consistent hash ring member
	manager.HandleFunc("/remove/", func(w http.ResponseWriter, req *http.Request) {
		// Path handling could be a bit more robust
		c.Remove(path.Base(req.URL.Path))
		http.Redirect(w, req, "/members", 302)
	})

	return manager
}

func main() {
	ring := consistent.New()

	// Replace with runtime arguments for initialization
	ring.Add("localhost:9091")
	ring.Add("localhost:9092")
	ring.Add("localhost:9093")
	ring.Add("localhost:9094")
	ring.Add("localhost:9095")

	// Initialize and run manager server in background via goroutine
	manager := CacheProxyManager(ring)
	go http.ListenAndServe(":9190", manager)

	proxy := CacheProxy(ring)
	http.ListenAndServe(":9090", proxy)
}
<commit_msg>RESTful management interface<commit_after>package main

import (
	"io"
	"net/http"
	"net/http/httputil"
	"path"
	"stathat.com/c/consistent"
	"strings"
)

// Builds the X-Backends header with a chain of nearest 3 backends on the
// consistent hash ring.
func BackendHeader(c *consistent.Consistent, name string) string {
	backends, _ := c.GetN(name, 3)
	// Need to shuffle these for reliable distribution of cache objects
	return strings.Join(backends, ",")
}

// Reverse proxy that selects the backend by nearest match to the request URL
// on the consistent hash ring.
func CacheProxy(c *consistent.Consistent) *httputil.ReverseProxy {
	return &httputil.ReverseProxy{
		Director: func(req *http.Request) {
			req.URL.Scheme = "http"
			backends, _ := c.GetN(req.URL.Path, 3)
			req.URL.Host = backends[0]
			req.Header.Set("X-Backends", strings.Join(backends, ","))
		},
		Transport: &http.Transport{},
	}
}

func CacheProxyManager(c *consistent.Consistent) *http.ServeMux {
	manager := http.NewServeMux()

	// RESTful interface for cache members
	// GET /members/
	// 	  Returns a list of current cache members

	// PUT /members/member[:port]
	//    Adds "member" as a cache member

	// DELETE /members/member[:port]
	//    Delets "member" from cache ring

	manager.HandleFunc("/members/", func(w http.ResponseWriter, req *http.Request) {
		switch {
		case req.Method == "PUT":
			c.Add(path.Base(req.URL.Path))
		case req.Method == "DELETE":
			c.Remove(path.Base(req.URL.Path))
		}
		io.WriteString(w, strings.Join(c.Members(), ","))
	})

	return manager
}

func main() {
	ring := consistent.New()

	// Replace with runtime arguments for initialization
	ring.Add("localhost:9091")
	ring.Add("localhost:9092")
	ring.Add("localhost:9093")
	ring.Add("localhost:9094")
	ring.Add("localhost:9095")

	// Initialize and run manager server in background via goroutine
	manager := CacheProxyManager(ring)
	go http.ListenAndServe(":9190", manager)

	proxy := CacheProxy(ring)
	http.ListenAndServe(":9090", proxy)
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"
	"io"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

const (
	VcapBackendHeader = "X-Vcap-Backend"
	VcapRouterHeader  = "X-Vcap-Router"
	VcapTraceHeader   = "X-Vcap-Trace"

	VcapCookieId    = "__VCAP_ID__"
	StickyCookieKey = "JSESSIONID"
)

type registerMessage struct {
	Host string            `json:"host"`
	Port uint16            `json:"port"`
	Uris []string          `json:"uris"`
	Tags map[string]string `json:"tags"`
	Dea  string            `json:"dea"`
	App  string            `json:"app"`

	Sticky string
}

func (r *registerMessage) HostPort() string {
	return fmt.Sprintf("%s:%d", r.Host, r.Port)
}

type Proxy struct {
	sync.Mutex

	r          map[string][]*registerMessage
	d          map[string]int
	varz       *Varz
	se         *SessionEncoder
	activeApps *AppList
}

func NewProxy(se *SessionEncoder, activeApps *AppList, varz *Varz) *Proxy {
	p := new(Proxy)

	p.r = make(map[string][]*registerMessage)
	p.d = make(map[string]int)

	p.se = se
	p.varz = varz
	p.activeApps = activeApps

	return p
}

func (p *Proxy) Register(m *registerMessage) {
	p.Lock()
	defer p.Unlock()

	// Store droplet in registry
	for _, uri := range m.Uris {
		uri = strings.ToLower(uri)
		s := p.r[uri]
		if s == nil {
			s = make([]*registerMessage, 0)
		}

		p.varz.RegisterApp(uri)

		exist := false
		for _, d := range s {
			if d.Host == m.Host && d.Port == m.Port {
				exist = true
				break
			}
		}

		if !exist {
			s = append(s, m)
			p.r[uri] = s
			p.d[m.HostPort()]++
		}
	}

	p.updateStatus()
}

func (p *Proxy) Unregister(m *registerMessage) {
	p.Lock()
	defer p.Unlock()

	hp := m.HostPort()

	// Delete droplets from registry
	for _, uri := range m.Uris {
		s := p.r[uri]
		if s == nil {
			continue
		}

		exist := false
		for i, d := range s {
			if d.Host == m.Host && d.Port == m.Port {
				s[i] = s[len(s)-1]
				exist = true
				break
			}
		}

		if exist {
			s = s[:len(s)-1]
			if len(s) == 0 {
				p.varz.UnregisterApp(uri)
				delete(p.r, uri)
			} else {
				p.r[uri] = s
			}

			p.d[hp]--
			if p.d[hp] == 0 {
				delete(p.d, hp)
			}
		}
	}

	p.updateStatus()
}

func (p *Proxy) updateStatus() {
	p.varz.Urls = len(p.r)
	p.varz.Droplets = len(p.d)
}

func (p *Proxy) lookup(req *http.Request) []*registerMessage {
	url := getUrl(req)

	return p.r[url]
}

func (p *Proxy) Lookup(req *http.Request) *registerMessage {
	p.Lock()
	defer p.Unlock()

	s := p.lookup(req)
	if s == nil {
		return nil
	}

	// If there's only one backend, choose that
	if len(s) == 1 {
		return s[0]
	}

	// Choose backend depending on sticky session
	var sticky string
	for _, v := range req.Cookies() {
		if v.Name == VcapCookieId {
			sticky = v.Value
			break
		}
	}

	var rm *registerMessage
	if sticky != "" {
		sHost, sPort := p.se.decryptStickyCookie(sticky)

		// Check sticky session
		if sHost != "" && sPort != 0 {
			for _, droplet := range s {
				if droplet.Host == sHost && droplet.Port == sPort {
					rm = droplet
					break
				}
			}
		}
	}

	// No valid sticky session found, choose one randomly
	if rm == nil {
		rm = s[rand.Intn(len(s))]
	}

	return rm
}

func (p *Proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	start := time.Now()

	// Return 200 OK for heartbeats from LB
	if req.UserAgent() == "HTTP-Monitor/1.1" {
		rw.WriteHeader(http.StatusOK)
		fmt.Fprintln(rw, "ok")
		return
	}

	p.varz.IncRequests()

	r := p.Lookup(req)
	if r == nil {
		p.recordStatus(400, start, nil)
		p.varz.IncBadRequests()

		rw.WriteHeader(http.StatusNotFound)
		return
	}

	// Save the app_id of active app
	p.activeApps.Insert(r.App)

	p.varz.IncRequestsWithTags(r.Tags)
	p.varz.IncAppRequests(getUrl(req))

	outreq := new(http.Request)
	*outreq = *req // includes shallow copies of maps, but okay

	outHost := fmt.Sprintf("%s:%d", r.Host, r.Port)
	outreq.URL.Scheme = "http"
	outreq.URL.Host = outHost

	outreq.Proto = "HTTP/1.1"
	outreq.ProtoMajor = 1
	outreq.ProtoMinor = 1
	outreq.Close = false

	// Remove the connection header to the backend.  We want a
	// persistent connection, regardless of what the client sent
	// to us.  This is modifying the same underlying map from req
	// (shallow copied above) so we only copy it if necessary.
	if outreq.Header.Get("Connection") != "" {
		outreq.Header = make(http.Header)
		copyHeader(outreq.Header, req.Header)
		outreq.Header.Del("Connection")
	}

	if clientIp, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
		outreq.Header.Set("X-Forwarded-For", clientIp)
	}

	res, err := http.DefaultTransport.RoundTrip(outreq)
	if err != nil {
		log.Errorf("http: proxy error: %v", err)

		p.recordStatus(500, start, r.Tags)
		p.varz.IncBadRequests()

		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	p.recordStatus(res.StatusCode, start, r.Tags)

	copyHeader(rw.Header(), res.Header)

	if req.Header.Get(VcapTraceHeader) != "" {
		rw.Header().Set(VcapRouterHeader, config.ip)
		rw.Header().Set(VcapBackendHeader, outHost)
	}

	needSticky := false
	for _, v := range res.Cookies() {
		if v.Name == StickyCookieKey {
			needSticky = true
			break
		}
	}

	if needSticky {
		cookie := &http.Cookie{
			Name:  VcapCookieId,
			Value: p.se.getStickyCookie(r),
		}
		http.SetCookie(rw, cookie)
	}

	rw.WriteHeader(res.StatusCode)

	if res.Body != nil {
		var dst io.Writer = rw
		io.Copy(dst, res.Body)
	}
}

func (p *Proxy) recordStatus(status int, start time.Time, tags map[string]string) {
	latency := int(time.Since(start).Nanoseconds() / 1000000)
	p.varz.RecordResponse(status, latency, tags)
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func getUrl(req *http.Request) string {
	host := req.Host

	// Remove :<port>
	i := strings.Index(host, ":")
	if i >= 0 {
		host = host[0:i]
	}

	return strings.ToLower(host)
}
<commit_msg>Use RWMutex and smaller lock scope for proxy<commit_after>package router

import (
	"fmt"
	"io"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

const (
	VcapBackendHeader = "X-Vcap-Backend"
	VcapRouterHeader  = "X-Vcap-Router"
	VcapTraceHeader   = "X-Vcap-Trace"

	VcapCookieId    = "__VCAP_ID__"
	StickyCookieKey = "JSESSIONID"
)

type registerMessage struct {
	Host string            `json:"host"`
	Port uint16            `json:"port"`
	Uris []string          `json:"uris"`
	Tags map[string]string `json:"tags"`
	Dea  string            `json:"dea"`
	App  string            `json:"app"`

	Sticky string
}

func (r *registerMessage) HostPort() string {
	return fmt.Sprintf("%s:%d", r.Host, r.Port)
}

type Proxy struct {
	sync.RWMutex

	r          map[string][]*registerMessage
	d          map[string]int
	varz       *Varz
	se         *SessionEncoder
	activeApps *AppList
}

func NewProxy(se *SessionEncoder, activeApps *AppList, varz *Varz) *Proxy {
	p := new(Proxy)

	p.r = make(map[string][]*registerMessage)
	p.d = make(map[string]int)

	p.se = se
	p.varz = varz
	p.activeApps = activeApps

	return p
}

func (p *Proxy) Register(m *registerMessage) {
	p.Lock()

	// Store droplet in registry
	for _, uri := range m.Uris {
		uri = strings.ToLower(uri)
		s := p.r[uri]
		if s == nil {
			s = make([]*registerMessage, 0)
		}

		p.varz.RegisterApp(uri)

		exist := false
		for _, d := range s {
			if d.Host == m.Host && d.Port == m.Port {
				exist = true
				break
			}
		}

		if !exist {
			s = append(s, m)
			p.r[uri] = s
			p.d[m.HostPort()]++
		}
	}

	p.Unlock()

	p.updateStatus()
}

func (p *Proxy) Unregister(m *registerMessage) {
	hp := m.HostPort()

	p.Lock()

	// Delete droplets from registry
	for _, uri := range m.Uris {
		s := p.r[uri]
		if s == nil {
			continue
		}

		exist := false
		for i, d := range s {
			if d.Host == m.Host && d.Port == m.Port {
				s[i] = s[len(s)-1]
				exist = true
				break
			}
		}

		if exist {
			s = s[:len(s)-1]
			if len(s) == 0 {
				p.varz.UnregisterApp(uri)
				delete(p.r, uri)
			} else {
				p.r[uri] = s
			}

			p.d[hp]--
			if p.d[hp] == 0 {
				delete(p.d, hp)
			}
		}
	}

	p.Unlock()

	p.updateStatus()
}

func (p *Proxy) updateStatus() {
	p.varz.Urls = len(p.r)
	p.varz.Droplets = len(p.d)
}

func (p *Proxy) lookup(req *http.Request) []*registerMessage {
	url := getUrl(req)

	return p.r[url]
}

func (p *Proxy) Lookup(req *http.Request) *registerMessage {
	p.RLock()
	s := p.lookup(req)
	p.RUnlock()

	if s == nil {
		return nil
	}

	// If there's only one backend, choose that
	if len(s) == 1 {
		return s[0]
	}

	// Choose backend depending on sticky session
	var sticky string
	for _, v := range req.Cookies() {
		if v.Name == VcapCookieId {
			sticky = v.Value
			break
		}
	}

	var rm *registerMessage
	if sticky != "" {
		sHost, sPort := p.se.decryptStickyCookie(sticky)

		// Check sticky session
		if sHost != "" && sPort != 0 {
			for _, droplet := range s {
				if droplet.Host == sHost && droplet.Port == sPort {
					rm = droplet
					break
				}
			}
		}
	}

	// No valid sticky session found, choose one randomly
	if rm == nil {
		rm = s[rand.Intn(len(s))]
	}

	return rm
}

func (p *Proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	start := time.Now()

	// Return 200 OK for heartbeats from LB
	if req.UserAgent() == "HTTP-Monitor/1.1" {
		rw.WriteHeader(http.StatusOK)
		fmt.Fprintln(rw, "ok")
		return
	}

	p.varz.IncRequests()

	r := p.Lookup(req)
	if r == nil {
		p.recordStatus(400, start, nil)
		p.varz.IncBadRequests()

		rw.WriteHeader(http.StatusNotFound)
		return
	}

	// Save the app_id of active app
	p.activeApps.Insert(r.App)

	p.varz.IncRequestsWithTags(r.Tags)
	p.varz.IncAppRequests(getUrl(req))

	outreq := new(http.Request)
	*outreq = *req // includes shallow copies of maps, but okay

	outHost := fmt.Sprintf("%s:%d", r.Host, r.Port)
	outreq.URL.Scheme = "http"
	outreq.URL.Host = outHost

	outreq.Proto = "HTTP/1.1"
	outreq.ProtoMajor = 1
	outreq.ProtoMinor = 1
	outreq.Close = false

	// Remove the connection header to the backend.  We want a
	// persistent connection, regardless of what the client sent
	// to us.  This is modifying the same underlying map from req
	// (shallow copied above) so we only copy it if necessary.
	if outreq.Header.Get("Connection") != "" {
		outreq.Header = make(http.Header)
		copyHeader(outreq.Header, req.Header)
		outreq.Header.Del("Connection")
	}

	if clientIp, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
		outreq.Header.Set("X-Forwarded-For", clientIp)
	}

	res, err := http.DefaultTransport.RoundTrip(outreq)
	if err != nil {
		log.Errorf("http: proxy error: %v", err)

		p.recordStatus(500, start, r.Tags)
		p.varz.IncBadRequests()

		rw.WriteHeader(http.StatusInternalServerError)
		return
	}

	p.recordStatus(res.StatusCode, start, r.Tags)

	copyHeader(rw.Header(), res.Header)

	if req.Header.Get(VcapTraceHeader) != "" {
		rw.Header().Set(VcapRouterHeader, config.ip)
		rw.Header().Set(VcapBackendHeader, outHost)
	}

	needSticky := false
	for _, v := range res.Cookies() {
		if v.Name == StickyCookieKey {
			needSticky = true
			break
		}
	}

	if needSticky {
		cookie := &http.Cookie{
			Name:  VcapCookieId,
			Value: p.se.getStickyCookie(r),
		}
		http.SetCookie(rw, cookie)
	}

	rw.WriteHeader(res.StatusCode)

	if res.Body != nil {
		var dst io.Writer = rw
		io.Copy(dst, res.Body)
	}
}

func (p *Proxy) recordStatus(status int, start time.Time, tags map[string]string) {
	latency := int(time.Since(start).Nanoseconds() / 1000000)
	p.varz.RecordResponse(status, latency, tags)
}

func copyHeader(dst, src http.Header) {
	for k, vv := range src {
		for _, v := range vv {
			dst.Add(k, v)
		}
	}
}

func getUrl(req *http.Request) string {
	host := req.Host

	// Remove :<port>
	i := strings.Index(host, ":")
	if i >= 0 {
		host = host[0:i]
	}

	return strings.ToLower(host)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/Sirupsen/logrus"
	"io"
	"net"
	"strconv"
	"sync"
)

const (
	NETBUF_SIZE = 2048 // bytes
)

var (
	poolNetBuffers sync.Pool
)

// var-constants
var (
	HEAD_CONNECTION            = []byte("CONNECTION")
	HEAD_CONNECTION_KEEP_ALIVE = []byte("KEEP-ALIVE")
	HEAD_CONTENT_LENGTH        = []byte("CONTENT-LENGTH")
)

// Get or create network buffer for proxy
func netbufGet() (buf []byte) {
	bufInterface := poolNetBuffers.Get()
	if bufInterface == nil {
		buf = make([]byte, NETBUF_SIZE)
	} else {
		buf = bufInterface.([]byte)
		// prevent data leak
		for i := range buf {
			buf[i] = 0
		}

	}
	return buf
}

func netbufPut(buf []byte) {
	poolNetBuffers.Put(buf)
}

func proxyHTTPHeaders(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) (keepalive bool, contentLength int64) {
	buf := netbufGet()
	defer netbufPut(buf)
	var totalReadBytes int

	// Read lines
readHeaderLines:
	for {
		var i int
		var headerStart []byte
		for i = 0; i < len(buf); i++ {
			readBytes, err := sourceConn.Read(buf[i : i+1])
			totalReadBytes += readBytes
			if err != nil {
				logrus.Debugf("Error while read header from '%v' cid '%v': %v", sourceConn.RemoteAddr(), cid, err)
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Can't read a byte from header from '%v' cid '%v'", sourceConn.RemoteAddr(), cid)
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if buf[i] == ':' || buf[i] == '\n' {
				headerStart = buf[:i+1]
				logrus.Debugf("Header Name '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, buf[:i])
				break
			}
		}
		if len(headerStart) == 0 {
			logrus.Infof("Header line longer then buffer (%v bytes). Force close connection. '%v' -> '%v' cid '%v'.", len(buf), sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
			targetConn.Close()
			sourceConn.Close()
			return
		}

		// Empty line - end http headers
		if bytes.Equal(headerStart, []byte("\n")) || bytes.Equal(headerStart, []byte("\r\n")) {
			break readHeaderLines
		}

		headerName := headerStart[:len(headerStart)-1]
		headerNameUpperCase := bytes.ToUpper(headerName) // Cut trailing colon from start

		skipHeader := false
		for _, ownHeader := range cutHeaders {
			if bytes.Equal(ownHeader, headerNameUpperCase) {
				skipHeader = true
				break
			}
			ownHeaderS := string(ownHeader)
			headerNameS := string(headerNameUpperCase)
			skipHeader = ownHeaderS == headerNameS
		}

		if skipHeader {
			logrus.Debugf("Skip header: '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerNameUpperCase)
			buf[0] = headerStart[len(headerStart)-1]

			for buf[0] != '\n' {
				_, err := sourceConn.Read(buf[:1])
				if err != nil {
					logrus.Infof("Error read header. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
					sourceConn.Close()
					targetConn.Close()
					return
				}
			}
			continue readHeaderLines
		}

		logrus.Debugf("Copy header: '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerName)

		// copy header without changes
		_, err := targetConn.Write(headerStart)
		if err != nil {
			logrus.Infof("Write header start, from '%v' to '%v' cid '%v', headerStart='%s': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerStart, err)
			sourceConn.Close()
			targetConn.Close()
			return
		}

		needHeaderContent := bytes.Equal(headerNameUpperCase, HEAD_CONTENT_LENGTH) || bytes.Equal(headerNameUpperCase, HEAD_CONNECTION)
		headerContent := bytes.NewBuffer(buf[1:])
		headerContent.Reset()

		for buf[0] != '\n' {
			readBytes, err := sourceConn.Read(buf[:1])
			if err != nil {
				logrus.Infof("Error read header to copy. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Header copy read bytes != 1. Error. Close connections. '%v' -> '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			_, err = targetConn.Write(buf[:1])
			if err != nil {
				logrus.Infof("Error write header. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if needHeaderContent {
				headerContent.WriteByte(buf[0])
			}
		}
		if needHeaderContent {
			switch {
			case bytes.Equal(headerNameUpperCase, HEAD_CONNECTION):
				keepalive = bytes.EqualFold(HEAD_CONNECTION_KEEP_ALIVE, bytes.TrimSpace(headerContent.Bytes()))

			case bytes.Equal(headerNameUpperCase, HEAD_CONTENT_LENGTH):
				contentLength, err = strconv.ParseInt(string(bytes.TrimSpace(headerContent.Bytes())), 10, 64)
				if err == nil {
					logrus.Debugf("Header content-length parsed from '%v' to '%v' cid '%v': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), cid, contentLength)
				} else {
					logrus.Infof("Can't header content-length parsed from '%v' to '%v' cid '%v' content '%s': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), cid, headerContent.Bytes(), err)
					contentLength = 0
				}

			default:
				logrus.Debugf("ERROR. Unknow why i need header content. Code error. From '%v' to '%v' cid '%v', header name '%s'",
					sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerNameUpperCase,
				)
			}
		}
	}

	remoteTcpAddr := sourceConn.RemoteAddr().(*net.TCPAddr)
	remoteAddrString := remoteTcpAddr.IP.String()

	headerBuf := bytes.NewBuffer(buf)
	headerBuf.Reset()

	// Write real IP
	for _, header := range realIPHeaderNames {
		headerBuf.Write(header)
		headerBuf.WriteByte(':')
		headerBuf.WriteString(remoteAddrString)
		headerBuf.WriteString("\r\n")
	}

	// Write CID
	if *connectionIDHeader != "" {
		headerBuf.WriteString(*connectionIDHeader)
		headerBuf.WriteString(": ")
		headerBuf.WriteString(cid.String())
		headerBuf.WriteString("\r\n")
	}

	headerBuf.Write(additionalHeaders)
	headerBuf.Write([]byte("\r\n")) // end http headers
	logrus.Debugf("Add headers. '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerBuf.Bytes())

	_, err := targetConn.Write(headerBuf.Bytes())
	if err != nil {
		logrus.Infof("Error while write real ip headers to target '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
	}

	return
}

func startProxy(cid ConnectionID, targetAddr net.TCPAddr, in net.Conn) {
	targetConnCommon, err := net.DialTimeout("tcp", targetAddr.String(), *targetConnTimeout)
	if err != nil {
		logrus.Warnf("Can't connect to target '%v' cid '%v': %v", targetAddr.String(), cid, err)
		return
	}

	targetConn, ok := targetConnCommon.(*net.TCPConn)
	targetConn.SetKeepAlive(true)
	targetConn.SetKeepAlivePeriod(*tcpKeepAliveInterval)

	if !ok {
		logrus.Errorf("Can't cast connection to tcp connection, target '%v' cid '%v'", targetAddr.String(), cid)
		return
	}

	switch *proxyMode {
	case "http":
		startProxyHTTP(cid, targetConn, in)
	case "tcp":
		startProxyTCP(cid, targetConn, in)
	default:
		logrus.Panicf("Unknow proxy mode cid '%v': %v", cid, *proxyMode)
	}
}

func startProxyHTTP(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) {
	logrus.Debugf("Start http-proxy connection from '%v' to '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)

	// answer from server proxy without changes
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error1 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()

	// proxy incoming traffic, parse every headers
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)
		var summBytesCopied int64
		for {
			keepalive, contentLength := proxyHTTPHeaders(cid, targetConn, sourceConn)
			if keepalive {
				logrus.Debugf("Start keep-alieved proxy. '%v' -> '%v' cid '%v', content-length '%v'", sourceConn.RemoteAddr(),
					targetConn.RemoteAddr(), cid, contentLength)

				bytesCopied, err := io.CopyBuffer(targetConn, io.LimitReader(sourceConn, contentLength), buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection chunk copied '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				if err != nil {
					logrus.Debugf("Connection closed '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				}
			} else {
				logrus.Debugf("Start proxy without support keepalive middle headers '%v' -> '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
				bytesCopied, err := io.CopyBuffer(targetConn, sourceConn, buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection closed '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
		}
	}()
}

func startProxyTCP(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start tcp-proxy connection from '%v' to'%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)

	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(targetConn, sourceConn, buf)
		logrus.Debugf("Connection closed with error2 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error3 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()
}
<commit_msg>close in connection while can't connect to target<commit_after>package main

import (
	"bytes"
	"github.com/Sirupsen/logrus"
	"io"
	"net"
	"strconv"
	"sync"
)

const (
	NETBUF_SIZE = 2048 // bytes
)

var (
	poolNetBuffers sync.Pool
)

// var-constants
var (
	HEAD_CONNECTION            = []byte("CONNECTION")
	HEAD_CONNECTION_KEEP_ALIVE = []byte("KEEP-ALIVE")
	HEAD_CONTENT_LENGTH        = []byte("CONTENT-LENGTH")
)

// Get or create network buffer for proxy
func netbufGet() (buf []byte) {
	bufInterface := poolNetBuffers.Get()
	if bufInterface == nil {
		buf = make([]byte, NETBUF_SIZE)
	} else {
		buf = bufInterface.([]byte)
		// prevent data leak
		for i := range buf {
			buf[i] = 0
		}

	}
	return buf
}

func netbufPut(buf []byte) {
	poolNetBuffers.Put(buf)
}

func proxyHTTPHeaders(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) (keepalive bool, contentLength int64) {
	buf := netbufGet()
	defer netbufPut(buf)
	var totalReadBytes int

	// Read lines
readHeaderLines:
	for {
		var i int
		var headerStart []byte
		for i = 0; i < len(buf); i++ {
			readBytes, err := sourceConn.Read(buf[i : i+1])
			totalReadBytes += readBytes
			if err != nil {
				logrus.Debugf("Error while read header from '%v' cid '%v': %v", sourceConn.RemoteAddr(), cid, err)
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Can't read a byte from header from '%v' cid '%v'", sourceConn.RemoteAddr(), cid)
				targetConn.Close()
				sourceConn.Close()
				return
			}
			if buf[i] == ':' || buf[i] == '\n' {
				headerStart = buf[:i+1]
				logrus.Debugf("Header Name '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, buf[:i])
				break
			}
		}
		if len(headerStart) == 0 {
			logrus.Infof("Header line longer then buffer (%v bytes). Force close connection. '%v' -> '%v' cid '%v'.", len(buf), sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
			targetConn.Close()
			sourceConn.Close()
			return
		}

		// Empty line - end http headers
		if bytes.Equal(headerStart, []byte("\n")) || bytes.Equal(headerStart, []byte("\r\n")) {
			break readHeaderLines
		}

		headerName := headerStart[:len(headerStart)-1]
		headerNameUpperCase := bytes.ToUpper(headerName) // Cut trailing colon from start

		skipHeader := false
		for _, ownHeader := range cutHeaders {
			if bytes.Equal(ownHeader, headerNameUpperCase) {
				skipHeader = true
				break
			}
			ownHeaderS := string(ownHeader)
			headerNameS := string(headerNameUpperCase)
			skipHeader = ownHeaderS == headerNameS
		}

		if skipHeader {
			logrus.Debugf("Skip header: '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerNameUpperCase)
			buf[0] = headerStart[len(headerStart)-1]

			for buf[0] != '\n' {
				_, err := sourceConn.Read(buf[:1])
				if err != nil {
					logrus.Infof("Error read header. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
					sourceConn.Close()
					targetConn.Close()
					return
				}
			}
			continue readHeaderLines
		}

		logrus.Debugf("Copy header: '%v' -> '%v' cid '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerName)

		// copy header without changes
		_, err := targetConn.Write(headerStart)
		if err != nil {
			logrus.Infof("Write header start, from '%v' to '%v' cid '%v', headerStart='%s': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerStart, err)
			sourceConn.Close()
			targetConn.Close()
			return
		}

		needHeaderContent := bytes.Equal(headerNameUpperCase, HEAD_CONTENT_LENGTH) || bytes.Equal(headerNameUpperCase, HEAD_CONNECTION)
		headerContent := bytes.NewBuffer(buf[1:])
		headerContent.Reset()

		for buf[0] != '\n' {
			readBytes, err := sourceConn.Read(buf[:1])
			if err != nil {
				logrus.Infof("Error read header to copy. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if readBytes != 1 {
				logrus.Infof("Header copy read bytes != 1. Error. Close connections. '%v' -> '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			_, err = targetConn.Write(buf[:1])
			if err != nil {
				logrus.Infof("Error write header. Close connections. '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
			if needHeaderContent {
				headerContent.WriteByte(buf[0])
			}
		}
		if needHeaderContent {
			switch {
			case bytes.Equal(headerNameUpperCase, HEAD_CONNECTION):
				keepalive = bytes.EqualFold(HEAD_CONNECTION_KEEP_ALIVE, bytes.TrimSpace(headerContent.Bytes()))

			case bytes.Equal(headerNameUpperCase, HEAD_CONTENT_LENGTH):
				contentLength, err = strconv.ParseInt(string(bytes.TrimSpace(headerContent.Bytes())), 10, 64)
				if err == nil {
					logrus.Debugf("Header content-length parsed from '%v' to '%v' cid '%v': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), cid, contentLength)
				} else {
					logrus.Infof("Can't header content-length parsed from '%v' to '%v' cid '%v' content '%s': %v", sourceConn.RemoteAddr(),
						targetConn.RemoteAddr(), cid, headerContent.Bytes(), err)
					contentLength = 0
				}

			default:
				logrus.Debugf("ERROR. Unknow why i need header content. Code error. From '%v' to '%v' cid '%v', header name '%s'",
					sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, headerNameUpperCase,
				)
			}
		}
	}

	remoteTcpAddr := sourceConn.RemoteAddr().(*net.TCPAddr)
	remoteAddrString := remoteTcpAddr.IP.String()

	headerBuf := bytes.NewBuffer(buf)
	headerBuf.Reset()

	// Write real IP
	for _, header := range realIPHeaderNames {
		headerBuf.Write(header)
		headerBuf.WriteByte(':')
		headerBuf.WriteString(remoteAddrString)
		headerBuf.WriteString("\r\n")
	}

	// Write CID
	if *connectionIDHeader != "" {
		headerBuf.WriteString(*connectionIDHeader)
		headerBuf.WriteString(": ")
		headerBuf.WriteString(cid.String())
		headerBuf.WriteString("\r\n")
	}

	headerBuf.Write(additionalHeaders)
	headerBuf.Write([]byte("\r\n")) // end http headers
	logrus.Debugf("Add headers. '%v' -> '%v': '%s'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), headerBuf.Bytes())

	_, err := targetConn.Write(headerBuf.Bytes())
	if err != nil {
		logrus.Infof("Error while write real ip headers to target '%v' -> '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), err)
	}

	return
}

func startProxy(cid ConnectionID, targetAddr net.TCPAddr, in net.Conn) {
	targetConnCommon, err := net.DialTimeout("tcp", targetAddr.String(), *targetConnTimeout)
	if err != nil {
		logrus.Warnf("Can't connect to target '%v' cid '%v': %v", targetAddr.String(), cid, err)
		in.Close()
		return
	}

	targetConn, ok := targetConnCommon.(*net.TCPConn)
	targetConn.SetKeepAlive(true)
	targetConn.SetKeepAlivePeriod(*tcpKeepAliveInterval)

	if !ok {
		logrus.Errorf("Can't cast connection to tcp connection, target '%v' cid '%v'", targetAddr.String(), cid)
		return
	}

	switch *proxyMode {
	case "http":
		startProxyHTTP(cid, targetConn, in)
	case "tcp":
		startProxyTCP(cid, targetConn, in)
	default:
		logrus.Panicf("Unknow proxy mode cid '%v': %v", cid, *proxyMode)
	}
}

func startProxyHTTP(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) {
	logrus.Debugf("Start http-proxy connection from '%v' to '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)

	// answer from server proxy without changes
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error1 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()

	// proxy incoming traffic, parse every headers
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)
		var summBytesCopied int64
		for {
			keepalive, contentLength := proxyHTTPHeaders(cid, targetConn, sourceConn)
			if keepalive {
				logrus.Debugf("Start keep-alieved proxy. '%v' -> '%v' cid '%v', content-length '%v'", sourceConn.RemoteAddr(),
					targetConn.RemoteAddr(), cid, contentLength)

				bytesCopied, err := io.CopyBuffer(targetConn, io.LimitReader(sourceConn, contentLength), buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection chunk copied '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				if err != nil {
					logrus.Debugf("Connection closed '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				}
			} else {
				logrus.Debugf("Start proxy without support keepalive middle headers '%v' -> '%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)
				bytesCopied, err := io.CopyBuffer(targetConn, sourceConn, buf)
				summBytesCopied += bytesCopied
				logrus.Debugf("Connection closed '%v' -> '%v' cid '%v', bytes transferred '%v' (%v), error: %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, bytesCopied, summBytesCopied, err)
				sourceConn.Close()
				targetConn.Close()
				return
			}
		}
	}()
}

func startProxyTCP(cid ConnectionID, targetConn net.Conn, sourceConn net.Conn) {
	logrus.Infof("Start tcp-proxy connection from '%v' to'%v' cid '%v'", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid)

	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(targetConn, sourceConn, buf)
		logrus.Debugf("Connection closed with error2 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()
	go func() {
		buf := netbufGet()
		defer netbufPut(buf)

		_, err := io.CopyBuffer(sourceConn, targetConn, buf)
		logrus.Debugf("Connection closed with error3 '%v' -> '%v' cid '%v': %v", sourceConn.RemoteAddr(), targetConn.RemoteAddr(), cid, err)
		sourceConn.Close()
		targetConn.Close()
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	cli "github.com/jawher/mow.cli"
	util "github.com/kopoli/go-util"
)

const (
	APIPath string = "/api/v1/image"
)

type Config struct {
	opts     util.Options
	files    []string
	jobCount int
	timeout  int
}

func runCli(c *Config, args []string) (err error) {
	progName := c.opts.Get("program-name", "paperless-uploader")
	progVersion := c.opts.Get("program-version", "undefined")
	app := cli.App(progName, "Upload tool to Paperless Office server.")

	app.Version("version", fmt.Sprintf("%s: %s\nBuilt with: %s/%s on %s/%s",
		progName, progVersion, runtime.Compiler, runtime.Version(),
		runtime.GOOS, runtime.GOARCH))

	app.Spec = "[OPTIONS] URL FILES..."

	optJobs := app.IntOpt("j jobs", runtime.NumCPU(), "Number of concurrent uploads")
	optVerbose := app.BoolOpt("v verbose", false, "Print upload statuses")
	optTags := app.StringOpt("t tag", "", "Comma separated list of tags.")
	optTimeout := app.IntOpt("timeout", 60, "HTTP timeout in seconds")
	argURL := app.StringArg("URL", "", "The upload HTTP URL.")
	argFiles := app.StringsArg("FILES", []string{}, "Image files to upload.")

	app.Action = func() {
		c.opts.Set("tags", *optTags)
		if *optVerbose {
			c.opts.Set("verbose", "t")
		}
		c.opts.Set("url", *argURL)
		c.jobCount = *optJobs
		c.timeout = *optTimeout

		c.files = *argFiles
	}

	err = app.Run(args)
	if err != nil {
		return
	}

	return
}

func checkArguments(c *Config) (err error) {
	var u *url.URL

	urlstr := c.opts.Get("url", "")
	u, err = url.Parse(urlstr)
	if err != nil {
		return
	}

	apipath, err := url.Parse(APIPath)
	if err != nil {
		return
	}

	// Set the proper URL here
	urlstr = u.ResolveReference(apipath).String()
	c.opts.Set("url", urlstr)

	if !u.IsAbs() {
		err = util.E.New("Supplied URL must be absolute: %s", urlstr)
		return
	}

	for i := range c.files {
		var st os.FileInfo
		st, err = os.Stat(c.files[i])
		if err != nil || !st.Mode().IsRegular() {
			err = util.E.New("Invalid file: %s", c.files[i])
			return
		}
	}

	if c.opts.IsSet("verbose") {
		fmt.Println("Uploading to URL:", urlstr)
	}

	return
}

func uploadFile(c *Config, file string) (err error) {
	fp, err := os.Open(file)
	if err != nil {
		return
	}
	defer fp.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("image", filepath.Base(file))
	if err != nil {
		return
	}

	_, err = io.Copy(part, fp)
	if err != nil {
		return
	}

	err = writer.WriteField("tags", c.opts.Get("tags", ""))
	if err != nil {
		return
	}
	err = writer.Close()
	if err != nil {
		return
	}

	req, err := http.NewRequest("POST", c.opts.Get("url", ""), body)
	if err != nil {
		return
	}

	req.Header.Set("Content-Type", writer.FormDataContentType())

	client := &http.Client{
		Timeout: time.Duration(c.timeout) * time.Second,
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	body.Reset()
	_, err = body.ReadFrom(resp.Body)
	if err != nil {
		return
	}
	resp.Body.Close()

	if c.opts.IsSet("verbose") {
		fmt.Println("Uploaded:", file, "Response:", body.String())
	}

	if resp.StatusCode != http.StatusCreated {
		err = util.E.New("Server responded unexpectedly with code: %d", resp.StatusCode)
		return
	}

	return
}

func upload(c *Config) (err error) {
	jobs := make(chan string, 10)
	wg := sync.WaitGroup{}
	worker := func(jobs <-chan string) {
		var err error
		for file := range jobs {
			err = uploadFile(c, file)
			if err != nil {
				fmt.Printf("%s: failed: %s\n", file, err)
			} else {
				fmt.Printf("%s: Uploaded ok\n", file)
			}
		}
		wg.Done()
	}

	for i := 0; i < c.jobCount; i++ {
		wg.Add(1)
		go worker(jobs)
	}

	for i := range c.files {
		jobs <- c.files[i]
	}

	close(jobs)
	wg.Wait()

	return nil
}

func main() {
	config := &Config{
		opts: util.NewOptions(),
	}

	config.opts.Set("program-name", os.Args[0])

	err := runCli(config, os.Args)
	if err != nil {
		err = util.E.Annotate(err, "Command line parsing failed")
		goto error
	}

	err = checkArguments(config)
	if err != nil {
		err = util.E.Annotate(err, "Invalid arguments")
		goto error
	}

	err = upload(config)
	if err != nil {
		goto error
	}

	os.Exit(0)

error:
	fmt.Fprintln(os.Stderr, "Error:", err)
	os.Exit(1)
}
<commit_msg>uploader: parse the messages sent by the server<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	cli "github.com/jawher/mow.cli"
	util "github.com/kopoli/go-util"
)

const (
	APIPath string = "/api/v1/image"
)

type Config struct {
	opts     util.Options
	files    []string
	jobCount int
	timeout  int
}

type JsendMsg struct {
	Message string `json:message`
	Status  string `json:status`
}

func runCli(c *Config, args []string) (err error) {
	progName := c.opts.Get("program-name", "paperless-uploader")
	progVersion := c.opts.Get("program-version", "undefined")
	app := cli.App(progName, "Upload tool to Paperless Office server.")

	app.Version("version", fmt.Sprintf("%s: %s\nBuilt with: %s/%s on %s/%s",
		progName, progVersion, runtime.Compiler, runtime.Version(),
		runtime.GOOS, runtime.GOARCH))

	app.Spec = "[OPTIONS] URL FILES..."

	optJobs := app.IntOpt("j jobs", runtime.NumCPU(), "Number of concurrent uploads")
	optVerbose := app.BoolOpt("v verbose", false, "Print upload statuses")
	optTags := app.StringOpt("t tag", "", "Comma separated list of tags.")
	optTimeout := app.IntOpt("timeout", 60, "HTTP timeout in seconds")
	argURL := app.StringArg("URL", "", "The upload HTTP URL.")
	argFiles := app.StringsArg("FILES", []string{}, "Image files to upload.")

	app.Action = func() {
		c.opts.Set("tags", *optTags)
		if *optVerbose {
			c.opts.Set("verbose", "t")
		}
		c.opts.Set("url", *argURL)
		c.jobCount = *optJobs
		c.timeout = *optTimeout

		c.files = *argFiles
	}

	err = app.Run(args)
	if err != nil {
		return
	}

	return
}

func checkArguments(c *Config) (err error) {
	var u *url.URL

	urlstr := c.opts.Get("url", "")
	u, err = url.Parse(urlstr)
	if err != nil {
		return
	}

	apipath, err := url.Parse(APIPath)
	if err != nil {
		return
	}

	// Set the proper URL here
	urlstr = u.ResolveReference(apipath).String()
	c.opts.Set("url", urlstr)

	if !u.IsAbs() {
		err = util.E.New("Supplied URL must be absolute: %s", urlstr)
		return
	}

	for i := range c.files {
		var st os.FileInfo
		st, err = os.Stat(c.files[i])
		if err != nil || !st.Mode().IsRegular() {
			err = util.E.New("Invalid file: %s", c.files[i])
			return
		}
	}

	if c.opts.IsSet("verbose") {
		fmt.Println("Uploading to URL:", urlstr)
	}

	return
}

func uploadFile(c *Config, file string) (err error) {
	fp, err := os.Open(file)
	if err != nil {
		return
	}
	defer fp.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("image", filepath.Base(file))
	if err != nil {
		return
	}

	_, err = io.Copy(part, fp)
	if err != nil {
		return
	}

	err = writer.WriteField("tags", c.opts.Get("tags", ""))
	if err != nil {
		return
	}
	err = writer.Close()
	if err != nil {
		return
	}

	req, err := http.NewRequest("POST", c.opts.Get("url", ""), body)
	if err != nil {
		return
	}

	req.Header.Set("Content-Type", writer.FormDataContentType())

	client := &http.Client{
		Timeout: time.Duration(c.timeout) * time.Second,
	}
	resp, err := client.Do(req)
	if err != nil {
		return
	}

	body.Reset()
	_, err = body.ReadFrom(resp.Body)
	if err != nil {
		return
	}
	resp.Body.Close()

	var jmsg JsendMsg

	err = json.Unmarshal(body.Bytes(), &jmsg)
	if err != nil {
		// Ignore improper jsend messages
		jmsg.Message = ""
	}

	if c.opts.IsSet("verbose") {
		fmt.Println("Uploaded:", file, "Response:", body.String())
	}

	if resp.StatusCode != http.StatusCreated {
		msg := ""
		if jmsg.Message != "" {
			msg = fmt.Sprintf(" (%s)", jmsg.Message)
		}
		err = util.E.New("Server responded with: %d%s", resp.StatusCode, msg)
		return
	}

	return
}

func upload(c *Config) (err error) {
	jobs := make(chan string, 10)
	wg := sync.WaitGroup{}
	worker := func(jobs <-chan string) {
		var err error
		for file := range jobs {
			err = uploadFile(c, file)
			if err != nil {
				fmt.Printf("%s: failed: %s\n", file, err)
			} else {
				fmt.Printf("%s: Uploaded ok\n", file)
			}
		}
		wg.Done()
	}

	for i := 0; i < c.jobCount; i++ {
		wg.Add(1)
		go worker(jobs)
	}

	for i := range c.files {
		jobs <- c.files[i]
	}

	close(jobs)
	wg.Wait()

	return nil
}

func main() {
	config := &Config{
		opts: util.NewOptions(),
	}

	config.opts.Set("program-name", os.Args[0])

	err := runCli(config, os.Args)
	if err != nil {
		err = util.E.Annotate(err, "Command line parsing failed")
		goto error
	}

	err = checkArguments(config)
	if err != nil {
		err = util.E.Annotate(err, "Invalid arguments")
		goto error
	}

	err = upload(config)
	if err != nil {
		goto error
	}

	os.Exit(0)

error:
	fmt.Fprintln(os.Stderr, "Error:", err)
	os.Exit(1)
}
<|endoftext|>