<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"strings"

	"github.com/mcuadros/go-version"
)

const TAG_PREFIX = "refs/tags/"

// IsTagExist returns true if given tag exists in the repository.
func IsTagExist(repoPath, name string) bool {
	return IsReferenceExist(repoPath, TAG_PREFIX+name)
}

func (repo *Repository) IsTagExist(name string) bool {
	return IsTagExist(repo.Path, name)
}

func (repo *Repository) CreateTag(name, revision string) error {
	_, err := NewCommand("tag", name, revision).RunInDir(repo.Path)
	return err
}

func (repo *Repository) getTag(id sha1) (*Tag, error) {
	t, ok := repo.tagCache.Get(id.String())
	if ok {
		log("Hit cache: %s", id)
		return t.(*Tag), nil
	}

	// Get tag type
	tp, err := NewCommand("cat-file", "-t", id.String()).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}
	tp = strings.TrimSpace(tp)

	// Tag is a commit.
	if ObjectType(tp) == OBJECT_COMMIT {
		tag := &Tag{
			ID:     id,
			Object: id,
			Type:   string(OBJECT_COMMIT),
			repo:   repo,
		}

		repo.tagCache.Set(id.String(), tag)
		return tag, nil
	}

	// Tag with message.
	data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}

	tag, err := parseTagData(data)
	if err != nil {
		return nil, err
	}

	tag.ID = id
	tag.repo = repo

	repo.tagCache.Set(id.String(), tag)
	return tag, nil
}

// GetTag returns a Git tag by given name.
func (repo *Repository) GetTag(name string) (*Tag, error) {
	stdout, err := NewCommand("show-ref", "--tags", name).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	id, err := NewIDFromString(strings.Split(stdout, " ")[0])
	if err != nil {
		return nil, err
	}

	tag, err := repo.getTag(id)
	if err != nil {
		return nil, err
	}
	tag.Name = name
	return tag, nil
}

// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
	cmd := NewCommand("tag", "-l")
	if version.Compare(gitVersion, "2.0.0", ">=") {
		cmd.AddArguments("--sort=-v:refname")
	}

	stdout, err := cmd.RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	tags := strings.Split(strings.TrimSpace(stdout), "\n")

	if version.Compare(gitVersion, "2.0.0", "<") {
		version.Sort(tags)

		// Reverse order
		for i := 0; i < len(tags) / 2; i++ {
			j := len(tags) - i - 1
			tags[i], tags[j] = tags[j], tags[i]
		}
	}

	return tags, nil
}
<commit_msg>Fixed returning non empty tag list when there are no tags.<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"strings"

	"github.com/mcuadros/go-version"
)

const TAG_PREFIX = "refs/tags/"

// IsTagExist returns true if given tag exists in the repository.
func IsTagExist(repoPath, name string) bool {
	return IsReferenceExist(repoPath, TAG_PREFIX+name)
}

func (repo *Repository) IsTagExist(name string) bool {
	return IsTagExist(repo.Path, name)
}

func (repo *Repository) CreateTag(name, revision string) error {
	_, err := NewCommand("tag", name, revision).RunInDir(repo.Path)
	return err
}

func (repo *Repository) getTag(id sha1) (*Tag, error) {
	t, ok := repo.tagCache.Get(id.String())
	if ok {
		log("Hit cache: %s", id)
		return t.(*Tag), nil
	}

	// Get tag type
	tp, err := NewCommand("cat-file", "-t", id.String()).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}
	tp = strings.TrimSpace(tp)

	// Tag is a commit.
	if ObjectType(tp) == OBJECT_COMMIT {
		tag := &Tag{
			ID:     id,
			Object: id,
			Type:   string(OBJECT_COMMIT),
			repo:   repo,
		}

		repo.tagCache.Set(id.String(), tag)
		return tag, nil
	}

	// Tag with message.
	data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}

	tag, err := parseTagData(data)
	if err != nil {
		return nil, err
	}

	tag.ID = id
	tag.repo = repo

	repo.tagCache.Set(id.String(), tag)
	return tag, nil
}

// GetTag returns a Git tag by given name.
func (repo *Repository) GetTag(name string) (*Tag, error) {
	stdout, err := NewCommand("show-ref", "--tags", name).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	id, err := NewIDFromString(strings.Split(stdout, " ")[0])
	if err != nil {
		return nil, err
	}

	tag, err := repo.getTag(id)
	if err != nil {
		return nil, err
	}
	tag.Name = name
	return tag, nil
}

// GetTags returns all tags of the repository.
func (repo *Repository) GetTags() ([]string, error) {
	cmd := NewCommand("tag", "-l")
	if version.Compare(gitVersion, "2.0.0", ">=") {
		cmd.AddArguments("--sort=-v:refname")
	}

	stdout, err := cmd.RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	tags := strings.Split(stdout, "\n")
	tags = tags[:len(tags)-1]

	if version.Compare(gitVersion, "2.0.0", "<") {
		version.Sort(tags)

		// Reverse order
		for i := 0; i < len(tags) / 2; i++ {
			j := len(tags) - i - 1
			tags[i], tags[j] = tags[j], tags[i]
		}
	}

	return tags, nil
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/coreos/go-etcd/etcd"
    "github.com/miekg/dns"
    "net"
    "strings"
    "bytes"
    "time"
)

type Resolver struct {
    etcd        *etcd.Client
    dns         *dns.Client
    rTimeout   time.Duration
}

func (r *Resolver) Lookup(req *dns.Msg, nameservers []string) (msg *dns.Msg) {
    q := req.Question[0]

    msg = new(dns.Msg)
    msg.SetReply(req)

    if q.Qclass == dns.ClassINET {

        // A records
        if q.Qtype == dns.TypeA || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupA(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // AAAA records
        if q.Qtype == dns.TypeAAAA || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupAAAA(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // TXT records
        if q.Qtype == dns.TypeTXT || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupTXT(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // CNAME records
        if q.Qtype == dns.TypeCNAME || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupCNAME(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // NS records
        if q.Qtype == dns.TypeNS || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupNS(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }
    }

    if len(msg.Answer) == 0 {
        c := make(chan *dns.Msg)
        for _, nameserver := range nameservers {
            go r.LookupNameserver(c, req, nameserver)
        }

        timeout := time.After(r.rTimeout)
        select {
        case result := <-c:
            return result
        case <-timeout:
            return
        }
    }

    return
}

func (r *Resolver) LookupNameserver(c chan *dns.Msg, req *dns.Msg, ns string) {
    msg, _, err := r.dns.Exchange(req, ns)
    if err != nil {
        return
    }
    c <- msg
}

func (r *Resolver) LookupA(name string, class uint16) (answers []*dns.A) {
    answers = make([]*dns.A, 0)

    key := nameToKey(name, "/.A")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    var nodes []*etcd.Node

    if response.Node.Dir == true {

        nodes = make([]*etcd.Node, len(response.Node.Nodes))
        for i := 0; i < len(response.Node.Nodes); i++ {
            nodes[i] = &response.Node.Nodes[i]
        }

    } else {

        nodes = make([]*etcd.Node, 1)
        nodes[0] = response.Node

    }

    answers = make([]*dns.A, len(nodes))

    for i := 0; i < len(nodes); i++ {

        node := nodes[i]
        ip := net.ParseIP(node.Value)
        if ip == nil {
            logger.Fatalf("Failed to parse IP value '%s'", node.Value)
        }

        rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeA, Ttl: 0}
        answers[i] = &dns.A{*rr_header, ip}
    }

    return
}

func (r *Resolver) LookupAAAA(name string, class uint16) (answers []*dns.AAAA) {
    answers = make([]*dns.AAAA, 0)

    key := nameToKey(name, "/.AAAA")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    ip := net.ParseIP(node.Value)
    if ip == nil {
        logger.Fatalf("Failed to parse IP value '%s'", node.Value)
    }

    answers = make([]*dns.AAAA, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeAAAA, Ttl: 0}
    answers[0] = &dns.AAAA{*rr_header, ip}

    return
}

func (r *Resolver) LookupTXT(name string, class uint16) (answers []*dns.TXT) {
    answers = make([]*dns.TXT, 0)

    key := nameToKey(name, "/.TXT")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.TXT, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeTXT, Ttl: 0}
    answers[0] = &dns.TXT{*rr_header, []string{node.Value}}

    return
}

func (r *Resolver) LookupCNAME(name string, class uint16) (answers []*dns.CNAME) {
    answers = make([]*dns.CNAME, 0)

    key := nameToKey(name, "/.CNAME")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.CNAME, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeCNAME, Ttl: 0}
    answers[0] = &dns.CNAME{*rr_header, node.Value}

    return
}

func (r *Resolver) LookupNS(name string, class uint16) (answers []*dns.NS) {
    answers = make([]*dns.NS, 0)

    key := nameToKey(name, "/.NS")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.NS, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeNS, Ttl: 0}
    answers[0] = &dns.NS{*rr_header, node.Value}

    return
}

func nameToKey(name string, suffix string) string {
    segments := strings.Split(name, ".")

    var keyBuffer bytes.Buffer
    for i := len(segments) - 1; i >= 0; i-- {
        if len(segments[i]) > 0 {
            keyBuffer.WriteString("/")
            keyBuffer.WriteString(segments[i])
        }
    }

    keyBuffer.WriteString(suffix)
    return keyBuffer.String()
}
<commit_msg>First crude attempt to compose RR creation from shared funcs -- only implemented for A records still, this is largely a straw man for me figuring out reuse and structural patterns in go<commit_after>package main

import (
    "github.com/coreos/go-etcd/etcd"
    "github.com/miekg/dns"
    "net"
    "strings"
    "bytes"
    "time"
)

type Resolver struct {
    etcd        *etcd.Client
    dns         *dns.Client
    rTimeout   time.Duration
}

// A resultsToRecords func converts raw values for etcd Nodes into dns Answers
type resultsToRecords func(rawRecords []*etcd.Node) (answers []dns.RR)

// a nodeToRecordMapper func turns a single 'file'-type etcd node into a dns resourcerecord
type nodeToRecordMapper func(node *etcd.Node) dns.RR

func nodeToIpAddr (node *etcd.Node) net.IP {

    ip := net.ParseIP(node.Value)
    if ip == nil {
        logger.Fatalf("Failed to parse IP value '%s'", node.Value)
    }

    return ip
}

// util function to run mapping function over everything in a list of etcd Nodes
func mapEachRecord(nodes []*etcd.Node, mapper nodeToRecordMapper) (answers []dns.RR) {

    answers = make([]dns.RR, len(nodes))

    for i := 0; i < len(nodes); i++ {
        answers[i] = mapper(nodes[i])
    }

    return
}

// search etcd for all records at a key )wether just one or a list from a 'directory'
func (r *Resolver) GetFromStorage(key string) (nodes []*etcd.Node) {

    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    if response.Node.Dir == true {

        nodes = make([]*etcd.Node, len(response.Node.Nodes))
        for i := 0; i < len(response.Node.Nodes); i++ {
            nodes[i] = &response.Node.Nodes[i]
        }

    } else {

        nodes = make([]*etcd.Node, 1)
        nodes[0] = response.Node

    }

    return
}

func (r *Resolver) Lookup(req *dns.Msg, nameservers []string) (msg *dns.Msg) {
    q := req.Question[0]

    msg = new(dns.Msg)
    msg.SetReply(req)

    // Define some useful typical callbacks for core record types:

    // shorthand for a RR_Header ctor bound to name & class
    makeRRHeader := func (rrtype uint16) dns.RR_Header {
        return dns.RR_Header{Name: q.Name, Class: q.Qclass, Rrtype: rrtype, Ttl: 0}
    }

    // for some reason appending a slice of answers from mapEachRecord isn't
    // working, so add a util to loop and append (urgh)
    addAnswers := func (items []dns.RR) {
        for _, a := range items {
            msg.Answer = append(msg.Answer, a)
        }
    }

    // check query type and act accordingly

    if q.Qclass == dns.ClassINET {

        // A records
        if q.Qtype == dns.TypeA || q.Qtype == dns.TypeANY {

            nodes := r.GetFromStorage(nameToKey(q.Name, "/.A"))
            answers := mapEachRecord(nodes, func (node *etcd.Node) dns.RR {
                ip := nodeToIpAddr(node)
                return &dns.A{makeRRHeader(dns.TypeA), ip}
            })

            // not really sure why this doesnt work, `append` docs claim it should:
            // msg.Answer = append(msg.Answer, answers)
            addAnswers(answers)
        }

        // AAAA records
        if q.Qtype == dns.TypeAAAA || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupAAAA(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // TXT records
        if q.Qtype == dns.TypeTXT || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupTXT(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // CNAME records
        if q.Qtype == dns.TypeCNAME || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupCNAME(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }

        // NS records
        if q.Qtype == dns.TypeNS || q.Qtype == dns.TypeANY {
            for _, a := range r.LookupNS(q.Name, q.Qclass) {
                msg.Answer = append(msg.Answer, a)
            }
        }
    }

    if len(msg.Answer) == 0 {
        c := make(chan *dns.Msg)
        for _, nameserver := range nameservers {
            go r.LookupNameserver(c, req, nameserver)
        }

        timeout := time.After(r.rTimeout)
        select {
        case result := <-c:
            return result
        case <-timeout:
            return
        }
    }

    return
}

func (r *Resolver) LookupNameserver(c chan *dns.Msg, req *dns.Msg, ns string) {
    msg, _, err := r.dns.Exchange(req, ns)
    if err != nil {
        return
    }
    c <- msg
}

func (r *Resolver) LookupAAAA(name string, class uint16) (answers []*dns.AAAA) {
    answers = make([]*dns.AAAA, 0)

    key := nameToKey(name, "/.AAAA")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    ip := net.ParseIP(node.Value)
    if ip == nil {
        logger.Fatalf("Failed to parse IP value '%s'", node.Value)
    }

    answers = make([]*dns.AAAA, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeAAAA, Ttl: 0}
    answers[0] = &dns.AAAA{*rr_header, ip}

    return
}

func (r *Resolver) LookupTXT(name string, class uint16) (answers []*dns.TXT) {
    answers = make([]*dns.TXT, 0)

    key := nameToKey(name, "/.TXT")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.TXT, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeTXT, Ttl: 0}
    answers[0] = &dns.TXT{*rr_header, []string{node.Value}}

    return
}

func (r *Resolver) LookupCNAME(name string, class uint16) (answers []*dns.CNAME) {
    answers = make([]*dns.CNAME, 0)

    key := nameToKey(name, "/.CNAME")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.CNAME, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeCNAME, Ttl: 0}
    answers[0] = &dns.CNAME{*rr_header, node.Value}

    return
}

func (r *Resolver) LookupNS(name string, class uint16) (answers []*dns.NS) {
    answers = make([]*dns.NS, 0)

    key := nameToKey(name, "/.NS")
    response, err := r.etcd.Get(key, false, false)
    if err != nil {
        logger.Printf("Error with etcd: %s", err)
        return
    }

    node := response.Node

    answers = make([]*dns.NS, 1)
    rr_header := &dns.RR_Header{Name: name, Class: class, Rrtype: dns.TypeNS, Ttl: 0}
    answers[0] = &dns.NS{*rr_header, node.Value}

    return
}

func nameToKey(name string, suffix string) string {
    segments := strings.Split(name, ".")

    var keyBuffer bytes.Buffer
    for i := len(segments) - 1; i >= 0; i-- {
        if len(segments[i]) > 0 {
            keyBuffer.WriteString("/")
            keyBuffer.WriteString(segments[i])
        }
    }

    keyBuffer.WriteString(suffix)
    return keyBuffer.String()
}
<|endoftext|>


<commit_before>package server

import (
  "fmt";
  "time";
  "net/http";
  "path/filepath";
  "encoding/json";
)

const (
  TEXT string = "TEXT_RESPONSE"
  FILE string = "FILE_RESPONSE"
  REDIRECT string = "REDIRECT_RESPONSE"
)

func newResponse(request *Request) *Response {
  return &Response{
    request: request,
    Code: 200,
  }
}

type Response struct {
  request *Request
  responseType string
  Code int
  Body string
  Duration time.Duration
}

func (r *Response) Plain(data string, params ...interface{}) *Response {
  r.responseType = TEXT
  r.Body = fmt.Sprintf(data, params...)
  return r
}

func (r *Response) Json(data interface{}) *Response {
  r.SetHeader("Content-Type", "application/json")
  marshaled, err := json.Marshal(data)
  if err != nil {
    return r.Error(500, "Unable to encode response.")
  }
  return r.Plain(string(marshaled))
}

func (r *Response) Error(code int, message string, params ...interface{}) *Response {
  r.SetCode(code)
  return r.Plain(message, params...)
}

func (r *Response) ErrorJson(code int, data interface{}) *Response {
  r.SetCode(code)
  return r.Json(data)
}

func (r *Response) File(path string) *Response {
  r.responseType = FILE
  r.Body = filepath.Join(r.request.server.config.StaticRoot, path)
  return r
}

func (r *Response) Redirect(code int, url string) *Response {
  r.responseType = REDIRECT
  r.Body = url
  return r.SetCode(code)
}

func (r *Response) SetCode(code int) *Response {
  r.Code = code
  return r
}

func (r *Response) SetHeader(key string, value string) *Response {
  r.request.responseWriter.Header().Set(key, value)
  return r
}

func (r *Response) write() {
  r.Duration = time.Now().Sub(r.request.createdAt)
  switch r.responseType {
    case TEXT:
      r.request.responseWriter.WriteHeader(r.Code)
      fmt.Fprint(r.request.responseWriter, r.Body)
    case FILE:
      http.ServeFile(r.request.responseWriter, r.request.rawRequest, r.Body)
    case REDIRECT:
      http.Redirect(r.request.responseWriter, r.request.rawRequest, r.Body, r.Code)
    default:
      fmt.Fprint(r.request.responseWriter, "No response type specified.")
  }
}<commit_msg>Show json encoding error message<commit_after>package server

import (
  "fmt";
  "time";
  "net/http";
  "path/filepath";
  "encoding/json";
)

const (
  TEXT string = "TEXT_RESPONSE"
  FILE string = "FILE_RESPONSE"
  REDIRECT string = "REDIRECT_RESPONSE"
)

func newResponse(request *Request) *Response {
  return &Response{
    request: request,
    Code: 200,
  }
}

type Response struct {
  request *Request
  responseType string
  Code int
  Body string
  Duration time.Duration
}

func (r *Response) Plain(data string, params ...interface{}) *Response {
  r.responseType = TEXT
  r.Body = fmt.Sprintf(data, params...)
  return r
}

func (r *Response) Json(data interface{}) *Response {
  r.SetHeader("Content-Type", "application/json")
  marshaled, err := json.Marshal(data)
  if err != nil {
    return r.Error(500, "Unable to encode response. %s", err.Error())
  }
  return r.Plain(string(marshaled))
}

func (r *Response) Error(code int, message string, params ...interface{}) *Response {
  r.SetCode(code)
  return r.Plain(message, params...)
}

func (r *Response) ErrorJson(code int, data interface{}) *Response {
  r.SetCode(code)
  return r.Json(data)
}

func (r *Response) File(path string) *Response {
  r.responseType = FILE
  r.Body = filepath.Join(r.request.server.config.StaticRoot, path)
  return r
}

func (r *Response) Redirect(code int, url string) *Response {
  r.responseType = REDIRECT
  r.Body = url
  return r.SetCode(code)
}

func (r *Response) SetCode(code int) *Response {
  r.Code = code
  return r
}

func (r *Response) SetHeader(key string, value string) *Response {
  r.request.responseWriter.Header().Set(key, value)
  return r
}

func (r *Response) write() {
  r.Duration = time.Now().Sub(r.request.createdAt)
  switch r.responseType {
    case TEXT:
      r.request.responseWriter.WriteHeader(r.Code)
      fmt.Fprint(r.request.responseWriter, r.Body)
    case FILE:
      http.ServeFile(r.request.responseWriter, r.request.rawRequest, r.Body)
    case REDIRECT:
      http.Redirect(r.request.responseWriter, r.request.rawRequest, r.Body, r.Code)
    default:
      fmt.Fprint(r.request.responseWriter, "No response type specified.")
  }
}<|endoftext|>


<commit_before>// Package demultiplex demultiplexes Docker attach streams
package demultiplex

import (
	"bytes"
	"encoding/binary"
	"io"
)

func Streams(r io.Reader) (stdout, stderr io.Reader) {
	outr, outw := io.Pipe()
	errr, errw := io.Pipe()
	go func() {
		read := frameReader(r)
		for {
			typ, data, err := read()
			if typ == frameTypeStderr {
				if _, err := errw.Write(data); err != nil {
					outw.Close()
					return
				}
			} else {
				if _, err := outw.Write(data); err != nil {
					errw.Close()
					return
				}
			}
			if err != nil {
				outw.CloseWithError(err)
				errw.CloseWithError(err)
				return
			}
		}
	}()
	return outr, errr
}

type frameType byte

const (
	frameTypeStdin frameType = iota
	frameTypeStdout
	frameTypeStderr
)

func frameReader(r io.Reader) func() (frameType, []byte, error) {
	var buf bytes.Buffer
	var header [8]byte
	return func() (frameType, []byte, error) {
		buf.Reset()
		if _, err := io.ReadFull(r, header[:]); err != nil {
			return 0, nil, err
		}
		ft := frameType(header[0])
		length := int(binary.BigEndian.Uint32(header[4:]))
		buf.Grow(length)
		data := buf.Bytes()[:length]
		n, err := io.ReadFull(r, data)
		return ft, data[:n], err
	}
}

func Clean(r io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		read := frameReader(r)
		for {
			_, data, err := read()
			if _, err := pw.Write(data); err != nil {
				return
			}
			if err != nil {
				pw.CloseWithError(err)
				return
			}
		}
	}()
	return pr
}
<commit_msg>demultiplex: Add Copy<commit_after>// Package demultiplex demultiplexes Docker attach streams
package demultiplex

import (
	"bytes"
	"encoding/binary"
	"io"
)

func Streams(r io.Reader) (stdout, stderr io.Reader) {
	outr, outw := io.Pipe()
	errr, errw := io.Pipe()
	go func() {
		read := frameReader(r)
		for {
			typ, data, err := read()
			if typ == frameTypeStderr {
				if _, err := errw.Write(data); err != nil {
					outw.Close()
					return
				}
			} else {
				if _, err := outw.Write(data); err != nil {
					errw.Close()
					return
				}
			}
			if err != nil {
				outw.CloseWithError(err)
				errw.CloseWithError(err)
				return
			}
		}
	}()
	return outr, errr
}

type frameType byte

const (
	frameTypeStdin frameType = iota
	frameTypeStdout
	frameTypeStderr
)

func frameReader(r io.Reader) func() (frameType, []byte, error) {
	var buf bytes.Buffer
	var header [8]byte
	return func() (frameType, []byte, error) {
		buf.Reset()
		if _, err := io.ReadFull(r, header[:]); err != nil {
			return 0, nil, err
		}
		ft := frameType(header[0])
		length := int(binary.BigEndian.Uint32(header[4:]))
		buf.Grow(length)
		data := buf.Bytes()[:length]
		n, err := io.ReadFull(r, data)
		return ft, data[:n], err
	}
}

func Clean(r io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		read := frameReader(r)
		for {
			_, data, err := read()
			if _, err := pw.Write(data); err != nil {
				return
			}
			if err != nil {
				pw.CloseWithError(err)
				return
			}
		}
	}()
	return pr
}

func Copy(stdout, stderr io.Writer, r io.Reader) error {
	read := frameReader(r)
	for {
		t, data, err := read()
		var ew error
		if stderr != nil && t == frameTypeStderr {
			_, ew = stderr.Write(data)
		} else {
			_, ew = stdout.Write(data)
		}
		if ew != nil {
			return ew
		}
		if err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>package repo

import (
	"database/sql/driver"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
)

// URI identifies a repository.
type URI string

// IsGitHubRepository returns true iff this repository is hosted on GitHub.
func (u URI) IsGitHubRepository() bool {
	return strings.HasPrefix(strings.ToLower(string(u)), "github.com/")
}

// MakeURI converts a repository clone URL, such as
// "git://github.com/user/repo.git", to a normalized URI string, such as
// "github.com/user/repo".
func MakeURI(cloneURL string) URI {
	if cloneURL == "" {
		panic("MakeURI: empty clone URL")
	}

	url, err := url.Parse(cloneURL)
	if err != nil {
		panic(fmt.Sprintf("MakeURI(%q): %s", cloneURL, err))
	}

	path := strings.TrimSuffix(url.Path, ".git")
	path = filepath.Clean(path)
	path = strings.TrimSuffix(path, "/")
	return URI(strings.ToLower(url.Host) + path)
}

// URIEqual returns true if a and b are equal, based on a case insensitive
// comparison.
func URIEqual(a, b URI) bool {
	return strings.ToLower(string(a)) == strings.ToLower(string(b))
}

// Scan implements database/sql.Scanner.
func (u *URI) Scan(v interface{}) error {
	if data, ok := v.([]byte); ok {
		*u = URI(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", u, v)
}

// Value implements database/sql/driver.Valuer
func (u URI) Value() (driver.Value, error) {
	return string(u), nil
}

// URIs is a wrapper type for a slice of URIs.
type URIs []URI

// Strings returns the URIs as strings.
func (us URIs) Strings() []string {
	s := make([]string, len(us))
	for i, u := range us {
		s[i] = string(u)
	}
	return s
}
<commit_msg>link to github page correctyl<commit_after>package repo

import (
	"database/sql/driver"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
)

// URI identifies a repository.
type URI string

// IsGitHubRepository returns true iff this repository is hosted on GitHub.
func (u URI) IsGitHubRepository() bool {
	return strings.HasPrefix(strings.ToLower(string(u)), "github.com/")
}

// GitHubURL returns the https://github.com/USER/REPO URL for this repository.
func (u URI) GitHubURL() string {
	return "https://" + string(u)
}

// MakeURI converts a repository clone URL, such as
// "git://github.com/user/repo.git", to a normalized URI string, such as
// "github.com/user/repo".
func MakeURI(cloneURL string) URI {
	if cloneURL == "" {
		panic("MakeURI: empty clone URL")
	}

	url, err := url.Parse(cloneURL)
	if err != nil {
		panic(fmt.Sprintf("MakeURI(%q): %s", cloneURL, err))
	}

	path := strings.TrimSuffix(url.Path, ".git")
	path = filepath.Clean(path)
	path = strings.TrimSuffix(path, "/")
	return URI(strings.ToLower(url.Host) + path)
}

// URIEqual returns true if a and b are equal, based on a case insensitive
// comparison.
func URIEqual(a, b URI) bool {
	return strings.ToLower(string(a)) == strings.ToLower(string(b))
}

// Scan implements database/sql.Scanner.
func (u *URI) Scan(v interface{}) error {
	if data, ok := v.([]byte); ok {
		*u = URI(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", u, v)
}

// Value implements database/sql/driver.Valuer
func (u URI) Value() (driver.Value, error) {
	return string(u), nil
}

// URIs is a wrapper type for a slice of URIs.
type URIs []URI

// Strings returns the URIs as strings.
func (us URIs) Strings() []string {
	s := make([]string, len(us))
	for i, u := range us {
		s[i] = string(u)
	}
	return s
}
<|endoftext|>


<commit_before>package mdata

import (
	"sync"
	"time"

	"github.com/grafana/metrictank/mdata/cache"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

// AggMetrics is an in-memory store of AggMetric objects
// note: they are keyed by MKey here because each
// AggMetric manages access to, and references of,
// their rollup archives themselves
type AggMetrics struct {
	store          Store
	cachePusher    cache.CachePusher
	dropFirstChunk bool
	sync.RWMutex
	Metrics        map[schema.MKey]*AggMetric
	chunkMaxStale  uint32
	metricMaxStale uint32
	gcInterval     time.Duration
}

func NewAggMetrics(store Store, cachePusher cache.CachePusher, dropFirstChunk bool, chunkMaxStale, metricMaxStale uint32, gcInterval time.Duration) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		cachePusher:    cachePusher,
		dropFirstChunk: dropFirstChunk,
		Metrics:        make(map[schema.MKey]*AggMetric),
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		gcInterval:     gcInterval,
	}

	// gcInterval = 0 can be useful in tests
	if gcInterval > 0 {
		go ms.GC()
	}
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - uint32(ms.chunkMaxStale)
		metricMinTs := now - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesn't matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]schema.MKey, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc()
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if a.GC(now, chunkMinTs, metricMinTs) {
				log.Debug("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				metricsActive.Set(len(ms.Metrics))
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) Get(key schema.MKey) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key schema.MKey, schemaId, aggId uint16) Metric {
	ms.Lock()
	m, ok := ms.Metrics[key]
	if !ok {
		k := schema.AMKey{
			MKey: key,
		}
		agg := Aggregations.Get(aggId)
		schema := Schemas.Get(schemaId)
		m = NewAggMetric(ms.store, ms.cachePusher, k, schema.Retentions, schema.ReorderWindow, &agg, ms.dropFirstChunk)
		ms.Metrics[key] = m
		metricsActive.Set(len(ms.Metrics))
	}
	ms.Unlock()
	return m
}
<commit_msg>AggMetrics.GetOrCreate leverage RWLock<commit_after>package mdata

import (
	"sync"
	"time"

	"github.com/grafana/metrictank/mdata/cache"
	"github.com/raintank/worldping-api/pkg/log"
	"gopkg.in/raintank/schema.v1"
)

// AggMetrics is an in-memory store of AggMetric objects
// note: they are keyed by MKey here because each
// AggMetric manages access to, and references of,
// their rollup archives themselves
type AggMetrics struct {
	store          Store
	cachePusher    cache.CachePusher
	dropFirstChunk bool
	sync.RWMutex
	Metrics        map[schema.MKey]*AggMetric
	chunkMaxStale  uint32
	metricMaxStale uint32
	gcInterval     time.Duration
}

func NewAggMetrics(store Store, cachePusher cache.CachePusher, dropFirstChunk bool, chunkMaxStale, metricMaxStale uint32, gcInterval time.Duration) *AggMetrics {
	ms := AggMetrics{
		store:          store,
		cachePusher:    cachePusher,
		dropFirstChunk: dropFirstChunk,
		Metrics:        make(map[schema.MKey]*AggMetric),
		chunkMaxStale:  chunkMaxStale,
		metricMaxStale: metricMaxStale,
		gcInterval:     gcInterval,
	}

	// gcInterval = 0 can be useful in tests
	if gcInterval > 0 {
		go ms.GC()
	}
	return &ms
}

// periodically scan chunks and close any that have not received data in a while
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - uint32(ms.chunkMaxStale)
		metricMinTs := now - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of actives metrics.
		// it doesn't matter if new metrics are added while we iterate this list.
		ms.RLock()
		keys := make([]schema.MKey, 0, len(ms.Metrics))
		for k := range ms.Metrics {
			keys = append(keys, k)
		}
		ms.RUnlock()
		for _, key := range keys {
			gcMetric.Inc()
			ms.RLock()
			a := ms.Metrics[key]
			ms.RUnlock()
			if a.GC(now, chunkMinTs, metricMinTs) {
				log.Debug("metric %s is stale. Purging data from memory.", key)
				ms.Lock()
				delete(ms.Metrics, key)
				metricsActive.Set(len(ms.Metrics))
				ms.Unlock()
			}
		}

	}
}

func (ms *AggMetrics) Get(key schema.MKey) (Metric, bool) {
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	return m, ok
}

func (ms *AggMetrics) GetOrCreate(key schema.MKey, schemaId, aggId uint16) Metric {

	// in the most common case, it's already there and an Rlock is all we need
	ms.RLock()
	m, ok := ms.Metrics[key]
	ms.RUnlock()
	if ok {
		return m
	}

	// if it wasn't there, get the write lock and prepare to add it
	// but first we need to check again if someone has added it in
	// the meantime
	ms.Lock()
	m, ok = ms.Metrics[key]
	if ok {
		ms.Unlock()
		return m
	}
	k := schema.AMKey{
		MKey: key,
	}
	agg := Aggregations.Get(aggId)
	schema := Schemas.Get(schemaId)
	m = NewAggMetric(ms.store, ms.cachePusher, k, schema.Retentions, schema.ReorderWindow, &agg, ms.dropFirstChunk)
	ms.Metrics[key] = m
	metricsActive.Set(len(ms.Metrics))
	ms.Unlock()
	return m
}
<|endoftext|>


<commit_before>package rest

import (
	"net/http"
)

type Fuler interface {
	ServeHTTP(http.ResponseWriter, *http.Request)
	Before(*Ful) bool
	After(*Ful) bool
	Get(*Ful)
	Post(*Ful)
	Put(*Ful)
	Delete(*Ful)
}

//  RESTful ServeHTTP 结构
type Ful struct {
	W      http.ResponseWriter
	R      *http.Request
	Path   string
	Before func(*Ful) bool //HiJack
	After  func(*Ful) bool
	Get    func(*Ful)
	Post   func(*Ful)
	Put    func(*Ful)
	Delete func(*Ful)
}

// 如果有错误，写错误信息，并返回是否有错误
func (p *Ful) WriteErr(err error) bool {
	if err != nil {
		p.W.WriteHeader(http.StatusBadRequest)
		p.W.Write([]byte(err.Error()))
	}
	return err != nil
}

// RESTful ServeHTTP 分派
func (p *Ful) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.W = w
	p.R = r
	if p.Before != nil && p.Before(p) {
		return
	}
	defer func() {
		if p.After != nil {
			p.After(p)
		}
	}()
	var f func(*Ful)

	switch r.Method {
	case "GET":
		f = p.Get
	case "POST":
		f = p.Post
	case "PUT":
		f = p.Put
	case "DELETE":
		f = p.Delete
	}
	if f == nil {
		http.Error(w, "Method Not Allowed", 405)
	} else {
		f(p)
	}
}
<commit_msg>rest ful 重构<commit_after>// 类型复合在开发中很常见，多数情况下，内层类型访问外层类型的属性，需要使用 reflect 包完成。
// 虽然 reflect 相当高效，但是随着代码规模的扩大，问题会越来越复杂。
// rest 包推荐直接使用类型完成任务的方式，而不是使用类型复合。
// 这种使用方式看上去有些古怪，不过开发者将获得很大的自由度，同时也降低了 rest 包本身的维护。
package rest

import (
	"net/http"
)

// Ful 是一个简单的 http.Handler 实现
// Ful 依据 Request.Method 调用对应的函数
// 示例: 直接使用而不是用struct再次包装
// 		http.Handle(pattern, &Ful{
// 			Get: func(fu *Ful) {
// 				something()
// 			},
//			After: func(fu *Ful) {
// 				something()
// 			},
// 		})
//
type Ful struct {
	W      http.ResponseWriter
	R      *http.Request
	Before func(fu *Ful) bool             // 调用对应函数前，先调用 Before ，返回 false,表示跳过函数调用
	After  func(fu *Ful, err interface{}) // 最后调用的函数，err 是函数调用中 recover 到的。
	Get    func(fu *Ful)
	Post   func(fu *Ful)
	Put    func(fu *Ful)
	Delete func(fu *Ful)
}

// HandleFunc 分派
// 如果没有设置对应 Request.Method 的函数，将向 ResponseWriter 写入 405 Method Not Allowed
// 如果
func (fu *Ful) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p := new(Ful)
	p.W, p.R = w, r
	p.Before, p.After, p.Get, p.Post, p.Put, p.Delete = fu.Before, fu.After, fu.Get, fu.Post, fu.Put, fu.Delete

	defer func() {
		err := recover()
		if p.After != nil {
			p.After(p, err)
			return
		}
		if err != nil {
			p.WriteHeader(500)
		}
	}()

	if p.Before != nil && !p.Before(p) {
		return
	}
	var f func(*Ful)

	switch r.Method {
	case "GET":
		f = p.Get
	case "POST":
		f = p.Post
	case "PUT":
		f = p.Put
	case "DELETE":
		f = p.Delete
	}
	if f == nil {
		p.WriteHeader(405).Write("Method Not Allowed")
	} else {
		f(p)
	}
}

func (p *Ful) Write(content string) *Ful {
	p.W.Write([]byte(content))
	return p
}

func (p *Ful) WriteHeader(stat int) *Ful {
	p.W.WriteHeader(stat)
	return p
}

func (p *Ful) Redirect(stat int, url string) *Ful {
	p.W.Header().Set("Location", url)
	p.W.WriteHeader(stat)
	return p
}

func (p *Ful) SetHeader(hdr string, val string) *Ful {
	p.W.Header().Set(hdr, val)
	return p
}

func (p *Ful) AddHeader(hdr string, val string) *Ful {
	p.W.Header().Add(hdr, val)
	return p
}

// 设置一个 Path=="/" 的 cookie
func (p *Ful) SetCookie(name string, value string, maxAge int) *Ful {
	cookie := &http.Cookie{Path: "/", Name: name, Value: value, MaxAge: maxAge}
	http.SetCookie(p.W, cookie)
	return p
}

// Fu 与 Ful 类似，区别在使用上，需要提供一个生成器或者使用闭包的形式
// 示例: 因为闭包的存在，Get 之类的函数不再需要定义参数
// 		http.Handle(pattern,FuGen(func() *Fu{
//			var fu *Fu
//			fu = &Fu{
// 				Get: func() {
//	 				something(fu)
//				},
//				After: func() {
//					something(fu)
//				},
// 			}
//			return fu
// 		}))
// 看上去确实有些古怪，此方式结合了闭包和生成器
type Fu struct {
	gen    func() *Fu
	W      http.ResponseWriter
	R      *http.Request
	Before func() bool           // 调用对应函数前，先调用 Before ，返回 false,表示跳过函数调用
	After  func(err interface{}) // 最后调用的函数，err 是函数调用中 recover 到的。
	Get    func()
	Post   func()
	Put    func()
	Delete func()
}

// 由生成 type Fu 的函数 gen,构建 http.Handel
func FuGen(gen func() *Fu) *Fu {
	return &Fu{gen: gen}
}

// HandleFunc 分派
// 如果没有设置对应 Request.Method 的函数，将向 ResponseWriter 写入 405 Method Not Allowed
// 如果
func (fu *Fu) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var p *Fu
	defer func() {
		err := recover()
		if p != nil && p.After != nil {
			p.After(err)
			return
		}
		if err != nil {
			w.WriteHeader(500)
		}
	}()
	p = fu.gen()
	p.W, p.R = w, r

	if p.Before != nil && !p.Before() {
		return
	}
	var f func()

	switch r.Method {
	case "GET":
		f = p.Get
	case "POST":
		f = p.Post
	case "PUT":
		f = p.Put
	case "DELETE":
		f = p.Delete
	}
	if f == nil {
		p.WriteHeader(405).Write("Method Not Allowed")
	} else {
		f()
	}
}

func (p *Fu) Write(content string) *Fu {
	p.W.Write([]byte(content))
	return p
}

func (p *Fu) WriteHeader(stat int) *Fu {
	p.W.WriteHeader(stat)
	return p
}

func (p *Fu) Redirect(stat int, url string) *Fu {
	p.W.Header().Set("Location", url)
	p.W.WriteHeader(stat)
	return p
}

func (p *Fu) SetHeader(hdr string, val string) *Fu {
	p.W.Header().Set(hdr, val)
	return p
}

func (p *Fu) AddHeader(hdr string, val string) *Fu {
	p.W.Header().Add(hdr, val)
	return p
}

// 设置一个 Path=="/" 的 cookie
func (p *Fu) SetCookie(name string, value string, maxAge int) *Fu {
	cookie := &http.Cookie{Path: "/", Name: name, Value: value, MaxAge: maxAge}
	http.SetCookie(p.W, cookie)
	return p
}
<|endoftext|>


<commit_before>package aeds

import (
	"bytes"
	"encoding/gob"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// interface for structures that can be stored in App Engine's datastore
type Entity interface {
	Kind() string
	StringId() string

	HookBeforePut() // Calculate derived fields before writing to datastore

	// CacheTtl indicates how long the entity should be cached in memcache.
	// Return zero to disable memcache.  If this method returns a non-zero
	// duration, the receiver should also implement the GobEncoder and
	// GobDecoder interfaces.
	CacheTtl() time.Duration
}

// Key returns a datastore key for this entity.
func Key(c appengine.Context, e Entity) *datastore.Key {
	return datastore.NewKey(c, e.Kind(), e.StringId(), 0, nil)
}

// Put stores an entity in the datastore.
func Put(c appengine.Context, e Entity) (*datastore.Key, error) {
	e.HookBeforePut()
	ttl := e.CacheTtl()

	// encode entity as a gob (before storing in datastore)
	var value bytes.Buffer
	if ttl > 0 {
		err := gob.NewEncoder(&value).Encode(e)
		if err != nil {
			return nil, err
		}
	}

	// store entity in the datastore
	lookupKey := Key(c, e)
	key, err := datastore.Put(c, lookupKey, e)
	if err != nil {
		return nil, err
	}

	// store entity in memcache too?
	if ttl > 0 {
		item := &memcache.Item{
			Key:        lookupKey.String(),
			Value:      value.Bytes(),
			Expiration: ttl,
		}
		err := memcache.Set(c, item)
		_ = err // ignore memcache errors
	}

	return key, nil
}

// Delete removes an entity from the datastore.
func Delete(c appengine.Context, e Entity) error {
	lookupKey := Key(c, e)

	// should the entity be removed from memcache too?
	if e.CacheTtl() > 0 {
		err := memcache.Delete(c, lookupKey.String())
		if err == memcache.ErrCacheMiss {
			// noop
		} else if err != nil {
			return err
		}
	}

	return datastore.Delete(c, lookupKey)
}

// FromId fetches an entity based on its ID.  The given entity
// should have enough data to calculate the entity's key.  On
// success, the entity is modified in place with all data from
// the datastore.
// Field mismatch errors are ignored.
func FromId(c appengine.Context, e Entity) (Entity, error) {
	lookupKey := Key(c, e)
	ttl := e.CacheTtl()

	// should we look in memcache too?
	cacheMiss := false
	if ttl > 0 {
		item, err := memcache.Get(c, lookupKey.String())
		if err == nil {
			buf := bytes.NewBuffer(item.Value)
			err := gob.NewDecoder(buf).Decode(e)
			return e, err
		}
		if err == memcache.ErrCacheMiss {
			cacheMiss = true
		}
		// ignore any memcache errors
	}

	// look in the datastore
	err := datastore.Get(c, lookupKey, e)
	if err == nil {
		// should we update memcache?
		if cacheMiss && ttl > 0 {
			e.HookBeforePut()

			// encode
			var value bytes.Buffer
			err := gob.NewEncoder(&value).Encode(e)
			if err != nil {
				return nil, err
			}

			// store
			item := &memcache.Item{
				Key:        lookupKey.String(),
				Value:      value.Bytes(),
				Expiration: ttl,
			}
			err = memcache.Set(c, item)
			_ = err // ignore memcache errors
		}

		return e, nil
	}
	if IsErrFieldMismatch(err) {
		return e, nil
	}
	return nil, err // unknown datastore error
}
<commit_msg>Add HookAfterGet hook<commit_after>package aeds

import (
	"bytes"
	"encoding/gob"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// interface for structures that can be stored in App Engine's datastore
type Entity interface {
	Kind() string
	StringId() string

	HookAfterGet()  // Calculate derived fields after fetching from datastore
	HookBeforePut() // Calculate derived fields before writing to datastore

	// CacheTtl indicates how long the entity should be cached in memcache.
	// Return zero to disable memcache.  If this method returns a non-zero
	// duration, the receiver should also implement the GobEncoder and
	// GobDecoder interfaces.
	CacheTtl() time.Duration
}

// Key returns a datastore key for this entity.
func Key(c appengine.Context, e Entity) *datastore.Key {
	return datastore.NewKey(c, e.Kind(), e.StringId(), 0, nil)
}

// Put stores an entity in the datastore.
func Put(c appengine.Context, e Entity) (*datastore.Key, error) {
	e.HookBeforePut()
	ttl := e.CacheTtl()

	// encode entity as a gob (before storing in datastore)
	var value bytes.Buffer
	if ttl > 0 {
		err := gob.NewEncoder(&value).Encode(e)
		if err != nil {
			return nil, err
		}
	}

	// store entity in the datastore
	lookupKey := Key(c, e)
	key, err := datastore.Put(c, lookupKey, e)
	if err != nil {
		return nil, err
	}

	// store entity in memcache too?
	if ttl > 0 {
		item := &memcache.Item{
			Key:        lookupKey.String(),
			Value:      value.Bytes(),
			Expiration: ttl,
		}
		err := memcache.Set(c, item)
		_ = err // ignore memcache errors
	}

	return key, nil
}

// Delete removes an entity from the datastore.
func Delete(c appengine.Context, e Entity) error {
	lookupKey := Key(c, e)

	// should the entity be removed from memcache too?
	if e.CacheTtl() > 0 {
		err := memcache.Delete(c, lookupKey.String())
		if err == memcache.ErrCacheMiss {
			// noop
		} else if err != nil {
			return err
		}
	}

	return datastore.Delete(c, lookupKey)
}

// FromId fetches an entity based on its ID.  The given entity
// should have enough data to calculate the entity's key.  On
// success, the entity is modified in place with all data from
// the datastore.
// Field mismatch errors are ignored.
func FromId(c appengine.Context, e Entity) (Entity, error) {
	lookupKey := Key(c, e)
	ttl := e.CacheTtl()

	// should we look in memcache too?
	cacheMiss := false
	if ttl > 0 {
		item, err := memcache.Get(c, lookupKey.String())
		if err == nil {
			buf := bytes.NewBuffer(item.Value)
			err := gob.NewDecoder(buf).Decode(e)
			e.HookAfterGet()
			return e, err
		}
		if err == memcache.ErrCacheMiss {
			cacheMiss = true
		}
		// ignore any memcache errors
	}

	// look in the datastore
	err := datastore.Get(c, lookupKey, e)
	if err == nil {
		e.HookAfterGet()

		// should we update memcache?
		if cacheMiss && ttl > 0 {
			e.HookBeforePut()

			// encode
			var value bytes.Buffer
			err := gob.NewEncoder(&value).Encode(e)
			if err != nil {
				return nil, err
			}

			// store
			item := &memcache.Item{
				Key:        lookupKey.String(),
				Value:      value.Bytes(),
				Expiration: ttl,
			}
			err = memcache.Set(c, item)
			_ = err // ignore memcache errors
		}

		return e, nil
	}
	if IsErrFieldMismatch(err) {
		e.HookAfterGet()
		return e, nil
	}
	return nil, err // unknown datastore error
}
<|endoftext|>


<commit_before>// Copyright 2014-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

// package uroot contains various functions that might be needed more than
// one place.
package uroot

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"syscall"
)

const (
	// Not all these paths may be populated or even exist but OTOH they might.
	PATHHEAD = "/ubin"
	PATHMID  = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin"
	PATHTAIL = "/buildbin"
	CmdsPath = "github.com/u-root/u-root/cmds"
)

// TODO: make this a map so it's easier to find dups.
type dir struct {
	name string
	mode os.FileMode
}

type file struct {
	contents string
	mode     os.FileMode
}

// TODO: make this a map so it's easier to find dups.
type mount struct {
	source string
	target string
	fstype string
	flags  uintptr
	opts   string
}

var (
	Profile string
	Envs    []string
	env     = map[string]string{
		"LD_LIBRARY_PATH": "/usr/local/lib",
		"GOROOT":          "/go",
		"GOPATH":          "/",
		"GOBIN":           "/ubin",
		"CGO_ENABLED":     "0",
	}

	dirs = []dir{
		{name: "/proc", mode: os.FileMode(0555)},
		{name: "/sys", mode: os.FileMode(0555)},
		{name: "/buildbin", mode: os.FileMode(0777)},
		{name: "/ubin", mode: os.FileMode(0777)},
		{name: "/tmp", mode: os.FileMode(0777)},
		{name: "/env", mode: os.FileMode(0777)},
		{name: "/etc", mode: os.FileMode(0777)},
		{name: "/tcz", mode: os.FileMode(0777)},
		{name: "/dev", mode: os.FileMode(0777)},
		{name: "/lib", mode: os.FileMode(0777)},
		{name: "/usr/lib", mode: os.FileMode(0777)},
		{name: "/go/pkg/linux_amd64", mode: os.FileMode(0777)},
		// This is for uroot packages. Is this a good idea? I don't know.
		{name: "/pkg", mode: os.FileMode(0777)},
	}
	namespace = []mount{
		{source: "proc", target: "/proc", fstype: "proc", flags: syscall.MS_MGC_VAL, opts: ""},
		{source: "sys", target: "/sys", fstype: "sysfs", flags: syscall.MS_MGC_VAL, opts: ""},
		// Kernel must be compiled with CONFIG_DEVTMPFS, otherwise
		// default to contents of dev.cpio.
		{source: "none", target: "/dev", fstype: "devtmpfs", flags: syscall.MS_MGC_VAL},
	}

	files = map[string]file{
		"/etc/resolv.conf": {contents: `nameserver 8.8.8.8`, mode: os.FileMode(0644)},
	}
)

// build the root file system.
func Rootfs() {
	// Pick some reasonable values in the (unlikely!) even that Uname fails.
	uname := "linux"
	mach := "amd64"
	// There are three possible places for go:
	// The first is in /go/bin/$OS_$ARCH
	// The second is in /go/bin [why they still use this path is anyone's guess]
	// The third is in /go/pkg/tool/$OS_$ARCH
	if u, err := Uname(); err != nil {
		log.Printf("uroot.Utsname fails: %v, so assume %v_%v\n", err, uname, mach)
	} else {
		// Sadly, go and the OS disagree on many things.
		uname = strings.ToLower(u.Sysname)
		mach = strings.ToLower(u.Machine)
		// Yes, we really have to do this stupid thing.
		if mach[0:3] == "arm" {
			mach = "arm"
		}
		if mach == "x86_64" {
			mach = "amd64"
		}
	}
	goPath := fmt.Sprintf("/go/bin/%s_%s:/go/bin:/go/pkg/tool/%s_%s", uname, mach, uname, mach)
	env["PATH"] = fmt.Sprintf("%v:%v:%v:%v", goPath, PATHHEAD, PATHMID, PATHTAIL)

	for k, v := range env {
		os.Setenv(k, v)
		Envs = append(Envs, k+"="+v)
	}

	// Some systems wipe out all the environment variables we so carefully craft.
	// There is a way out -- we can put them into /etc/profile.d/uroot if we want.
	// The PATH variable has to change, however.
	env["PATH"] = fmt.Sprintf("%v:%v:%v:%v", goPath, PATHHEAD, "$PATH", PATHTAIL)
	for k, v := range env {
		Profile += "export " + k + "=" + v + "\n"
	}
	// The IFS lets us force a rehash every time we type a command, so that when we
	// build uroot commands we don't keep rebuilding them.
	Profile += "IFS=`hash -r`\n"
	// IF the profile is used, THEN when the user logs in they will need a private
	// tmpfs. There's no good way to do this on linux. The closest we can get for now
	// is to mount a tmpfs of /go/pkg/%s_%s :-(
	// Same applies to ubin. Each user should have their own.
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /go/pkg/%s_%s\n", uname, mach)
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /ubin\n")
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /pkg\n")

	for _, m := range dirs {
		if err := os.MkdirAll(m.name, m.mode); err != nil {
			log.Printf("mkdir :%s: mode %o: %v\n", m.name, m.mode, err)
			continue
		}
	}

	for _, m := range namespace {
		if err := syscall.Mount(m.source, m.target, m.fstype, m.flags, m.opts); err != nil {
			log.Printf("Mount :%s: on :%s: type :%s: flags %x opts: %s: %v\n", m.source, m.target, m.fstype, m.flags, m.opts, err)
		}

	}

	for name, m := range files {
		if err := ioutil.WriteFile(name, []byte(m.contents), m.mode); err != nil {
			log.Printf("Error writeing %v: %v", name, err)
		}
	}

	// only in case of emergency.
	if false {
		if err := filepath.Walk("/", func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" WALK FAIL%v: %v\n", name, err)
				// That's ok, sometimes things are not there.
				return nil
			}
			fmt.Printf("%v\n", name)
			return nil
		}); err != nil {
			log.Printf("WALK fails %v\n", err)
		}
	}
}
<commit_msg>uroot: bring back device creation<commit_after>// Copyright 2014-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

// package uroot contains various functions that might be needed more than
// one place.
package uroot

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"syscall"
)

const (
	// Not all these paths may be populated or even exist but OTOH they might.
	PATHHEAD = "/ubin"
	PATHMID  = "/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin"
	PATHTAIL = "/buildbin"
	CmdsPath = "github.com/u-root/u-root/cmds"
)

// TODO: make this a map so it's easier to find dups.
type dir struct {
	name string
	mode os.FileMode
}

type file struct {
	contents string
	mode     os.FileMode
}

// TODO: make this a map so it's easier to find dups.
type dev struct {
	name    string
	mode    uint32
	dev     int
	howmany int
}
type mount struct {
	source string
	target string
	fstype string
	flags  uintptr
	opts   string
}

var (
	Profile string
	Envs    []string
	env     = map[string]string{
		"LD_LIBRARY_PATH": "/usr/local/lib",
		"GOROOT":          "/go",
		"GOPATH":          "/",
		"GOBIN":           "/ubin",
		"CGO_ENABLED":     "0",
	}

	dirs = []dir{
		{name: "/proc", mode: os.FileMode(0555)},
		{name: "/sys", mode: os.FileMode(0555)},
		{name: "/buildbin", mode: os.FileMode(0777)},
		{name: "/ubin", mode: os.FileMode(0777)},
		{name: "/tmp", mode: os.FileMode(0777)},
		{name: "/env", mode: os.FileMode(0777)},
		{name: "/etc", mode: os.FileMode(0777)},
		{name: "/tcz", mode: os.FileMode(0777)},
		{name: "/dev", mode: os.FileMode(0777)},
		{name: "/lib", mode: os.FileMode(0777)},
		{name: "/usr/lib", mode: os.FileMode(0777)},
		{name: "/go/pkg/linux_amd64", mode: os.FileMode(0777)},
		// This is for uroot packages. Is this a good idea? I don't know.
		{name: "/pkg", mode: os.FileMode(0777)},
	}
	devs = []dev{
		// chicken and egg: these need to be there before you start. So, sadly,
		// we will always need dev.cpio or something like it.
		//{name: "/dev/null", mode: uint32(syscall.S_IFCHR) | 0666, dev: 0x0103},
		//{name: "/dev/console", mode: uint32(syscall.S_IFCHR) | 0666, dev: 0x0501},
		{name: "/dev/tty", mode: uint32(syscall.S_IFCHR) | 0666, dev: 0x0501},
	}
	namespace = []mount{
		{source: "proc", target: "/proc", fstype: "proc", flags: syscall.MS_MGC_VAL, opts: ""},
		{source: "sys", target: "/sys", fstype: "sysfs", flags: syscall.MS_MGC_VAL, opts: ""},
		// Kernel must be compiled with CONFIG_DEVTMPFS, otherwise
		// default to contents of dev.cpio.
		{source: "none", target: "/dev", fstype: "devtmpfs", flags: syscall.MS_MGC_VAL},
	}

	files = map[string]file{
		"/etc/resolv.conf": {contents: `nameserver 8.8.8.8`, mode: os.FileMode(0644)},
	}
)

// build the root file system.
func Rootfs() {
	// Pick some reasonable values in the (unlikely!) even that Uname fails.
	uname := "linux"
	mach := "amd64"
	// There are three possible places for go:
	// The first is in /go/bin/$OS_$ARCH
	// The second is in /go/bin [why they still use this path is anyone's guess]
	// The third is in /go/pkg/tool/$OS_$ARCH
	if u, err := Uname(); err != nil {
		log.Printf("uroot.Utsname fails: %v, so assume %v_%v\n", err, uname, mach)
	} else {
		// Sadly, go and the OS disagree on many things.
		uname = strings.ToLower(u.Sysname)
		mach = strings.ToLower(u.Machine)
		// Yes, we really have to do this stupid thing.
		if mach[0:3] == "arm" {
			mach = "arm"
		}
		if mach == "x86_64" {
			mach = "amd64"
		}
	}
	goPath := fmt.Sprintf("/go/bin/%s_%s:/go/bin:/go/pkg/tool/%s_%s", uname, mach, uname, mach)
	env["PATH"] = fmt.Sprintf("%v:%v:%v:%v", goPath, PATHHEAD, PATHMID, PATHTAIL)

	for k, v := range env {
		os.Setenv(k, v)
		Envs = append(Envs, k+"="+v)
	}

	// Some systems wipe out all the environment variables we so carefully craft.
	// There is a way out -- we can put them into /etc/profile.d/uroot if we want.
	// The PATH variable has to change, however.
	env["PATH"] = fmt.Sprintf("%v:%v:%v:%v", goPath, PATHHEAD, "$PATH", PATHTAIL)
	for k, v := range env {
		Profile += "export " + k + "=" + v + "\n"
	}
	// The IFS lets us force a rehash every time we type a command, so that when we
	// build uroot commands we don't keep rebuilding them.
	Profile += "IFS=`hash -r`\n"
	// IF the profile is used, THEN when the user logs in they will need a private
	// tmpfs. There's no good way to do this on linux. The closest we can get for now
	// is to mount a tmpfs of /go/pkg/%s_%s :-(
	// Same applies to ubin. Each user should have their own.
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /go/pkg/%s_%s\n", uname, mach)
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /ubin\n")
	Profile += fmt.Sprintf("sudo mount -t tmpfs none /pkg\n")

	for _, m := range dirs {
		if err := os.MkdirAll(m.name, m.mode); err != nil {
			log.Printf("mkdir :%s: mode %o: %v\n", m.name, m.mode, err)
			continue
		}
	}

	for _, d := range devs {
		syscall.Unlink(d.name)
		if err := syscall.Mknod(d.name, d.mode, d.dev); err != nil {
			log.Printf("mknod :%q: mode: %#o: magic: %v: %v\n", d.name, d.mode, d.dev, err)
			continue
		}
	}

	for _, m := range namespace {
		if err := syscall.Mount(m.source, m.target, m.fstype, m.flags, m.opts); err != nil {
			log.Printf("Mount :%s: on :%s: type :%s: flags %x opts: %s: %v\n", m.source, m.target, m.fstype, m.flags, m.opts, err)
		}

	}

	for name, m := range files {
		if err := ioutil.WriteFile(name, []byte(m.contents), m.mode); err != nil {
			log.Printf("Error writeing %v: %v", name, err)
		}
	}

	// only in case of emergency.
	if false {
		if err := filepath.Walk("/", func(name string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" WALK FAIL%v: %v\n", name, err)
				// That's ok, sometimes things are not there.
				return nil
			}
			fmt.Printf("%v\n", name)
			return nil
		}); err != nil {
			log.Printf("WALK fails %v\n", err)
		}
	}
}
<|endoftext|>


<commit_before>package urlcheck

type Scenario []Test

func (s Scenario) Test() (err error) {
	for _, t := range s {
		err = t.Test()
		if err != nil {
			break
		}
	}
	return
}
<commit_msg>Scenario.Test(): add step number to error string<commit_after>package urlcheck

import "errors"
import "strconv"

type Scenario []Test

func (s Scenario) Test() error {
	for i, t := range s {
		err := t.Test()
		if err != nil {
			return errors.New("Step " + strconv.Itoa(i+1) + ": " + err.Error())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/fcgi"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/pixiv/go-thumber/thumbnail"
)

var local = flag.String("local", "", "serve as webserver, example: 0.0.0.0:8000")
var timeout = flag.Int("timeout", 3, "timeout for upstream HTTP requests, in seconds")
var show_version = flag.Bool("version", false, "show version and exit")

var client http.Client

var version string

const maxDimension = 65000
const maxPixels = 10000000

var http_stats struct {
	received int64
	inflight int64
	ok int64
	thumb_error int64
	upstream_error int64
	arg_error int64
	total_time_us int64
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func errorServer(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func statusServer(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, "version %s\n", version)
	fmt.Fprintf(w, "received %d\n", atomic.LoadInt64(&http_stats.received))
	fmt.Fprintf(w, "inflight %d\n", atomic.LoadInt64(&http_stats.inflight))
	fmt.Fprintf(w, "ok %d\n", atomic.LoadInt64(&http_stats.ok))
	fmt.Fprintf(w, "thumb_error %d\n", atomic.LoadInt64(&http_stats.thumb_error))
	fmt.Fprintf(w, "upstream_error %d\n", atomic.LoadInt64(&http_stats.upstream_error))
	fmt.Fprintf(w, "arg_error %d\n", atomic.LoadInt64(&http_stats.arg_error))
	fmt.Fprintf(w, "total_time_us %d\n", atomic.LoadInt64(&http_stats.total_time_us))
}

func thumbServer(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	defer func() {
		elapsed := int64(time.Now().Sub(startTime) / 1000)
		atomic.AddInt64(&http_stats.total_time_us, elapsed)
	}()

	atomic.AddInt64(&http_stats.received, 1)
	atomic.AddInt64(&http_stats.inflight, 1)
	defer atomic.AddInt64(&http_stats.inflight, -1)

	path := r.URL.RequestURI()

	// Defaults
	var params = thumbnail.ThumbnailParameters{
		Upscale:        true,
		ForceAspect:    true,
		Quality:        90,
		Optimize:       false,
		PrescaleFactor: 2.0,
	}

	if path[0] != '/' {
		http.Error(w, "Path should start with /", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	parts := strings.SplitN(path[1:], "/", 2)
	if len(parts) < 2 {
		http.Error(w, "Path needs to have at least two components", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	for _, arg := range strings.Split(parts[0], ",") {
		tup := strings.SplitN(arg, "=", 2)
		if len(tup) != 2 {
			http.Error(w, "Arguments must have the form name=value", http.StatusBadRequest)
			atomic.AddInt64(&http_stats.arg_error, 1)
			return
		}
		switch tup[0] {
		case "w", "h", "q", "u", "a", "o":
			val, err := strconv.Atoi(tup[1])
			if err != nil {
				http.Error(w, "Invalid integer value for "+tup[0], http.StatusBadRequest)
				atomic.AddInt64(&http_stats.arg_error, 1)
				return
			}
			switch tup[0] {
			case "w":
				params.Width = val
			case "h":
				params.Height = val
			case "q":
				params.Quality = val
			case "u":
				params.Upscale = val != 0
			case "a":
				params.ForceAspect = val != 0
			case "o":
				params.Optimize = val != 0
			}
		case "p":
			val, err := strconv.ParseFloat(tup[1], 64)
			if err != nil {
				http.Error(w, "Invalid float value for "+tup[0], http.StatusBadRequest)
				atomic.AddInt64(&http_stats.arg_error, 1)
				return
			}
			params.PrescaleFactor = val
		}
	}
	if params.Width <= 0 || params.Width > maxDimension {
		http.Error(w, "Width (w) not specified or invalid", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Height <= 0 || params.Height > maxDimension {
		http.Error(w, "Height (h) not specified or invalid", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Width*params.Height > maxPixels {
		http.Error(w, "Image dimensions are insane", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Quality > 100 || params.Quality < 0 {
		http.Error(w, "Quality must be between 0 and 100", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}

	srcReader, err := client.Get("http://" + parts[1])
	if err != nil {
		http.Error(w, "Upstream failed: "+err.Error(), http.StatusBadGateway)
		atomic.AddInt64(&http_stats.upstream_error, 1)
		return
	}
	if srcReader.StatusCode != http.StatusOK {
		http.Error(w, "Upstream failed: "+srcReader.Status, srcReader.StatusCode)
		atomic.AddInt64(&http_stats.upstream_error, 1)
		return
	}

	w.Header().Set("Content-Type", "image/jpeg")
	err = thumbnail.MakeThumbnail(srcReader.Body, w, params)
	if err != nil {
		switch err := err.(type) {
		case *url.Error:
			http.Error(w, "Upstream failed: "+err.Error(), http.StatusBadGateway)
			atomic.AddInt64(&http_stats.upstream_error, 1)
			return
		default:
			http.Error(w, "Thumbnailing failed: "+err.Error(), http.StatusInternalServerError)
			atomic.AddInt64(&http_stats.thumb_error, 1)
			return
		}
	}
	srcReader.Body.Close()
	atomic.AddInt64(&http_stats.ok, 1)
}

func main() {
	flag.Parse()
	if *show_version {
		fmt.Printf("thumberd %s\n", version)
		return
	}

	client.Timeout = time.Duration(*timeout) * time.Second

	var err error

	http.HandleFunc("/server-status", statusServer)
	http.HandleFunc("/favicon.ico", errorServer)

	http.HandleFunc("/", thumbServer)

	if *local != "" { // Run as a local web server
		err = http.ListenAndServe(*local, nil)
	} else { // Run as FCGI via standard I/O
		err = fcgi.Serve(nil, nil)
	}
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Send a Last-Modified header containing the current time<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/fcgi"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/pixiv/go-thumber/thumbnail"
)

var local = flag.String("local", "", "serve as webserver, example: 0.0.0.0:8000")
var timeout = flag.Int("timeout", 3, "timeout for upstream HTTP requests, in seconds")
var show_version = flag.Bool("version", false, "show version and exit")

var client http.Client

var version string

const maxDimension = 65000
const maxPixels = 10000000

var http_stats struct {
	received int64
	inflight int64
	ok int64
	thumb_error int64
	upstream_error int64
	arg_error int64
	total_time_us int64
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func errorServer(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "404 Not Found", http.StatusNotFound)
}

func statusServer(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, "version %s\n", version)
	fmt.Fprintf(w, "received %d\n", atomic.LoadInt64(&http_stats.received))
	fmt.Fprintf(w, "inflight %d\n", atomic.LoadInt64(&http_stats.inflight))
	fmt.Fprintf(w, "ok %d\n", atomic.LoadInt64(&http_stats.ok))
	fmt.Fprintf(w, "thumb_error %d\n", atomic.LoadInt64(&http_stats.thumb_error))
	fmt.Fprintf(w, "upstream_error %d\n", atomic.LoadInt64(&http_stats.upstream_error))
	fmt.Fprintf(w, "arg_error %d\n", atomic.LoadInt64(&http_stats.arg_error))
	fmt.Fprintf(w, "total_time_us %d\n", atomic.LoadInt64(&http_stats.total_time_us))
}

func thumbServer(w http.ResponseWriter, r *http.Request) {
	startTime := time.Now()
	defer func() {
		elapsed := int64(time.Now().Sub(startTime) / 1000)
		atomic.AddInt64(&http_stats.total_time_us, elapsed)
	}()

	atomic.AddInt64(&http_stats.received, 1)
	atomic.AddInt64(&http_stats.inflight, 1)
	defer atomic.AddInt64(&http_stats.inflight, -1)

	path := r.URL.RequestURI()

	// Defaults
	var params = thumbnail.ThumbnailParameters{
		Upscale:        true,
		ForceAspect:    true,
		Quality:        90,
		Optimize:       false,
		PrescaleFactor: 2.0,
	}

	if path[0] != '/' {
		http.Error(w, "Path should start with /", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	parts := strings.SplitN(path[1:], "/", 2)
	if len(parts) < 2 {
		http.Error(w, "Path needs to have at least two components", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	for _, arg := range strings.Split(parts[0], ",") {
		tup := strings.SplitN(arg, "=", 2)
		if len(tup) != 2 {
			http.Error(w, "Arguments must have the form name=value", http.StatusBadRequest)
			atomic.AddInt64(&http_stats.arg_error, 1)
			return
		}
		switch tup[0] {
		case "w", "h", "q", "u", "a", "o":
			val, err := strconv.Atoi(tup[1])
			if err != nil {
				http.Error(w, "Invalid integer value for "+tup[0], http.StatusBadRequest)
				atomic.AddInt64(&http_stats.arg_error, 1)
				return
			}
			switch tup[0] {
			case "w":
				params.Width = val
			case "h":
				params.Height = val
			case "q":
				params.Quality = val
			case "u":
				params.Upscale = val != 0
			case "a":
				params.ForceAspect = val != 0
			case "o":
				params.Optimize = val != 0
			}
		case "p":
			val, err := strconv.ParseFloat(tup[1], 64)
			if err != nil {
				http.Error(w, "Invalid float value for "+tup[0], http.StatusBadRequest)
				atomic.AddInt64(&http_stats.arg_error, 1)
				return
			}
			params.PrescaleFactor = val
		}
	}
	if params.Width <= 0 || params.Width > maxDimension {
		http.Error(w, "Width (w) not specified or invalid", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Height <= 0 || params.Height > maxDimension {
		http.Error(w, "Height (h) not specified or invalid", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Width*params.Height > maxPixels {
		http.Error(w, "Image dimensions are insane", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}
	if params.Quality > 100 || params.Quality < 0 {
		http.Error(w, "Quality must be between 0 and 100", http.StatusBadRequest)
		atomic.AddInt64(&http_stats.arg_error, 1)
		return
	}

	srcReader, err := client.Get("http://" + parts[1])
	if err != nil {
		http.Error(w, "Upstream failed: "+err.Error(), http.StatusBadGateway)
		atomic.AddInt64(&http_stats.upstream_error, 1)
		return
	}
	if srcReader.StatusCode != http.StatusOK {
		http.Error(w, "Upstream failed: "+srcReader.Status, srcReader.StatusCode)
		atomic.AddInt64(&http_stats.upstream_error, 1)
		return
	}

	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
	err = thumbnail.MakeThumbnail(srcReader.Body, w, params)
	if err != nil {
		switch err := err.(type) {
		case *url.Error:
			http.Error(w, "Upstream failed: "+err.Error(), http.StatusBadGateway)
			atomic.AddInt64(&http_stats.upstream_error, 1)
			return
		default:
			http.Error(w, "Thumbnailing failed: "+err.Error(), http.StatusInternalServerError)
			atomic.AddInt64(&http_stats.thumb_error, 1)
			return
		}
	}
	srcReader.Body.Close()
	atomic.AddInt64(&http_stats.ok, 1)
}

func main() {
	flag.Parse()
	if *show_version {
		fmt.Printf("thumberd %s\n", version)
		return
	}

	client.Timeout = time.Duration(*timeout) * time.Second

	var err error

	http.HandleFunc("/server-status", statusServer)
	http.HandleFunc("/favicon.ico", errorServer)

	http.HandleFunc("/", thumbServer)

	if *local != "" { // Run as a local web server
		err = http.ListenAndServe(*local, nil)
	} else { // Run as FCGI via standard I/O
		err = fcgi.Serve(nil, nil)
	}
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// DNS resolver client: see RFC 1035.

package dns

import (
	"os"
	"net"
)

const packErr = "Failed to pack message"
const servErr = "No servers could be reached"

type Resolver struct {
	Servers  []string            // servers to use
	Search   []string            // suffixes to append to local name
	Port     string              // what port to use
	Ndots    int                 // number of dots in name to trigger absolute lookup -- TODO
	Timeout  int                 // seconds before giving up on packet
	Attempts int                 // lost packets before giving up on server
	Rotate   bool                // round robin among servers -- TODO
	Tcp      bool                // use TCP
	Mangle   func([]byte) []byte // mangle the packet
	// rtt map[string]int server->int, smaller is faster 0, -1 is unreacheble
}

// Basic usage pattern for setting up a resolver:
//
//        res := new(Resolver)
//        res.Servers = []string{"127.0.0.1"}           // set the nameserver
//
//        m := new(Msg)                                 // prepare a new message
//        m.MsgHdr.Recursion_desired = true             // header bits
//        m.Question = make([]Question, 1)              // 1 RR in question section
//        m.Question[0] = Question{"miek.nl", TypeSOA, ClassINET}
//        in, err := res.Query(m)                       // Ask the question
//
// Note that message id checking is left to the caller.
func (res *Resolver) Query(q *Msg) (d *Msg, err os.Error) {
	var (
		c    net.Conn
		in   *Msg
		port string
	)
	if len(res.Servers) == 0 {
		return nil, &Error{Error: "No servers defined"}
	}
	// len(res.Server) == 0 can be perfectly valid, when setting up the resolver
	// It is now
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	if q.Id == 0 {
		// No Id sed, set it
                q.Id = Id()
	}
	sending, ok := q.Pack()
	if !ok {
		return nil, &Error{Error: packErr}
	}

	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		if res.Tcp {
			c, err = net.Dial("tcp", "", server)
		} else {
			c, err = net.Dial("udp", "", server)
		}
		if err != nil {
			continue
		}
		if res.Tcp {
			in, err = exchangeTCP(c, sending, res, true)
		} else {
			in, err = exchangeUDP(c, sending, res, true)
		}

		// Check id in.id != out.id, should be checked in the client!
		c.Close()
		if err != nil {
			continue
		}
		break
	}
	if err != nil {
		return nil, err
	}
	return in, nil
}

// Xfr is used in communicating with *xfr functions.
// This structure is returned on the channel.
type Xfr struct {
	Add bool // true is to be added, otherwise false
	RR
}

// Start an IXFR, q should contain a *Msg with the question
// for an IXFR: "miek.nl" ANY IXFR. RRs that should be added
// have Xfr.Add set to true otherwise it is false.
// Channel m is closed when the IXFR ends.
func (res *Resolver) Ixfr(q *Msg, m chan Xfr) {
	var port string
	var err os.Error
	var in *Msg
	var x Xfr
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	var _ = err // TODO(mg)

	if q.Id == 0 {
                q.Id = Id()
	}

	defer close(m)
	sending, ok := q.Pack()
	if !ok {
		return
	}

Server:
	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		c, cerr := net.Dial("tcp", "", server)
		if cerr != nil {
			err = cerr
			continue Server
		}
		first := true
		var serial uint32 // The first serial seen is the current server serial
		var _ = serial

		defer c.Close()
		for {
			if first {
				in, cerr = exchangeTCP(c, sending, res, true)
			} else {
				in, err = exchangeTCP(c, sending, res, false)
			}

			if cerr != nil {
				// Failed to send, try the next
				err = cerr
				c.Close()
				continue Server
			}
			if in.Id != q.Id {
				return
			}

			if first {
				// A single SOA RR signals "no changes"
				if len(in.Answer) == 1 && checkAxfrSOA(in, true) {
					return
				}

				// But still check if the returned answer is ok
				if !checkAxfrSOA(in, true) {
					c.Close()
					continue Server
				}
				// This serial is important
				serial = in.Answer[0].(*RR_SOA).Serial
				first = !first
			}

			// Now we need to check each message for SOA records, to see what we need to do
			x.Add = true
			if !first {
				for k, r := range in.Answer {
					// If the last record in the IXFR contains the servers' SOA,  we should quit
					if r.Header().Rrtype == TypeSOA {
						switch {
						case r.(*RR_SOA).Serial == serial:
							if k == len(in.Answer)-1 {
								// last rr is SOA with correct serial
								//m <- r dont' send it
								return
							}
							x.Add = true
							if k != 0 {
								// Intermediate SOA
								continue
							}
						case r.(*RR_SOA).Serial != serial:
							x.Add = false
							continue // Don't need to see this SOA
						}
					}
					x.RR = r
					m <- x
				}
			}
			return
		}
		panic("not reached")
		return
	}
	return
}

// Start an AXFR, q should contain a message with the question
// for an AXFR: "miek.nl" ANY AXFR. The closing SOA isn't
// returned over the channel, so the caller will receive
// the zone as-is. Xfr.Add is always true.
// The channel is closed to signal the end of the AXFR.
func (res *Resolver) Axfr(q *Msg, m chan Xfr) {
	var port string
	var err os.Error
	var in *Msg
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	var _ = err // TODO(mg)

	if q.Id == 0 {
		q.Id = Id()
	}

	defer close(m)
	sending, ok := q.Pack()
	if !ok {
		return
	}
Server:
	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		c, cerr := net.Dial("tcp", "", server)
		if cerr != nil {
			err = cerr
			continue Server
		}
		first := true
		defer c.Close() // TODO(mg): if not open?
		for {
			if first {
				in, cerr = exchangeTCP(c, sending, res, true)
			} else {
				in, err = exchangeTCP(c, sending, res, false)
			}

			if cerr != nil {
				// Failed to send, try the next
				err = cerr
				c.Close()
				println("AGIAIN")
				continue Server
			}
			if in.Id != q.Id {
				c.Close()
				return
			}

			if first {
				if !checkAxfrSOA(in, true) {
					c.Close()
					continue Server
				}
				first = !first
			}

			if !first {
				if !checkAxfrSOA(in, false) {
					// Soa record not the last one
					sendFromMsg(in, m, false)
					continue
				} else {
					sendFromMsg(in, m, true)
					return
				}
			}
		}
		panic("not reached")
		return
	}
	return
}

// Send a request on the connection and hope for a reply.
// Up to res.Attempts attempts. If send is false, nothing
// is send.
func exchangeUDP(c net.Conn, m []byte, r *Resolver, send bool) (*Msg, os.Error) {
	var timeout int64
	var attempts int
	if r.Mangle != nil {
		m = r.Mangle(m)
	}
	if r.Timeout == 0 {
		timeout = 1
	} else {
		timeout = int64(r.Timeout)
	}
	if r.Attempts == 0 {
		attempts = 1
	} else {
		attempts = r.Attempts
	}
	for a := 0; a < attempts; a++ {
		if send {
			err := sendUDP(m, c)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return nil, err
			}
		}

		c.SetReadTimeout(timeout * 1e9) // nanoseconds
		buf, err := recvUDP(c)
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Timeout() {
				continue
			}
			return nil, err
		}

		in := new(Msg)
		if !in.Unpack(buf) {
			continue
		}
		return in, nil
	}
	return nil, &Error{Error: servErr}
}

// Up to res.Attempts attempts.
func exchangeTCP(c net.Conn, m []byte, r *Resolver, send bool) (*Msg, os.Error) {
	var timeout int64
	var attempts int
	if r.Mangle != nil {
		m = r.Mangle(m)
	}
	if r.Timeout == 0 {
		timeout = 1
	} else {
		timeout = int64(r.Timeout)
	}
	if r.Attempts == 0 {
		attempts = 1
	} else {
		attempts = r.Attempts
	}

	for a := 0; a < attempts; a++ {
		// only send something when told so
		if send {
			err := sendTCP(m, c)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return nil, err
			}
		}

		c.SetReadTimeout(timeout * 1e9) // nanoseconds
		// The server replies with two bytes length
		buf, err := recvTCP(c)
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Timeout() {
				continue
			}
			return nil, err
		}
		in := new(Msg)
		if !in.Unpack(buf) {
			continue
		}
		return in, nil
	}
	return nil, &Error{Error: servErr}
}

func sendUDP(m []byte, c net.Conn) os.Error {
	_, err := c.Write(m)
	if err != nil {
		return err
	}
	return nil
}

func recvUDP(c net.Conn) ([]byte, os.Error) {
	m := make([]byte, DefaultMsgSize) // More than enough???
	n, err := c.Read(m)
	if err != nil {
		return nil, err
	}
	m = m[:n]
	return m, nil
}

func sendTCP(m []byte, c net.Conn) os.Error {
	l := make([]byte, 2)
	l[0] = byte(len(m) >> 8)
	l[1] = byte(len(m))
	// First we send the length
	_, err := c.Write(l)
	if err != nil {
		return err
	}
	// And the the message
	_, err = c.Write(m)
	if err != nil {
		return err
	}
	return nil
}

func recvTCP(c net.Conn) ([]byte, os.Error) {
	l := make([]byte, 2) // receiver length
	// The server replies with two bytes length
	_, err := c.Read(l)
	if err != nil {
		return nil, err
	}
	length := uint16(l[0])<<8 | uint16(l[1])
	if length == 0 {
		return nil, &Error{Error: "received nil msg length", Server: c.RemoteAddr().String()}
	}
	m := make([]byte, length)
	n, cerr := c.Read(m)
	if cerr != nil {
		return nil, cerr
	}
	i := n
	for i < int(length) {
		n, err = c.Read(m[i:])
		if err != nil {
			return nil, err
		}
		i += n
	}
	return m, nil
}

// Check if he SOA record exists in the Answer section of
// the packet. If first is true the first RR must be a soa
// if false, the last one should be a SOA
func checkAxfrSOA(in *Msg, first bool) bool {
	if len(in.Answer) > 0 {
		if first {
			return in.Answer[0].Header().Rrtype == TypeSOA
		} else {
			return in.Answer[len(in.Answer)-1].Header().Rrtype == TypeSOA
		}
	}
	return false
}

// Send the answer section to the channel
func sendFromMsg(in *Msg, c chan Xfr, nosoa bool) {
	x := Xfr{Add: true}
	for k, r := range in.Answer {
		if nosoa && k == len(in.Answer)-1 {
			continue
		}
		x.RR = r
		c <- x
	}
}
<commit_msg>Remember rrt in the resolver<commit_after>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// DNS resolver client: see RFC 1035.

package dns

import (
	"os"
	"net"
	"time"
)

const packErr = "Failed to pack message"
const servErr = "No servers could be reached"

type Resolver struct {
	Servers  []string            // servers to use
	Search   []string            // suffixes to append to local name
	Port     string              // what port to use
	Ndots    int                 // number of dots in name to trigger absolute lookup -- TODO
	Timeout  int                 // seconds before giving up on packet
	Attempts int                 // lost packets before giving up on server
	Rotate   bool                // round robin among servers -- TODO
	Tcp      bool                // use TCP
	Mangle   func([]byte) []byte // mangle the packet
	Rtt      map[string]int64    // Store round trip times
}

// Basic usage pattern for setting up a resolver:
//
//        res := new(Resolver)
//        res.Servers = []string{"127.0.0.1"}           // set the nameserver
//
//        m := new(Msg)                                 // prepare a new message
//        m.MsgHdr.Recursion_desired = true             // header bits
//        m.Question = make([]Question, 1)              // 1 RR in question section
//        m.Question[0] = Question{"miek.nl", TypeSOA, ClassINET}
//        in, err := res.Query(m)                       // Ask the question
//
// Note that message id checking is left to the caller.
func (res *Resolver) Query(q *Msg) (d *Msg, err os.Error) {
	var (
		c    net.Conn
		in   *Msg
		port string
	)
	if len(res.Servers) == 0 {
		return nil, &Error{Error: "No servers defined"}
	}
        if res.Rtt == nil {
                res.Rtt = make(map[string]int64)
        }
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	if q.Id == 0 {
		// No Id sed, set it
		q.Id = Id()
	}
	sending, ok := q.Pack()
	if !ok {
		return nil, &Error{Error: packErr}
	}

	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		t := time.Nanoseconds()
		if res.Tcp {
			c, err = net.Dial("tcp", "", server)
		} else {
			c, err = net.Dial("udp", "", server)
		}
		if err != nil {
			continue
		}
		if res.Tcp {
			in, err = exchangeTCP(c, sending, res, true)
		} else {
			in, err = exchangeUDP(c, sending, res, true)
		}
                res.Rtt[server] = time.Nanoseconds() - t

		// Check id in.id != out.id, should be checked in the client!
		c.Close()
		if err != nil {
			continue
		}
		break
	}
	if err != nil {
		return nil, err
	}
	return in, nil
}

// Xfr is used in communicating with *xfr functions.
// This structure is returned on the channel.
type Xfr struct {
	Add bool // true is to be added, otherwise false
	RR
}

// Start an IXFR, q should contain a *Msg with the question
// for an IXFR: "miek.nl" ANY IXFR. RRs that should be added
// have Xfr.Add set to true otherwise it is false.
// Channel m is closed when the IXFR ends.
func (res *Resolver) Ixfr(q *Msg, m chan Xfr) {
	var port string
	var err os.Error
	var in *Msg
	var x Xfr
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	var _ = err // TODO(mg)

	if q.Id == 0 {
		q.Id = Id()
	}

	defer close(m)
	sending, ok := q.Pack()
	if !ok {
		return
	}

Server:
	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		c, cerr := net.Dial("tcp", "", server)
		if cerr != nil {
			err = cerr
			continue Server
		}
		first := true
		var serial uint32 // The first serial seen is the current server serial

		defer c.Close()
		for {
			if first {
				in, cerr = exchangeTCP(c, sending, res, true)
			} else {
				in, err = exchangeTCP(c, sending, res, false)
			}

			if cerr != nil {
				// Failed to send, try the next
				err = cerr
				c.Close()
				continue Server
			}
			if in.Id != q.Id {
				return
			}

			if first {
				// A single SOA RR signals "no changes"
				if len(in.Answer) == 1 && checkAxfrSOA(in, true) {
					return
				}

				// But still check if the returned answer is ok
				if !checkAxfrSOA(in, true) {
					c.Close()
					continue Server
				}
				// This serial is important
				serial = in.Answer[0].(*RR_SOA).Serial
				first = !first
			}

			// Now we need to check each message for SOA records, to see what we need to do
			x.Add = true
			if !first {
				for k, r := range in.Answer {
					// If the last record in the IXFR contains the servers' SOA,  we should quit
					if r.Header().Rrtype == TypeSOA {
						switch {
						case r.(*RR_SOA).Serial == serial:
							if k == len(in.Answer)-1 {
								// last rr is SOA with correct serial
								//m <- r dont' send it
								return
							}
							x.Add = true
							if k != 0 {
								// Intermediate SOA
								continue
							}
						case r.(*RR_SOA).Serial != serial:
							x.Add = false
							continue // Don't need to see this SOA
						}
					}
					x.RR = r
					m <- x
				}
			}
			return
		}
		panic("not reached")
		return
	}
	return
}

// Start an AXFR, q should contain a message with the question
// for an AXFR: "miek.nl" ANY AXFR. The closing SOA isn't
// returned over the channel, so the caller will receive
// the zone as-is. Xfr.Add is always true.
// The channel is closed to signal the end of the AXFR.
func (res *Resolver) Axfr(q *Msg, m chan Xfr) {
	var port string
	var err os.Error
	var in *Msg
	if res.Port == "" {
		port = "53"
	} else {
		port = res.Port
	}

	var _ = err // TODO(mg)

	if q.Id == 0 {
		q.Id = Id()
	}

	defer close(m)
	sending, ok := q.Pack()
	if !ok {
		return
	}
Server:
	for i := 0; i < len(res.Servers); i++ {
		server := res.Servers[i] + ":" + port
		c, cerr := net.Dial("tcp", "", server)
		if cerr != nil {
			err = cerr
			continue Server
		}
		first := true
		defer c.Close() // TODO(mg): if not open?
		for {
			if first {
				in, cerr = exchangeTCP(c, sending, res, true)
			} else {
				in, err = exchangeTCP(c, sending, res, false)
			}

			if cerr != nil {
				// Failed to send, try the next
				err = cerr
				c.Close()
				println("AGIAIN")
				continue Server
			}
			if in.Id != q.Id {
				c.Close()
				return
			}

			if first {
				if !checkAxfrSOA(in, true) {
					c.Close()
					continue Server
				}
				first = !first
			}

			if !first {
				if !checkAxfrSOA(in, false) {
					// Soa record not the last one
					sendFromMsg(in, m, false)
					continue
				} else {
					sendFromMsg(in, m, true)
					return
				}
			}
		}
		panic("not reached")
		return
	}
	return
}

// Send a request on the connection and hope for a reply.
// Up to res.Attempts attempts. If send is false, nothing
// is send.
func exchangeUDP(c net.Conn, m []byte, r *Resolver, send bool) (*Msg, os.Error) {
	var timeout int64
	var attempts int
	if r.Mangle != nil {
		m = r.Mangle(m)
	}
	if r.Timeout == 0 {
		timeout = 1
	} else {
		timeout = int64(r.Timeout)
	}
	if r.Attempts == 0 {
		attempts = 1
	} else {
		attempts = r.Attempts
	}
	for a := 0; a < attempts; a++ {
		if send {
			err := sendUDP(m, c)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return nil, err
			}
		}

		c.SetReadTimeout(timeout * 1e9) // nanoseconds
		buf, err := recvUDP(c)
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Timeout() {
				continue
			}
			return nil, err
		}

		in := new(Msg)
		if !in.Unpack(buf) {
			continue
		}
		return in, nil
	}
	return nil, &Error{Error: servErr}
}

// Up to res.Attempts attempts.
func exchangeTCP(c net.Conn, m []byte, r *Resolver, send bool) (*Msg, os.Error) {
	var timeout int64
	var attempts int
	if r.Mangle != nil {
		m = r.Mangle(m)
	}
	if r.Timeout == 0 {
		timeout = 1
	} else {
		timeout = int64(r.Timeout)
	}
	if r.Attempts == 0 {
		attempts = 1
	} else {
		attempts = r.Attempts
	}

	for a := 0; a < attempts; a++ {
		// only send something when told so
		if send {
			err := sendTCP(m, c)
			if err != nil {
				if e, ok := err.(net.Error); ok && e.Timeout() {
					continue
				}
				return nil, err
			}
		}

		c.SetReadTimeout(timeout * 1e9) // nanoseconds
		// The server replies with two bytes length
		buf, err := recvTCP(c)
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Timeout() {
				continue
			}
			return nil, err
		}
		in := new(Msg)
		if !in.Unpack(buf) {
			continue
		}
		return in, nil
	}
	return nil, &Error{Error: servErr}
}

func sendUDP(m []byte, c net.Conn) os.Error {
	_, err := c.Write(m)
	if err != nil {
		return err
	}
	return nil
}

func recvUDP(c net.Conn) ([]byte, os.Error) {
	m := make([]byte, DefaultMsgSize) // More than enough???
	n, err := c.Read(m)
	if err != nil {
		return nil, err
	}
	m = m[:n]
	return m, nil
}

func sendTCP(m []byte, c net.Conn) os.Error {
	l := make([]byte, 2)
	l[0] = byte(len(m) >> 8)
	l[1] = byte(len(m))
	// First we send the length
	_, err := c.Write(l)
	if err != nil {
		return err
	}
	// And the the message
	_, err = c.Write(m)
	if err != nil {
		return err
	}
	return nil
}

func recvTCP(c net.Conn) ([]byte, os.Error) {
	l := make([]byte, 2) // receiver length
	// The server replies with two bytes length
	_, err := c.Read(l)
	if err != nil {
		return nil, err
	}
	length := uint16(l[0])<<8 | uint16(l[1])
	if length == 0 {
		return nil, &Error{Error: "received nil msg length", Server: c.RemoteAddr().String()}
	}
	m := make([]byte, length)
	n, cerr := c.Read(m)
	if cerr != nil {
		return nil, cerr
	}
	i := n
	for i < int(length) {
		n, err = c.Read(m[i:])
		if err != nil {
			return nil, err
		}
		i += n
	}
	return m, nil
}

// Check if he SOA record exists in the Answer section of
// the packet. If first is true the first RR must be a soa
// if false, the last one should be a SOA
func checkAxfrSOA(in *Msg, first bool) bool {
	if len(in.Answer) > 0 {
		if first {
			return in.Answer[0].Header().Rrtype == TypeSOA
		} else {
			return in.Answer[len(in.Answer)-1].Header().Rrtype == TypeSOA
		}
	}
	return false
}

// Send the answer section to the channel
func sendFromMsg(in *Msg, c chan Xfr, nosoa bool) {
	x := Xfr{Add: true}
	for k, r := range in.Answer {
		if nosoa && k == len(in.Answer)-1 {
			continue
		}
		x.RR = r
		c <- x
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (C) 2013  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package receiver

import (
	"net"
	"net/http"
	"os"
	"os/signal"

	"github.com/cider/go-cider/cider/services/logging"
	"github.com/cider/go-cider/cider/services/pubsub"
	zlogging "github.com/cider/go-cider/cider/transports/zmq3/logging"
	zpubsub "github.com/cider/go-cider/cider/transports/zmq3/pubsub"

	zmq "github.com/pebbe/zmq3"
)

// API functions ---------------------------------------------------------------

var (
	Logger *logging.Service
	PubSub *pubsub.Service
)

// Serve POST requests using the handler passed into ListenAndServe.
// This function blocks until a signal is received. So signals are being
// handled by this function, no need to do it manually.
func ListenAndServe(handler http.Handler) {
	// Load all the required environment variables, panic if any is not set.
	// This is placed here and not outside to make testing easier (possible).
	// The applications do not have to really connect to Cider to run tests.
	var (
		alias = mustBeSet(os.Getenv("CIDER_ALIAS"))
		addr  = mustBeSet(os.Getenv("LISTEN_ADDRESS"))
		token = mustBeSet(os.Getenv("ACCESS_TOKEN"))
	)

	// Start catching interrupts.
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt)

	// Initialise Logging service from environmental variables.
	var err error
	Logger, err = logging.NewService(func() (logging.Transport, error) {
		factory := zlogging.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_LOGGING_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(err)
	}
	Logger.Info("Logging service initialised")

	// Make sure ZeroMQ is terminated properly.
	defer func() {
		Logger.Info("Waiting for ZeroMQ context to terminate...")
		Logger.Close()
		zmq.Term()
	}()

	// Initialise PubSub service from environmental variables.
	PubSub, err = pubsub.NewService(func() (pubsub.Transport, error) {
		factory := zpubsub.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_PUBSUB_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(Logger.Critical(err))
	}
	defer PubSub.Close()
	Logger.Info("PubSub service initialised")

	// Listen.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(Logger.Critical(err))
	}

	// Start processing interrupts.
	var interrupted bool
	go func() {
		<-signalCh
		interrupted = true
		listener.Close()
	}()

	// Keep serving until interrupted.
	err = http.Serve(listener, authenticatedServer(token, handler))
	if err != nil && !interrupted {
		panic(Logger.Critical(err))
	}
}

// Helpers ---------------------------------------------------------------------

func mustBeSet(v string) string {
	if v == "" {
		panic("Required variable is not set")
	}
	return v
}

func authenticatedServer(token string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the POST method only.
		if r.Method != "POST" {
			http.Error(w, "POST Method Expected", http.StatusMethodNotAllowed)
			return
		}

		// Make sure that the token query parameter is set correctly.
		if r.FormValue("token") != token {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		// If everything is ok, serve the user-defined handler.
		handler.ServeHTTP(w, r)
	})
}
<commit_msg>Make sure processing signals is thread-safe<commit_after>/*
   Copyright (C) 2013  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package receiver

import (
	"net"
	"net/http"
	"os"
	"os/signal"

	"github.com/cider/go-cider/cider/services/logging"
	"github.com/cider/go-cider/cider/services/pubsub"
	zlogging "github.com/cider/go-cider/cider/transports/zmq3/logging"
	zpubsub "github.com/cider/go-cider/cider/transports/zmq3/pubsub"

	zmq "github.com/pebbe/zmq3"
)

// API functions ---------------------------------------------------------------

var (
	Logger *logging.Service
	PubSub *pubsub.Service
)

// Serve POST requests using the handler passed into ListenAndServe.
// This function blocks until a signal is received. So signals are being
// handled by this function, no need to do it manually.
func ListenAndServe(handler http.Handler) {
	// Load all the required environment variables, panic if any is not set.
	// This is placed here and not outside to make testing easier (possible).
	// The applications do not have to really connect to Cider to run tests.
	var (
		alias = mustBeSet(os.Getenv("CIDER_ALIAS"))
		addr  = mustBeSet(os.Getenv("LISTEN_ADDRESS"))
		token = mustBeSet(os.Getenv("ACCESS_TOKEN"))
	)

	// Start catching interrupts.
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt)

	// Initialise Logging service from environmental variables.
	var err error
	Logger, err = logging.NewService(func() (logging.Transport, error) {
		factory := zlogging.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_LOGGING_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(err)
	}
	Logger.Info("Logging service initialised")

	// Make sure ZeroMQ is terminated properly.
	defer func() {
		Logger.Info("Waiting for ZeroMQ context to terminate...")
		Logger.Close()
		zmq.Term()
	}()

	// Initialise PubSub service from environmental variables.
	PubSub, err = pubsub.NewService(func() (pubsub.Transport, error) {
		factory := zpubsub.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_PUBSUB_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(Logger.Critical(err))
	}
	defer PubSub.Close()
	Logger.Info("PubSub service initialised")

	// Listen.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(Logger.Critical(err))
	}

	// Start processing interrupts.
	interruptedCh := make(chan bool, 1)
	go func() {
		<-signalCh
		interruptedCh <- true
		listener.Close()
	}()

	// Keep serving until interrupted.
	err = http.Serve(listener, authenticatedServer(token, handler))
	if err != nil {
		select {
		case <-interruptedCh:
		default:
			panic(Logger.Critical(err))
		}
	}
}

// Helpers ---------------------------------------------------------------------

func mustBeSet(v string) string {
	if v == "" {
		panic("Required variable is not set")
	}
	return v
}

func authenticatedServer(token string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the POST method only.
		if r.Method != "POST" {
			http.Error(w, "POST Method Expected", http.StatusMethodNotAllowed)
			return
		}

		// Make sure that the token query parameter is set correctly.
		if r.FormValue("token") != token {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		// If everything is ok, serve the user-defined handler.
		handler.ServeHTTP(w, r)
	})
}
<|endoftext|>