<commit_before>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/m13253/dns-over-https/json-dns"
	"github.com/miekg/dns"
	"golang.org/x/net/http2"
	"golang.org/x/net/idna"
)

type Client struct {
	conf                 *config
	bootstrap            []string
	passthrough          []string
	udpClient            *dns.Client
	tcpClient            *dns.Client
	udpServers           []*dns.Server
	tcpServers           []*dns.Server
	bootstrapResolver    *net.Resolver
	cookieJar            http.CookieJar
	httpClientMux        *sync.RWMutex
	httpTransport        *http.Transport
	httpClient           *http.Client
	httpClientLastCreate time.Time
}

type DNSRequest struct {
	response          *http.Response
	reply             *dns.Msg
	udpSize           uint16
	ednsClientAddress net.IP
	ednsClientNetmask uint8
	currentUpstream   string
	err               error
}

func NewClient(conf *config) (c *Client, err error) {
	c = &Client{
		conf: conf,
	}

	udpHandler := dns.HandlerFunc(c.udpHandlerFunc)
	tcpHandler := dns.HandlerFunc(c.tcpHandlerFunc)
	c.udpClient = &dns.Client{
		Net:     "udp",
		UDPSize: dns.DefaultMsgSize,
		Timeout: time.Duration(conf.Timeout) * time.Second,
	}
	c.tcpClient = &dns.Client{
		Net:     "tcp",
		Timeout: time.Duration(conf.Timeout) * time.Second,
	}
	for _, addr := range conf.Listen {
		c.udpServers = append(c.udpServers, &dns.Server{
			Addr:    addr,
			Net:     "udp",
			Handler: udpHandler,
			UDPSize: dns.DefaultMsgSize,
		})
		c.tcpServers = append(c.tcpServers, &dns.Server{
			Addr:    addr,
			Net:     "tcp",
			Handler: tcpHandler,
		})
	}
	c.bootstrapResolver = net.DefaultResolver
	if len(conf.Bootstrap) != 0 {
		c.bootstrap = make([]string, len(conf.Bootstrap))
		for i, bootstrap := range conf.Bootstrap {
			bootstrapAddr, err := net.ResolveUDPAddr("udp", bootstrap)
			if err != nil {
				bootstrapAddr, err = net.ResolveUDPAddr("udp", "["+bootstrap+"]:53")
			}
			if err != nil {
				return nil, err
			}
			c.bootstrap[i] = bootstrapAddr.String()
		}
		c.bootstrapResolver = &net.Resolver{
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				var d net.Dialer
				numServers := len(c.bootstrap)
				bootstrap := c.bootstrap[rand.Intn(numServers)]
				conn, err := d.DialContext(ctx, network, bootstrap)
				return conn, err
			},
		}
		if len(conf.Passthrough) != 0 {
			c.passthrough = make([]string, len(conf.Passthrough))
			for i, passthrough := range conf.Passthrough {
				if punycode, err := idna.ToASCII(passthrough); err != nil {
					passthrough = punycode
				}
				c.passthrough[i] = "." + strings.ToLower(strings.Trim(passthrough, ".")) + "."
			}
		}
	}
	// Most CDNs require Cookie support to prevent DDoS attack.
	// Disabling Cookie does not effectively prevent tracking,
	// so I will leave it on to make anti-DDoS services happy.
	if !c.conf.NoCookies {
		c.cookieJar, err = cookiejar.New(nil)
		if err != nil {
			return nil, err
		}
	} else {
		c.cookieJar = nil
	}

	c.httpClientMux = new(sync.RWMutex)
	err = c.newHTTPClient()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c *Client) newHTTPClient() error {
	c.httpClientMux.Lock()
	defer c.httpClientMux.Unlock()
	if !c.httpClientLastCreate.IsZero() && time.Now().Sub(c.httpClientLastCreate) < time.Duration(c.conf.Timeout)*time.Second {
		return nil
	}
	if c.httpTransport != nil {
		c.httpTransport.CloseIdleConnections()
	}
	dialer := &net.Dialer{
		Timeout:   time.Duration(c.conf.Timeout) * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
		Resolver:  c.bootstrapResolver,
	}
	c.httpTransport = &http.Transport{
		DialContext:           dialer.DialContext,
		ExpectContinueTimeout: 1 * time.Second,
		IdleConnTimeout:       90 * time.Second,
		MaxIdleConns:          100,
		MaxIdleConnsPerHost:   10,
		Proxy:                 http.ProxyFromEnvironment,
		TLSHandshakeTimeout:   time.Duration(c.conf.Timeout) * time.Second,
	}
	if c.conf.NoIPv6 {
		c.httpTransport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
			if strings.HasPrefix(network, "tcp") {
				network = "tcp4"
			}
			return dialer.DialContext(ctx, network, address)
		}
	}
	err := http2.ConfigureTransport(c.httpTransport)
	if err != nil {
		return err
	}
	c.httpClient = &http.Client{
		Transport: c.httpTransport,
		Jar:       c.cookieJar,
	}
	c.httpClientLastCreate = time.Now()
	return nil
}

func (c *Client) Start() error {
	results := make(chan error, len(c.udpServers)+len(c.tcpServers))
	for _, srv := range append(c.udpServers, c.tcpServers...) {
		go func(srv *dns.Server) {
			err := srv.ListenAndServe()
			if err != nil {
				log.Println(err)
			}
			results <- err
		}(srv)
	}

	for i := 0; i < cap(results); i++ {
		err := <-results
		if err != nil {
			return err
		}
	}
	close(results)
	return nil
}

func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.conf.Timeout)*time.Second)
	defer cancel()

	if r.Response == true {
		log.Println("Received a response packet")
		return
	}

	if len(r.Question) != 1 {
		log.Println("Number of questions is not 1")
		reply := jsonDNS.PrepareReply(r)
		reply.Rcode = dns.RcodeFormatError
		w.WriteMsg(reply)
		return
	}
	question := &r.Question[0]
	questionName := question.Name
	questionClass := ""
	if qclass, ok := dns.ClassToString[question.Qclass]; ok {
		questionClass = qclass
	} else {
		questionClass = strconv.FormatUint(uint64(question.Qclass), 10)
	}
	questionType := ""
	if qtype, ok := dns.TypeToString[question.Qtype]; ok {
		questionType = qtype
	} else {
		questionType = strconv.FormatUint(uint64(question.Qtype), 10)
	}
	if c.conf.Verbose {
		fmt.Printf("%s - - [%s] \"%s %s %s\"\n", w.RemoteAddr(), time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
	}

	shouldPassthrough := false
	passthroughQuestionName := questionName
	if punycode, err := idna.ToASCII(passthroughQuestionName); err != nil {
		passthroughQuestionName = punycode
	}
	passthroughQuestionName = "." + strings.ToLower(strings.Trim(passthroughQuestionName, ".")) + "."
	for _, passthrough := range c.passthrough {
		if strings.HasSuffix(passthroughQuestionName, passthrough) {
			shouldPassthrough = true
			break
		}
	}
	if shouldPassthrough {
		numServers := len(c.bootstrap)
		upstream := c.bootstrap[rand.Intn(numServers)]
		log.Printf("Request \"%s %s %s\" is passed through %s.\n", questionName, questionClass, questionType, upstream)
		var reply *dns.Msg
		var err error
		if !isTCP {
			reply, _, err = c.udpClient.Exchange(r, upstream)
		} else {
			reply, _, err = c.tcpClient.Exchange(r, upstream)
		}
		if err == nil || err == dns.ErrTruncated {
			w.WriteMsg(reply)
			return
		}
		log.Println(err)
		reply = jsonDNS.PrepareReply(r)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	requestType := ""
	if len(c.conf.UpstreamIETF) == 0 {
		requestType = "application/dns-json"
	} else if len(c.conf.UpstreamGoogle) == 0 {
		requestType = "application/dns-message"
	} else {
		numServers := len(c.conf.UpstreamGoogle) + len(c.conf.UpstreamIETF)
		random := rand.Intn(numServers)
		if random < len(c.conf.UpstreamGoogle) {
			requestType = "application/dns-json"
		} else {
			requestType = "application/dns-message"
		}
	}

	var req *DNSRequest
	if requestType == "application/dns-json" {
		req = c.generateRequestGoogle(ctx, w, r, isTCP)
	} else if requestType == "application/dns-message" {
		req = c.generateRequestIETF(ctx, w, r, isTCP)
	} else {
		panic("Unknown request Content-Type")
	}

	if req.response != nil {
		defer req.response.Body.Close()
		for _, header := range c.conf.DebugHTTPHeaders {
			if value := req.response.Header.Get(header); value != "" {
				log.Printf("%s: %s\n", header, value)
			}
		}
	}
	if req.err != nil {
		return
	}

	contentType := ""
	candidateType := strings.SplitN(req.response.Header.Get("Content-Type"), ";", 2)[0]
	if candidateType == "application/json" {
		contentType = "application/json"
	} else if candidateType == "application/dns-message" {
		contentType = "application/dns-message"
	} else if candidateType == "application/dns-udpwireformat" {
		contentType = "application/dns-message"
	} else {
		if requestType == "application/dns-json" {
			contentType = "application/json"
		} else if requestType == "application/dns-message" {
			contentType = "application/dns-message"
		}
	}

	if contentType == "application/json" {
		c.parseResponseGoogle(ctx, w, r, isTCP, req)
	} else if contentType == "application/dns-message" {
		c.parseResponseIETF(ctx, w, r, isTCP, req)
	} else {
		panic("Unknown response Content-Type")
	}
}

func (c *Client) udpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, false)
}

func (c *Client) tcpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, true)
}

var (
	ipv4Mask24 = net.IPMask{255, 255, 255, 0}
	ipv6Mask56 = net.CIDRMask(56, 128)
)

func (c *Client) findClientIP(w dns.ResponseWriter, r *dns.Msg) (ednsClientAddress net.IP, ednsClientNetmask uint8) {
	ednsClientNetmask = 255
	if c.conf.NoECS {
		return net.IPv4(0, 0, 0, 0), 0
	}
	if opt := r.IsEdns0(); opt != nil {
		for _, option := range opt.Option {
			if option.Option() == dns.EDNS0SUBNET {
				edns0Subnet := option.(*dns.EDNS0_SUBNET)
				ednsClientAddress = edns0Subnet.Address
				ednsClientNetmask = edns0Subnet.SourceNetmask
				return
			}
		}
	}
	remoteAddr, err := net.ResolveUDPAddr("udp", w.RemoteAddr().String())
	if err != nil {
		return
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		if ipv4 := ip.To4(); ipv4 != nil {
			ednsClientAddress = ipv4.Mask(ipv4Mask24)
			ednsClientNetmask = 24
		} else {
			ednsClientAddress = ip.Mask(ipv6Mask56)
			ednsClientNetmask = 56
		}
	}
	return
}
<commit_msg>Use time.Since to replace time.Now().Sub<commit_after>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/m13253/dns-over-https/json-dns"
	"github.com/miekg/dns"
	"golang.org/x/net/http2"
	"golang.org/x/net/idna"
)

type Client struct {
	conf                 *config
	bootstrap            []string
	passthrough          []string
	udpClient            *dns.Client
	tcpClient            *dns.Client
	udpServers           []*dns.Server
	tcpServers           []*dns.Server
	bootstrapResolver    *net.Resolver
	cookieJar            http.CookieJar
	httpClientMux        *sync.RWMutex
	httpTransport        *http.Transport
	httpClient           *http.Client
	httpClientLastCreate time.Time
}

type DNSRequest struct {
	response          *http.Response
	reply             *dns.Msg
	udpSize           uint16
	ednsClientAddress net.IP
	ednsClientNetmask uint8
	currentUpstream   string
	err               error
}

func NewClient(conf *config) (c *Client, err error) {
	c = &Client{
		conf: conf,
	}

	udpHandler := dns.HandlerFunc(c.udpHandlerFunc)
	tcpHandler := dns.HandlerFunc(c.tcpHandlerFunc)
	c.udpClient = &dns.Client{
		Net:     "udp",
		UDPSize: dns.DefaultMsgSize,
		Timeout: time.Duration(conf.Timeout) * time.Second,
	}
	c.tcpClient = &dns.Client{
		Net:     "tcp",
		Timeout: time.Duration(conf.Timeout) * time.Second,
	}
	for _, addr := range conf.Listen {
		c.udpServers = append(c.udpServers, &dns.Server{
			Addr:    addr,
			Net:     "udp",
			Handler: udpHandler,
			UDPSize: dns.DefaultMsgSize,
		})
		c.tcpServers = append(c.tcpServers, &dns.Server{
			Addr:    addr,
			Net:     "tcp",
			Handler: tcpHandler,
		})
	}
	c.bootstrapResolver = net.DefaultResolver
	if len(conf.Bootstrap) != 0 {
		c.bootstrap = make([]string, len(conf.Bootstrap))
		for i, bootstrap := range conf.Bootstrap {
			bootstrapAddr, err := net.ResolveUDPAddr("udp", bootstrap)
			if err != nil {
				bootstrapAddr, err = net.ResolveUDPAddr("udp", "["+bootstrap+"]:53")
			}
			if err != nil {
				return nil, err
			}
			c.bootstrap[i] = bootstrapAddr.String()
		}
		c.bootstrapResolver = &net.Resolver{
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				var d net.Dialer
				numServers := len(c.bootstrap)
				bootstrap := c.bootstrap[rand.Intn(numServers)]
				conn, err := d.DialContext(ctx, network, bootstrap)
				return conn, err
			},
		}
		if len(conf.Passthrough) != 0 {
			c.passthrough = make([]string, len(conf.Passthrough))
			for i, passthrough := range conf.Passthrough {
				if punycode, err := idna.ToASCII(passthrough); err != nil {
					passthrough = punycode
				}
				c.passthrough[i] = "." + strings.ToLower(strings.Trim(passthrough, ".")) + "."
			}
		}
	}
	// Most CDNs require Cookie support to prevent DDoS attack.
	// Disabling Cookie does not effectively prevent tracking,
	// so I will leave it on to make anti-DDoS services happy.
	if !c.conf.NoCookies {
		c.cookieJar, err = cookiejar.New(nil)
		if err != nil {
			return nil, err
		}
	} else {
		c.cookieJar = nil
	}

	c.httpClientMux = new(sync.RWMutex)
	err = c.newHTTPClient()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func (c *Client) newHTTPClient() error {
	c.httpClientMux.Lock()
	defer c.httpClientMux.Unlock()
	if !c.httpClientLastCreate.IsZero() && time.Since(c.httpClientLastCreate) < time.Duration(c.conf.Timeout)*time.Second {
		return nil
	}
	if c.httpTransport != nil {
		c.httpTransport.CloseIdleConnections()
	}
	dialer := &net.Dialer{
		Timeout:   time.Duration(c.conf.Timeout) * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
		Resolver:  c.bootstrapResolver,
	}
	c.httpTransport = &http.Transport{
		DialContext:           dialer.DialContext,
		ExpectContinueTimeout: 1 * time.Second,
		IdleConnTimeout:       90 * time.Second,
		MaxIdleConns:          100,
		MaxIdleConnsPerHost:   10,
		Proxy:                 http.ProxyFromEnvironment,
		TLSHandshakeTimeout:   time.Duration(c.conf.Timeout) * time.Second,
	}
	if c.conf.NoIPv6 {
		c.httpTransport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
			if strings.HasPrefix(network, "tcp") {
				network = "tcp4"
			}
			return dialer.DialContext(ctx, network, address)
		}
	}
	err := http2.ConfigureTransport(c.httpTransport)
	if err != nil {
		return err
	}
	c.httpClient = &http.Client{
		Transport: c.httpTransport,
		Jar:       c.cookieJar,
	}
	c.httpClientLastCreate = time.Now()
	return nil
}

func (c *Client) Start() error {
	results := make(chan error, len(c.udpServers)+len(c.tcpServers))
	for _, srv := range append(c.udpServers, c.tcpServers...) {
		go func(srv *dns.Server) {
			err := srv.ListenAndServe()
			if err != nil {
				log.Println(err)
			}
			results <- err
		}(srv)
	}

	for i := 0; i < cap(results); i++ {
		err := <-results
		if err != nil {
			return err
		}
	}
	close(results)
	return nil
}

func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(c.conf.Timeout)*time.Second)
	defer cancel()

	if r.Response {
		log.Println("Received a response packet")
		return
	}

	if len(r.Question) != 1 {
		log.Println("Number of questions is not 1")
		reply := jsonDNS.PrepareReply(r)
		reply.Rcode = dns.RcodeFormatError
		w.WriteMsg(reply)
		return
	}
	question := &r.Question[0]
	questionName := question.Name
	questionClass := ""
	if qclass, ok := dns.ClassToString[question.Qclass]; ok {
		questionClass = qclass
	} else {
		questionClass = strconv.FormatUint(uint64(question.Qclass), 10)
	}
	questionType := ""
	if qtype, ok := dns.TypeToString[question.Qtype]; ok {
		questionType = qtype
	} else {
		questionType = strconv.FormatUint(uint64(question.Qtype), 10)
	}
	if c.conf.Verbose {
		fmt.Printf("%s - - [%s] \"%s %s %s\"\n", w.RemoteAddr(), time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionClass, questionType)
	}

	shouldPassthrough := false
	passthroughQuestionName := questionName
	if punycode, err := idna.ToASCII(passthroughQuestionName); err != nil {
		passthroughQuestionName = punycode
	}
	passthroughQuestionName = "." + strings.ToLower(strings.Trim(passthroughQuestionName, ".")) + "."
	for _, passthrough := range c.passthrough {
		if strings.HasSuffix(passthroughQuestionName, passthrough) {
			shouldPassthrough = true
			break
		}
	}
	if shouldPassthrough {
		numServers := len(c.bootstrap)
		upstream := c.bootstrap[rand.Intn(numServers)]
		log.Printf("Request \"%s %s %s\" is passed through %s.\n", questionName, questionClass, questionType, upstream)
		var reply *dns.Msg
		var err error
		if !isTCP {
			reply, _, err = c.udpClient.Exchange(r, upstream)
		} else {
			reply, _, err = c.tcpClient.Exchange(r, upstream)
		}
		if err == nil || err == dns.ErrTruncated {
			w.WriteMsg(reply)
			return
		}
		log.Println(err)
		reply = jsonDNS.PrepareReply(r)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	requestType := ""
	if len(c.conf.UpstreamIETF) == 0 {
		requestType = "application/dns-json"
	} else if len(c.conf.UpstreamGoogle) == 0 {
		requestType = "application/dns-message"
	} else {
		numServers := len(c.conf.UpstreamGoogle) + len(c.conf.UpstreamIETF)
		random := rand.Intn(numServers)
		if random < len(c.conf.UpstreamGoogle) {
			requestType = "application/dns-json"
		} else {
			requestType = "application/dns-message"
		}
	}

	var req *DNSRequest
	if requestType == "application/dns-json" {
		req = c.generateRequestGoogle(ctx, w, r, isTCP)
	} else if requestType == "application/dns-message" {
		req = c.generateRequestIETF(ctx, w, r, isTCP)
	} else {
		panic("Unknown request Content-Type")
	}

	if req.response != nil {
		defer req.response.Body.Close()
		for _, header := range c.conf.DebugHTTPHeaders {
			if value := req.response.Header.Get(header); value != "" {
				log.Printf("%s: %s\n", header, value)
			}
		}
	}
	if req.err != nil {
		return
	}

	contentType := ""
	candidateType := strings.SplitN(req.response.Header.Get("Content-Type"), ";", 2)[0]
	if candidateType == "application/json" {
		contentType = "application/json"
	} else if candidateType == "application/dns-message" {
		contentType = "application/dns-message"
	} else if candidateType == "application/dns-udpwireformat" {
		contentType = "application/dns-message"
	} else {
		if requestType == "application/dns-json" {
			contentType = "application/json"
		} else if requestType == "application/dns-message" {
			contentType = "application/dns-message"
		}
	}

	if contentType == "application/json" {
		c.parseResponseGoogle(ctx, w, r, isTCP, req)
	} else if contentType == "application/dns-message" {
		c.parseResponseIETF(ctx, w, r, isTCP, req)
	} else {
		panic("Unknown response Content-Type")
	}
}

func (c *Client) udpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, false)
}

func (c *Client) tcpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, true)
}

var (
	ipv4Mask24 = net.IPMask{255, 255, 255, 0}
	ipv6Mask56 = net.CIDRMask(56, 128)
)

func (c *Client) findClientIP(w dns.ResponseWriter, r *dns.Msg) (ednsClientAddress net.IP, ednsClientNetmask uint8) {
	ednsClientNetmask = 255
	if c.conf.NoECS {
		return net.IPv4(0, 0, 0, 0), 0
	}
	if opt := r.IsEdns0(); opt != nil {
		for _, option := range opt.Option {
			if option.Option() == dns.EDNS0SUBNET {
				edns0Subnet := option.(*dns.EDNS0_SUBNET)
				ednsClientAddress = edns0Subnet.Address
				ednsClientNetmask = edns0Subnet.SourceNetmask
				return
			}
		}
	}
	remoteAddr, err := net.ResolveUDPAddr("udp", w.RemoteAddr().String())
	if err != nil {
		return
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		if ipv4 := ip.To4(); ipv4 != nil {
			ednsClientAddress = ipv4.Mask(ipv4Mask24)
			ednsClientNetmask = 24
		} else {
			ednsClientAddress = ip.Mask(ipv6Mask56)
			ednsClientNetmask = 56
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gorilla/handlers"
	jsonDNS "github.com/m13253/dns-over-https/json-dns"
	"github.com/miekg/dns"
)

type Server struct {
	conf         *config
	udpClient    *dns.Client
	tcpClient    *dns.Client
	tcpClientTLS *dns.Client
	servemux     *http.ServeMux
}

type DNSRequest struct {
	request         *dns.Msg
	response        *dns.Msg
	transactionID   uint16
	currentUpstream string
	isTailored      bool
	errcode         int
	errtext         string
}

func NewServer(conf *config) (*Server, error) {
	timeout := time.Duration(conf.Timeout) * time.Second
	s := &Server{
		conf: conf,
		udpClient: &dns.Client{
			Net:     "udp",
			UDPSize: dns.DefaultMsgSize,
			Timeout: timeout,
		},
		tcpClient: &dns.Client{
			Net:     "tcp",
			Timeout: timeout,
		},
		tcpClientTLS: &dns.Client{
			Net:     "tcp-tls",
			Timeout: timeout,
		},
		servemux: http.NewServeMux(),
	}
	if conf.LocalAddr != "" {
		udpLocalAddr, err := net.ResolveUDPAddr("udp", conf.LocalAddr)
		if err != nil {
			return nil, err
		}
		tcpLocalAddr, err := net.ResolveTCPAddr("tcp", conf.LocalAddr)
		if err != nil {
			return nil, err
		}
		s.udpClient.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: udpLocalAddr,
		}
		s.tcpClient.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: tcpLocalAddr,
		}
		s.tcpClientTLS.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: tcpLocalAddr,
		}
	}
	s.servemux.HandleFunc(conf.Path, s.handlerFunc)
	return s, nil
}

func (s *Server) Start() error {
	servemux := http.Handler(s.servemux)
	if s.conf.Verbose {
		servemux = handlers.CombinedLoggingHandler(os.Stdout, servemux)
	}
	results := make(chan error, len(s.conf.Listen))
	for _, addr := range s.conf.Listen {
		go func(addr string) {
			var err error
			if s.conf.Cert != "" || s.conf.Key != "" {
				err = http.ListenAndServeTLS(addr, s.conf.Cert, s.conf.Key, servemux)
			} else {
				err = http.ListenAndServe(addr, servemux)
			}
			if err != nil {
				log.Println(err)
			}
			results <- err
		}(addr)
	}
	// wait for all handlers
	for i := 0; i < cap(results); i++ {
		err := <-results
		if err != nil {
			return err
		}
	}
	close(results)
	return nil
}

func (s *Server) handlerFunc(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
	w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS, POST")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Max-Age", "3600")
	w.Header().Set("Server", USER_AGENT)
	w.Header().Set("X-Powered-By", USER_AGENT)

	if r.Method == "OPTIONS" {
		w.Header().Set("Content-Length", "0")
		return
	}

	if r.Form == nil {
		const maxMemory = 32 << 20 // 32 MB
		r.ParseMultipartForm(maxMemory)
	}

	for _, header := range s.conf.DebugHTTPHeaders {
		if value := r.Header.Get(header); value != "" {
			log.Printf("%s: %s\n", header, value)
		}
	}

	contentType := r.Header.Get("Content-Type")
	if ct := r.FormValue("ct"); ct != "" {
		contentType = ct
	}
	if contentType == "" {
		// Guess request Content-Type based on other parameters
		if r.FormValue("name") != "" {
			contentType = "application/dns-json"
		} else if r.FormValue("dns") != "" {
			contentType = "application/dns-message"
		}
	}
	var responseType string
	for _, responseCandidate := range strings.Split(r.Header.Get("Accept"), ",") {
		responseCandidate = strings.SplitN(responseCandidate, ";", 2)[0]
		if responseCandidate == "application/json" {
			responseType = "application/json"
			break
		} else if responseCandidate == "application/dns-udpwireformat" {
			responseType = "application/dns-message"
			break
		} else if responseCandidate == "application/dns-message" {
			responseType = "application/dns-message"
			break
		}
	}
	if responseType == "" {
		// Guess response Content-Type based on request Content-Type
		if contentType == "application/dns-json" {
			responseType = "application/json"
		} else if contentType == "application/dns-message" {
			responseType = "application/dns-message"
		} else if contentType == "application/dns-udpwireformat" {
			responseType = "application/dns-message"
		}
	}

	var req *DNSRequest
	if contentType == "application/dns-json" {
		req = s.parseRequestGoogle(ctx, w, r)
	} else if contentType == "application/dns-message" {
		req = s.parseRequestIETF(ctx, w, r)
	} else if contentType == "application/dns-udpwireformat" {
		req = s.parseRequestIETF(ctx, w, r)
	} else {
		jsonDNS.FormatError(w, fmt.Sprintf("Invalid argument value: \"ct\" = %q", contentType), 415)
		return
	}
	if req.errcode == 444 {
		return
	}
	if req.errcode != 0 {
		jsonDNS.FormatError(w, req.errtext, req.errcode)
		return
	}

	req = s.patchRootRD(req)

	var err error
	req, err = s.doDNSQuery(ctx, req)
	if err != nil {
		jsonDNS.FormatError(w, fmt.Sprintf("DNS query failure (%s)", err.Error()), 503)
		return
	}

	if responseType == "application/json" {
		s.generateResponseGoogle(ctx, w, r, req)
	} else if responseType == "application/dns-message" {
		s.generateResponseIETF(ctx, w, r, req)
	} else {
		panic("Unknown response Content-Type")
	}
}

func (s *Server) findClientIP(r *http.Request) net.IP {
	XForwardedFor := r.Header.Get("X-Forwarded-For")
	if XForwardedFor != "" {
		for _, addr := range strings.Split(XForwardedFor, ",") {
			addr = strings.TrimSpace(addr)
			ip := net.ParseIP(addr)
			if jsonDNS.IsGlobalIP(ip) {
				return ip
			}
		}
	}
	XRealIP := r.Header.Get("X-Real-IP")
	if XRealIP != "" {
		addr := strings.TrimSpace(XRealIP)
		ip := net.ParseIP(addr)
		if jsonDNS.IsGlobalIP(ip) {
			return ip
		}
	}
	remoteAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	if err != nil {
		return nil
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		return ip
	}
	return nil
}

// Workaround a bug causing Unbound to refuse returning anything about the root
func (s *Server) patchRootRD(req *DNSRequest) *DNSRequest {
	for _, question := range req.request.Question {
		if question.Name == "." {
			req.request.RecursionDesired = true
		}
	}
	return req
}

// Return the position index for the question of qtype from a DNS msg, otherwise return -1
func (s *Server) indexQuestionType(msg *dns.Msg, qtype uint16) int {
	for i, question := range msg.Question {
		if question.Qtype == qtype {
			return i
		}
	}
	return -1
}

func (s *Server) doDNSQuery(ctx context.Context, req *DNSRequest) (resp *DNSRequest, err error) {
	// TODO(m13253): Make ctx work. Waiting for a patch for ExchangeContext from miekg/dns.
	numServers := len(s.conf.Upstream)
	for i := uint(0); i < s.conf.Tries; i++ {
		req.currentUpstream = s.conf.Upstream[rand.Intn(numServers)]

		upstream, t := addressAndType(req.currentUpstream)

		switch t {
		default:
			log.Printf("invalid DNS type %q in upstream %q", t, upstream)
			return nil, &configError{"invalid DNS type"}
		// Use DNS-over-TLS (DoT) if configured to do so
		case "tcp-tls":
			req.response, _, err = s.tcpClientTLS.Exchange(req.request, upstream)
		case "tcp", "udp":
			// Use TCP if always configured to or if the Query type dictates it (AXFR)
			if t == "tcp" || (s.indexQuestionType(req.request, dns.TypeAXFR) > -1) {
				req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
			} else {
				req.response, _, err = s.udpClient.Exchange(req.request, upstream)
				if err == nil && req.response != nil && req.response.Truncated {
					log.Println(err)
					req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
				}

				// Retry with TCP if this was an IXFR request and we only received an SOA
				if (s.indexQuestionType(req.request, dns.TypeIXFR) > -1) &&
					(len(req.response.Answer) == 1) &&
					(req.response.Answer[0].Header().Rrtype == dns.TypeSOA) {
					req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
				}
			}
		}

		if err == nil {
			return req, nil
		}
		log.Printf("DNS error from upstream %s: %s\n", req.currentUpstream, err.Error())
	}
	return req, err
}
<commit_msg>add no_ecs argument in query string support<commit_after>/*
   DNS-over-HTTPS
   Copyright (C) 2017-2018 Star Brilliant <m13253@hotmail.com>

   Permission is hereby granted, free of charge, to any person obtaining a
   copy of this software and associated documentation files (the "Software"),
   to deal in the Software without restriction, including without limitation
   the rights to use, copy, modify, merge, publish, distribute, sublicense,
   and/or sell copies of the Software, and to permit persons to whom the
   Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
   DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gorilla/handlers"
	jsonDNS "github.com/m13253/dns-over-https/json-dns"
	"github.com/miekg/dns"
)

type Server struct {
	conf         *config
	udpClient    *dns.Client
	tcpClient    *dns.Client
	tcpClientTLS *dns.Client
	servemux     *http.ServeMux
}

type DNSRequest struct {
	request         *dns.Msg
	response        *dns.Msg
	transactionID   uint16
	currentUpstream string
	isTailored      bool
	errcode         int
	errtext         string
}

func NewServer(conf *config) (*Server, error) {
	timeout := time.Duration(conf.Timeout) * time.Second
	s := &Server{
		conf: conf,
		udpClient: &dns.Client{
			Net:     "udp",
			UDPSize: dns.DefaultMsgSize,
			Timeout: timeout,
		},
		tcpClient: &dns.Client{
			Net:     "tcp",
			Timeout: timeout,
		},
		tcpClientTLS: &dns.Client{
			Net:     "tcp-tls",
			Timeout: timeout,
		},
		servemux: http.NewServeMux(),
	}
	if conf.LocalAddr != "" {
		udpLocalAddr, err := net.ResolveUDPAddr("udp", conf.LocalAddr)
		if err != nil {
			return nil, err
		}
		tcpLocalAddr, err := net.ResolveTCPAddr("tcp", conf.LocalAddr)
		if err != nil {
			return nil, err
		}
		s.udpClient.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: udpLocalAddr,
		}
		s.tcpClient.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: tcpLocalAddr,
		}
		s.tcpClientTLS.Dialer = &net.Dialer{
			Timeout:   timeout,
			LocalAddr: tcpLocalAddr,
		}
	}
	s.servemux.HandleFunc(conf.Path, s.handlerFunc)
	return s, nil
}

func (s *Server) Start() error {
	servemux := http.Handler(s.servemux)
	if s.conf.Verbose {
		servemux = handlers.CombinedLoggingHandler(os.Stdout, servemux)
	}
	results := make(chan error, len(s.conf.Listen))
	for _, addr := range s.conf.Listen {
		go func(addr string) {
			var err error
			if s.conf.Cert != "" || s.conf.Key != "" {
				err = http.ListenAndServeTLS(addr, s.conf.Cert, s.conf.Key, servemux)
			} else {
				err = http.ListenAndServe(addr, servemux)
			}
			if err != nil {
				log.Println(err)
			}
			results <- err
		}(addr)
	}
	// wait for all handlers
	for i := 0; i < cap(results); i++ {
		err := <-results
		if err != nil {
			return err
		}
	}
	close(results)
	return nil
}

func (s *Server) handlerFunc(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
	w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS, POST")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Max-Age", "3600")
	w.Header().Set("Server", USER_AGENT)
	w.Header().Set("X-Powered-By", USER_AGENT)

	if r.Method == "OPTIONS" {
		w.Header().Set("Content-Length", "0")
		return
	}

	if r.Form == nil {
		const maxMemory = 32 << 20 // 32 MB
		r.ParseMultipartForm(maxMemory)
	}

	for _, header := range s.conf.DebugHTTPHeaders {
		if value := r.Header.Get(header); value != "" {
			log.Printf("%s: %s\n", header, value)
		}
	}

	contentType := r.Header.Get("Content-Type")
	if ct := r.FormValue("ct"); ct != "" {
		contentType = ct
	}
	if contentType == "" {
		// Guess request Content-Type based on other parameters
		if r.FormValue("name") != "" {
			contentType = "application/dns-json"
		} else if r.FormValue("dns") != "" {
			contentType = "application/dns-message"
		}
	}
	var responseType string
	for _, responseCandidate := range strings.Split(r.Header.Get("Accept"), ",") {
		responseCandidate = strings.SplitN(responseCandidate, ";", 2)[0]
		if responseCandidate == "application/json" {
			responseType = "application/json"
			break
		} else if responseCandidate == "application/dns-udpwireformat" {
			responseType = "application/dns-message"
			break
		} else if responseCandidate == "application/dns-message" {
			responseType = "application/dns-message"
			break
		}
	}
	if responseType == "" {
		// Guess response Content-Type based on request Content-Type
		if contentType == "application/dns-json" {
			responseType = "application/json"
		} else if contentType == "application/dns-message" {
			responseType = "application/dns-message"
		} else if contentType == "application/dns-udpwireformat" {
			responseType = "application/dns-message"
		}
	}

	var req *DNSRequest
	if contentType == "application/dns-json" {
		req = s.parseRequestGoogle(ctx, w, r)
	} else if contentType == "application/dns-message" {
		req = s.parseRequestIETF(ctx, w, r)
	} else if contentType == "application/dns-udpwireformat" {
		req = s.parseRequestIETF(ctx, w, r)
	} else {
		jsonDNS.FormatError(w, fmt.Sprintf("Invalid argument value: \"ct\" = %q", contentType), 415)
		return
	}
	if req.errcode == 444 {
		return
	}
	if req.errcode != 0 {
		jsonDNS.FormatError(w, req.errtext, req.errcode)
		return
	}

	req = s.patchRootRD(req)

	var err error
	req, err = s.doDNSQuery(ctx, req)
	if err != nil {
		jsonDNS.FormatError(w, fmt.Sprintf("DNS query failure (%s)", err.Error()), 503)
		return
	}

	if responseType == "application/json" {
		s.generateResponseGoogle(ctx, w, r, req)
	} else if responseType == "application/dns-message" {
		s.generateResponseIETF(ctx, w, r, req)
	} else {
		panic("Unknown response Content-Type")
	}
}

func (s *Server) findClientIP(r *http.Request) net.IP {
	noEcs := r.URL.Query().Get("no_ecs")
	if strings.ToLower(noEcs) == "true" {
		return nil
	}

	XForwardedFor := r.Header.Get("X-Forwarded-For")
	if XForwardedFor != "" {
		for _, addr := range strings.Split(XForwardedFor, ",") {
			addr = strings.TrimSpace(addr)
			ip := net.ParseIP(addr)
			if jsonDNS.IsGlobalIP(ip) {
				return ip
			}
		}
	}
	XRealIP := r.Header.Get("X-Real-IP")
	if XRealIP != "" {
		addr := strings.TrimSpace(XRealIP)
		ip := net.ParseIP(addr)
		if jsonDNS.IsGlobalIP(ip) {
			return ip
		}
	}
	remoteAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	if err != nil {
		return nil
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		return ip
	}
	return nil
}

// Workaround a bug causing Unbound to refuse returning anything about the root
func (s *Server) patchRootRD(req *DNSRequest) *DNSRequest {
	for _, question := range req.request.Question {
		if question.Name == "." {
			req.request.RecursionDesired = true
		}
	}
	return req
}

// Return the position index for the question of qtype from a DNS msg, otherwise return -1
func (s *Server) indexQuestionType(msg *dns.Msg, qtype uint16) int {
	for i, question := range msg.Question {
		if question.Qtype == qtype {
			return i
		}
	}
	return -1
}

func (s *Server) doDNSQuery(ctx context.Context, req *DNSRequest) (resp *DNSRequest, err error) {
	// TODO(m13253): Make ctx work. Waiting for a patch for ExchangeContext from miekg/dns.
	numServers := len(s.conf.Upstream)
	for i := uint(0); i < s.conf.Tries; i++ {
		req.currentUpstream = s.conf.Upstream[rand.Intn(numServers)]

		upstream, t := addressAndType(req.currentUpstream)

		switch t {
		default:
			log.Printf("invalid DNS type %q in upstream %q", t, upstream)
			return nil, &configError{"invalid DNS type"}
		// Use DNS-over-TLS (DoT) if configured to do so
		case "tcp-tls":
			req.response, _, err = s.tcpClientTLS.Exchange(req.request, upstream)
		case "tcp", "udp":
			// Use TCP if always configured to or if the Query type dictates it (AXFR)
			if t == "tcp" || (s.indexQuestionType(req.request, dns.TypeAXFR) > -1) {
				req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
			} else {
				req.response, _, err = s.udpClient.Exchange(req.request, upstream)
				if err == nil && req.response != nil && req.response.Truncated {
					log.Println(err)
					req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
				}

				// Retry with TCP if this was an IXFR request and we only received an SOA
				if (s.indexQuestionType(req.request, dns.TypeIXFR) > -1) &&
					(len(req.response.Answer) == 1) &&
					(req.response.Answer[0].Header().Rrtype == dns.TypeSOA) {
					req.response, _, err = s.tcpClient.Exchange(req.request, upstream)
				}
			}
		}

		if err == nil {
			return req, nil
		}
		log.Printf("DNS error from upstream %s: %s\n", req.currentUpstream, err.Error())
	}
	return req, err
}
<|endoftext|>


<commit_before>package herd

import (
	subproto "github.com/Symantec/Dominator/proto/sub"
)

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	// TODO(rgooch): Implement this.
}
<commit_msg>Incremental work on buildUpdateRequest().<commit_after>package herd

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
)

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	compareDirectories(&subFS.Directory, &requiredFS.Directory, filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(subDirectory, requiredDirectory *filesystem.Directory,
	filter *filter.Filter) {
	// First look for entries that should be deleted.

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"github.com/temoto/robotstxt-go"
	"io/ioutil"
	"log"
	"net/http"
	urlparse "net/url"
	"path"
	"strings"
	"time"
)

func (c *Crawler) startDownloader(quit chan bool) {
	downloader := func(url *urlparse.URL) {
		urlString := url.String()

		if _, err := c.pagestore.IsKnownURL(url); err != nil {
			log.Printf("%s has skipped because an error occurred: %v", urlString, err)
		} else {
			if c.checkRobotsPolicy(url) {
				page, redirectChain, err := c.download(url)
				if err != nil {
					log.Println(err)
				} else {
					if urls, err := c.detectURLs(page); err == nil {
						for _, url := range urls {
							c.wqueue <- url
						}
					}

					c.pagestore.Save(page)
					for _, page := range redirectChain {
						c.pagestore.Save(page)
					}
				}
			} else {
				log.Printf("%s has skipped because denied crawling by robots.txt", url.String())
			}
		}
	}

	urlchan := make(chan *urlparse.URL, 1)
	go func() {
		for {
			url, err := c.cqueue.Pop()
			if err == QueueEmpty {
				time.Sleep(1 * time.Second)
			} else {
				urlchan <- url
			}
		}
	}()

loop:
	for {
		select {
		case <-quit:
			break loop
		case url := <-urlchan:
			downloader(url)
			continue
		}
	}

	quit <- true
	log.Printf("Stopped downloader")
}

func (c *Crawler) download(url *urlparse.URL) (p *Page, redirectChain []*Page, err error) {
	redirectChain = make([]*Page, 0)
	chkredirect := func(req *http.Request, via []*http.Request) error {
		if len(via) > 10 || req.URL.String() == via[len(via)-1].URL.String() {
			return ERR_MANY_REDIRECT
		}
		page := NewPage(via[len(via)-1].URL.String(), 0, "", []byte{}, req.URL.String(), time.Now().UTC())
		redirectChain = append(redirectChain, page)
		return nil
	}
	client := &http.Client{CheckRedirect: chkredirect}

	request := &http.Request{
		Method:     "GET",
		URL:        url,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Body:       nil,
		Host:       url.Host,
	}
	request.Header.Add("User-Agent", USER_AGENT)

	var response *http.Response
	done := make(chan bool, 1)
	go func() {
		response, err = client.Do(request)
		if err != nil {
			log.Println(err)
			err = ERR_DOWNLOAD
			return
		}
		done <- true
	}()

	select {
	case <-time.After(2 * time.Second):
		err = ERR_TIMEOUT
		return
	case <-done:
	}

	body := []byte{}
	if response.StatusCode == http.StatusOK {
		defer response.Body.Close()
		if body, err = ioutil.ReadAll(response.Body); err != nil {
			log.Println(err)
			err = ERR_INTERNAL
			return
		}
	}

	p = NewPage(response.Request.URL.String(), response.StatusCode, response.Header.Get("Content-Type"), body, "", time.Now().UTC())
	return
}

func (c *Crawler) checkRobotsPolicy(url *urlparse.URL) bool {
	robotstxtURL := &urlparse.URL{
		Scheme: url.Scheme,
		User:   url.User,
		Host:   url.Host,
		Path:   "/robots.txt"}

	var robotstxtData *Page
	var err error
	if robotstxtData, err = c.pagestore.Get(robotstxtURL.String()); err != nil {
		return true
	} else if robotstxtData == nil {
		robotstxtData, _, err = c.download(robotstxtURL)
		if err != nil {
			log.Printf("Error occurred during downloading robots.txt: %v", err)
			return true
		} else if robotstxtData == nil {
			return true
		}

		if robotstxtData.URL != robotstxtURL.String() {
			// redirected
			return true
		} else {
			if err = c.pagestore.Save(robotstxtData); err != nil {
				log.Printf("Error occurred during saving robots.txt: %v", err)
				return true
			}
		}
	}

	if robotstxtData.State.LastStatusCode != 200 {
		return true
	}

	robots, err := robotstxt.FromBytes(robotstxtData.Body)
	if err != nil {
		log.Println("Error occurred during parsing robots.tx: %v", err)
		return true
	}

	robotsGroup := robots.FindGroup(CRAWLER_NAME)
	if url.RawQuery == "" {
		return robotsGroup.Test(url.Path)
	} else {
		return robotsGroup.Test(url.Path + "?" + url.RawQuery)
	}
}

func (c *Crawler) detectURLs(p *Page) ([]*urlparse.URL, error) {
	if !strings.HasPrefix(p.ContentType, "text/html") && !strings.HasPrefix(p.ContentType, "application/xhtml+xml") {
		return nil, ERR_NOT_HTML
	}

	doc, err := html.Parse(bytes.NewReader(p.Body))
	if err != nil {
		log.Printf("Failed to parse HTML of %s via %v", p.URL, err)
		return nil, ERR_HTML_PARSE_ERROR
	}

	toAbs := func(base *urlparse.URL, url *urlparse.URL) {
		if url.Scheme == "" {
			url.Scheme = base.Scheme
		}

		if url.Host == "" {
			url.Host = base.Host
		}

		if url.Path == "" {
			url.Path = base.Path
		} else if !strings.HasPrefix(url.Path, "/") {
			url.Path = path.Join(base.Path, url.Path)
		}

		if url.Opaque == "" {
			url.Opaque = "//" + url.Host + url.Path
		}
	}

	base, _ := urlparse.Parse(p.URL)
	URLs := make(map[string]bool)
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			for _, attr := range n.Attr {
				if attr.Key == "href" && !URLs[attr.Val] {
					URLs[attr.Val] = true
					break
				}
			}
		} else if n.Type == html.ElementNode && n.Data == "base" {
			for _, attr := range n.Attr {
				if attr.Key == "href" {
					if _base, err := urlparse.Parse(attr.Val); err == nil {
						toAbs(base, _base)
						base = _base
					}
					break
				}
			}
		}

		for child := n.FirstChild; child != nil; child = child.NextSibling {
			f(child)
		}
	}
	f(doc)

	result := make([]*urlparse.URL, 0, len(URLs))
	for _url := range URLs {
		url, err := urlparse.Parse(_url)
		if err != nil || url.Scheme != "" && url.Scheme != "http" && url.Scheme != "https" {
			continue
		}

		toAbs(base, url)
		if url.Fragment != "" {
			url.Fragment = ""
		}

		result = append(result, url)
	}

	return result, nil
}
<commit_msg>remove extra log output<commit_after>package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	"github.com/temoto/robotstxt-go"
	"io/ioutil"
	"log"
	"net/http"
	urlparse "net/url"
	"path"
	"strings"
	"time"
)

func (c *Crawler) startDownloader(quit chan bool) {
	downloader := func(url *urlparse.URL) {
		urlString := url.String()

		if _, err := c.pagestore.IsKnownURL(url); err != nil {
			log.Printf("%s has skipped because an error occurred: %v", urlString, err)
		} else {
			if c.checkRobotsPolicy(url) {
				page, redirectChain, err := c.download(url)
				if err != nil {
					log.Println(err)
				} else {
					if urls, err := c.detectURLs(page); err == nil {
						for _, url := range urls {
							c.wqueue <- url
						}
					}

					c.pagestore.Save(page)
					for _, page := range redirectChain {
						c.pagestore.Save(page)
					}
				}
			} else {
				log.Printf("%s has skipped because denied crawling by robots.txt", url.String())
			}
		}
	}

	urlchan := make(chan *urlparse.URL, 1)
	go func() {
		for {
			url, err := c.cqueue.Pop()
			if err == QueueEmpty {
				time.Sleep(1 * time.Second)
			} else {
				urlchan <- url
			}
		}
	}()

loop:
	for {
		select {
		case <-quit:
			break loop
		case url := <-urlchan:
			downloader(url)
			continue
		}
	}

	quit <- true
	log.Printf("Stopped downloader")
}

func (c *Crawler) download(url *urlparse.URL) (p *Page, redirectChain []*Page, err error) {
	redirectChain = make([]*Page, 0)
	chkredirect := func(req *http.Request, via []*http.Request) error {
		if len(via) > 10 || req.URL.String() == via[len(via)-1].URL.String() {
			return ERR_MANY_REDIRECT
		}
		page := NewPage(via[len(via)-1].URL.String(), 0, "", []byte{}, req.URL.String(), time.Now().UTC())
		redirectChain = append(redirectChain, page)
		return nil
	}
	client := &http.Client{CheckRedirect: chkredirect}

	request := &http.Request{
		Method:     "GET",
		URL:        url,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Header:     make(http.Header),
		Body:       nil,
		Host:       url.Host,
	}
	request.Header.Add("User-Agent", USER_AGENT)

	var response *http.Response
	done := make(chan bool, 1)
	go func() {
		response, err = client.Do(request)
		if err != nil {
			log.Println(err)
			err = ERR_DOWNLOAD
			return
		}
		done <- true
	}()

	select {
	case <-time.After(2 * time.Second):
		err = ERR_TIMEOUT
		return
	case <-done:
	}

	body := []byte{}
	if response.StatusCode == http.StatusOK {
		defer response.Body.Close()
		if body, err = ioutil.ReadAll(response.Body); err != nil {
			log.Println(err)
			err = ERR_INTERNAL
			return
		}
	}

	p = NewPage(response.Request.URL.String(), response.StatusCode, response.Header.Get("Content-Type"), body, "", time.Now().UTC())
	return
}

func (c *Crawler) checkRobotsPolicy(url *urlparse.URL) bool {
	robotstxtURL := &urlparse.URL{
		Scheme: url.Scheme,
		User:   url.User,
		Host:   url.Host,
		Path:   "/robots.txt"}

	var robotstxtData *Page
	var err error
	if robotstxtData, err = c.pagestore.Get(robotstxtURL.String()); err != nil {
		return true
	} else if robotstxtData == nil {
		robotstxtData, _, err = c.download(robotstxtURL)
		if err != nil {
			log.Printf("Error occurred during downloading robots.txt: %v", err)
			return true
		} else if robotstxtData == nil {
			return true
		}

		if robotstxtData.URL != robotstxtURL.String() {
			// redirected
			return true
		} else {
			if err = c.pagestore.Save(robotstxtData); err != nil {
				log.Printf("Error occurred during saving robots.txt: %v", err)
				return true
			}
		}
	}

	if robotstxtData.State.LastStatusCode != 200 {
		return true
	}

	robots, err := robotstxt.FromBytes(robotstxtData.Body)
	if err != nil {
		log.Println("Error occurred during parsing robots.tx: %v", err)
		return true
	}

	robotsGroup := robots.FindGroup(CRAWLER_NAME)
	if url.RawQuery == "" {
		return robotsGroup.Test(url.Path)
	} else {
		return robotsGroup.Test(url.Path + "?" + url.RawQuery)
	}
}

func (c *Crawler) detectURLs(p *Page) ([]*urlparse.URL, error) {
	if !strings.HasPrefix(p.ContentType, "text/html") && !strings.HasPrefix(p.ContentType, "application/xhtml+xml") {
		return nil, ERR_NOT_HTML
	}

	doc, err := html.Parse(bytes.NewReader(p.Body))
	if err != nil {
		return nil, ERR_HTML_PARSE_ERROR
	}

	toAbs := func(base *urlparse.URL, url *urlparse.URL) {
		if url.Scheme == "" {
			url.Scheme = base.Scheme
		}

		if url.Host == "" {
			url.Host = base.Host
		}

		if url.Path == "" {
			url.Path = base.Path
		} else if !strings.HasPrefix(url.Path, "/") {
			url.Path = path.Join(base.Path, url.Path)
		}

		if url.Opaque == "" {
			url.Opaque = "//" + url.Host + url.Path
		}
	}

	base, _ := urlparse.Parse(p.URL)
	URLs := make(map[string]bool)
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			for _, attr := range n.Attr {
				if attr.Key == "href" && !URLs[attr.Val] {
					URLs[attr.Val] = true
					break
				}
			}
		} else if n.Type == html.ElementNode && n.Data == "base" {
			for _, attr := range n.Attr {
				if attr.Key == "href" {
					if _base, err := urlparse.Parse(attr.Val); err == nil {
						toAbs(base, _base)
						base = _base
					}
					break
				}
			}
		}

		for child := n.FirstChild; child != nil; child = child.NextSibling {
			f(child)
		}
	}
	f(doc)

	result := make([]*urlparse.URL, 0, len(URLs))
	for _url := range URLs {
		url, err := urlparse.Parse(_url)
		if err != nil || url.Scheme != "" && url.Scheme != "http" && url.Scheme != "https" {
			continue
		}

		toAbs(base, url)
		if url.Fragment != "" {
			url.Fragment = ""
		}

		result = append(result, url)
	}

	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	//urls  []string
	//wg sync.WaitGroup
	c = make(chan string, 100) // Allocate a channel.

)

func main() {
	c <- popToCrawlURL()

	for url := range c {
		crawl(url)
		c <- popToCrawlURL()
		time.Sleep(1 * time.Second) // should be a more polite value
	}

}

func crawl(url string) {

	fmt.Println("Trying to crawl: ", url)

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}

	var s = string(body)

	// find links
	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]

		// normalize url
		urlFound, err = normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		insertToCrawlURL(urlFound)
		fmt.Println("Found new url: ", urlFound)

	}
}

func popToCrawlURL() string {
	// Connect to Database
	db, err := sql.Open("mysql", "root:asd@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return url
}

func insertToCrawlURL(url string) {
	// Connect to Database
	db, err := sql.Open("mysql", "root:asd@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url) // Insert tuples (i, i^2)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<commit_msg>added more functions for better readability<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	c = make(chan string, 100) // Allocate a channel.
)

func main() {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// get first url to crawl
	c <- popToCrawlURL(db)

	for url := range c {
		crawl(db, url)

		// get next url to crawl
		c <- popToCrawlURL(db)
		time.Sleep(1 * time.Second) // should be a more polite value
	}

}

func crawl(db *sql.DB, url string) {

	fmt.Println("Trying to crawl: ", url)

	var s, err = getBody(url)
	if err != nil {
		return
	}

	// find links
	urlsFound := findLinks(s)

	for _, urlFound := range urlsFound {
		// normalize url
		urlFound, err := normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// insert into "to_crawl" table of db
		insertToCrawlURL(db, urlFound)
		fmt.Println("Found new url: ", urlFound)
	}
}

func getBody(url string) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	return string(body), nil
}

func findLinks(s string) []string {
	var urlsFound []string

	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]
		urlsFound = append(urlsFound, urlFound)
	}
	return urlsFound
}

func popToCrawlURL(db *sql.DB) string {
	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return url
}

func insertToCrawlURL(db *sql.DB, url string) {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"strconv"
	"time"

	"github.com/btcsuite/btcd/wire"
)

type CrawlError struct {
	errLoc string
	Err    error
}

// Error returns a formatted error about a crawl
func (e *CrawlError) Error() string {
	return "err: " + e.errLoc + ": " + e.Err.Error()
}

// crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
// list of currently active addresses
func crawlTwistee(tw *Twistee) {

	tw.crawlActive = true
	tw.crawlStart = time.Now()

	defer crawlEnd(tw)

	if config.debug {
		log.Printf("status - start crawl: twistee %s status: %v:%v lastcrawl: %s\n",
			net.JoinHostPort(tw.na.IP.String(),
				strconv.Itoa(int(tw.na.Port))),
			tw.status,
			tw.rating,
			time.Since(tw.crawlStart).String())
	}

	// connect to the remote ip and ask them for their addr list
	ras, e := crawlIP(tw)
	if e != nil {
		// update the fact that we have not connected to this twistee
		tw.lastTry = time.Now()
		tw.connectFails++
		tw.statusStr = e.Error()

		// update the status of this failed twistee
		switch tw.status {
		case statusRG:
			if tw.rating += 25; tw.rating > 30 {
				tw.status = statusWG
				tw.statusTime = time.Now()
			}
		case statusCG:
			if tw.rating += 25; tw.rating >= 50 {
				tw.status = statusWG
				tw.statusTime = time.Now()
			}
		case statusWG:
			if tw.rating += 30; tw.rating >= 100 {
				tw.status = statusNG // not able to connect to this twistee so ignore
				tw.statusTime = time.Now()
			}
		}
		// no more to do so return which will shutdown the goroutine & call
		// the deffered cleanup
		if config.verbose {
			log.Printf("debug - failed crawl: twistee %s s:r:f: %v:%v:%v %s\n",
				net.JoinHostPort(tw.na.IP.String(),
					strconv.Itoa(int(tw.na.Port))),
				tw.status,
				tw.rating,
				tw.connectFails,
				tw.statusStr)
		}
		return
	}

	// succesful connection and addresses received so mark status
	if tw.status != statusCG {
		tw.status = statusCG
		tw.statusTime = time.Now()
	}
	tw.rating = 0
	tw.connectFails = 0
	tw.lastConnect = time.Now()
	tw.lastTry = time.Now()
	tw.statusStr = "ok: received remote address list"

	added := 0

	// loop through all the received network addresses and add to thelist if not present
	for _, na := range ras {
		// a new network address so add to the system
		if x := config.seeder.addNa(na); x == true {
			added++
		}
	}

	if config.verbose {
		log.Printf("status - crawl done: twistee: %s newstatus: %v.%v received_addr: %v added_addr: %v CrawlTime: %s\n",
			net.JoinHostPort(tw.na.IP.String(),
				strconv.Itoa(int(tw.na.Port))),
			tw.status,
			tw.rating,
			len(ras),
			added,
			time.Since(tw.crawlStart).String())
	}

	// goroutine ends. deffered cleanup runs
}

// crawlEnd is a deffered func to update theList after a crawl is all done
func crawlEnd(tw *Twistee) {
	tw.crawlActive = false
	// FIXME - scan for long term crawl active twistees. Dial timeout is 10 seconds
	// so should be done in under 5 minutes
}

// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(tw *Twistee) ([]*wire.NetAddress, *CrawlError) {

	ip := tw.na.IP.String()
	port := strconv.Itoa(int(tw.na.Port))
	// get correct formatting for ipv6 addresses
	dialString := net.JoinHostPort(ip, port)

	conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
	if err != nil {
		if config.debug {
			log.Printf("error - Could not connect to %s - %v\n", ip, err)
		}
		return nil, &CrawlError{"", err}
	}

	defer conn.Close()
	if config.verbose {
		log.Printf("%s - Connected to remote address. Last connect was %v ago\n", ip, time.Since(tw.lastConnect).String())
	}

	// set a deadline for all comms to be done by. After this all i/o will error
	conn.SetDeadline(time.Now().Add(time.Second * MAXTO))

	// First command to remote end needs to be a version command
	// last parameter is lastblock
	msgver, err := wire.NewMsgVersionFromConn(conn, NOUNCE, 0)
	if err != nil {
		return nil, &CrawlError{"Create NewMsgVersionFromConn", err}
	}

	err = wire.WriteMessage(conn, msgver, PVER, TWISTNET)
	if err != nil {
		// Log and handle the error
		return nil, &CrawlError{"Write Version Message", err}
	}

	// first message received should be version
	msg, _, err := wire.ReadMessage(conn, PVER, TWISTNET)
	if err != nil {
		// Log and handle the error
		return nil, &CrawlError{"Read message after sending Version", err}
	}

	switch msg := msg.(type) {
	case *wire.MsgVersion:
		// The message is a pointer to a MsgVersion struct.
		if config.debug {
			log.Printf("%s - Remote version: %v\n", ip, msg.ProtocolVersion)
		}
	default:
		return nil, &CrawlError{"Did not receive expected Version message from remote client", err}
	}

	// FIXME - update twistee client version with what they just said

	// send verack command
	msgverack := wire.NewMsgVerAck()

	err = wire.WriteMessage(conn, msgverack, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"writing message VerAck", err}
	}

	// second message received should be verack
	msg, _, err = wire.ReadMessage(conn, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"reading expected Ver Ack from remote client", err}
	}

	switch msg.(type) {
	case *wire.MsgVerAck:
		if config.debug {
			log.Printf("%s - received Version Ack\n", ip)
		}
	default:
		return nil, &CrawlError{"Did not receive expected Ver Ack message from remote client", err}
	}

	// send getaddr command
	msgGetAddr := wire.NewMsgGetAddr()

	err = wire.WriteMessage(conn, msgGetAddr, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"writing Addr message to remote client", err}
	}

	c := 0
	dowhile := true
	for dowhile == true {

		// Using the Bitcoin lib for the Twister Net means it does not understand some
		// of the commands and will error. We can ignore these as we are only
		// interested in the addr message and its content.
		msgaddr, _, _ := wire.ReadMessage(conn, PVER, TWISTNET)
		if msgaddr != nil {
			switch msg := msgaddr.(type) {
			case *wire.MsgAddr:
				// received the addrt message so return the result
				if config.debug {
					log.Printf("%s - received valid addr message\n", ip)
				}
				dowhile = false
				return msg.AddrList, nil
			default:
				if config.debug {
					log.Printf("%s - ignoring message - %v\n", ip, msg.Command())
				}
			}
		}
		// if we get more than 25 messages before the addr we asked for then give up on this client
		if c++; c >= 25 {
			dowhile = false
		}
	}

	// received too many messages before requested Addr
	return nil, &CrawlError{"message loop - did not receive remote addresses in first 25 messages from remote client", err}
}

/*

 */
<commit_msg>Bug fix on nil error<commit_after>package main

import (
	"errors"
	"log"
	"net"
	"strconv"
	"time"

	"github.com/btcsuite/btcd/wire"
)

type CrawlError struct {
	errLoc string
	Err    error
}

// Error returns a formatted error about a crawl
func (e *CrawlError) Error() string {
	return "err: " + e.errLoc + ": " + e.Err.Error()
}

// crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
// list of currently active addresses
func crawlTwistee(tw *Twistee) {

	tw.crawlActive = true
	tw.crawlStart = time.Now()

	defer crawlEnd(tw)

	if config.debug {
		log.Printf("status - start crawl: twistee %s status: %v:%v lastcrawl: %s\n",
			net.JoinHostPort(tw.na.IP.String(),
				strconv.Itoa(int(tw.na.Port))),
			tw.status,
			tw.rating,
			time.Since(tw.crawlStart).String())
	}

	// connect to the remote ip and ask them for their addr list
	ras, e := crawlIP(tw)
	if e != nil {
		// update the fact that we have not connected to this twistee
		tw.lastTry = time.Now()
		tw.connectFails++
		tw.statusStr = e.Error()

		// update the status of this failed twistee
		switch tw.status {
		case statusRG:
			if tw.rating += 25; tw.rating > 30 {
				tw.status = statusWG
				tw.statusTime = time.Now()
			}
		case statusCG:
			if tw.rating += 25; tw.rating >= 50 {
				tw.status = statusWG
				tw.statusTime = time.Now()
			}
		case statusWG:
			if tw.rating += 30; tw.rating >= 100 {
				tw.status = statusNG // not able to connect to this twistee so ignore
				tw.statusTime = time.Now()
			}
		}
		// no more to do so return which will shutdown the goroutine & call
		// the deffered cleanup
		if config.verbose {
			log.Printf("debug - failed crawl: twistee %s s:r:f: %v:%v:%v %s\n",
				net.JoinHostPort(tw.na.IP.String(),
					strconv.Itoa(int(tw.na.Port))),
				tw.status,
				tw.rating,
				tw.connectFails,
				tw.statusStr)
		}
		return
	}

	// succesful connection and addresses received so mark status
	if tw.status != statusCG {
		tw.status = statusCG
		tw.statusTime = time.Now()
	}
	tw.rating = 0
	tw.connectFails = 0
	tw.lastConnect = time.Now()
	tw.lastTry = time.Now()
	tw.statusStr = "ok: received remote address list"

	added := 0

	// loop through all the received network addresses and add to thelist if not present
	for _, na := range ras {
		// a new network address so add to the system
		if x := config.seeder.addNa(na); x == true {
			added++
		}
	}

	if config.verbose {
		log.Printf("status - crawl done: twistee: %s newstatus: %v.%v received_addr: %v added_addr: %v CrawlTime: %s\n",
			net.JoinHostPort(tw.na.IP.String(),
				strconv.Itoa(int(tw.na.Port))),
			tw.status,
			tw.rating,
			len(ras),
			added,
			time.Since(tw.crawlStart).String())
	}

	// goroutine ends. deffered cleanup runs
}

// crawlEnd is a deffered func to update theList after a crawl is all done
func crawlEnd(tw *Twistee) {
	tw.crawlActive = false
	// FIXME - scan for long term crawl active twistees. Dial timeout is 10 seconds
	// so should be done in under 5 minutes
}

// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(tw *Twistee) ([]*wire.NetAddress, *CrawlError) {

	ip := tw.na.IP.String()
	port := strconv.Itoa(int(tw.na.Port))
	// get correct formatting for ipv6 addresses
	dialString := net.JoinHostPort(ip, port)

	conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
	if err != nil {
		if config.debug {
			log.Printf("error - Could not connect to %s - %v\n", ip, err)
		}
		return nil, &CrawlError{"", err}
	}

	defer conn.Close()
	if config.verbose {
		log.Printf("%s - Connected to remote address. Last connect was %v ago\n", ip, time.Since(tw.lastConnect).String())
	}

	// set a deadline for all comms to be done by. After this all i/o will error
	conn.SetDeadline(time.Now().Add(time.Second * MAXTO))

	// First command to remote end needs to be a version command
	// last parameter is lastblock
	msgver, err := wire.NewMsgVersionFromConn(conn, NOUNCE, 0)
	if err != nil {
		return nil, &CrawlError{"Create NewMsgVersionFromConn", err}
	}

	err = wire.WriteMessage(conn, msgver, PVER, TWISTNET)
	if err != nil {
		// Log and handle the error
		return nil, &CrawlError{"Write Version Message", err}
	}

	// first message received should be version
	msg, _, err := wire.ReadMessage(conn, PVER, TWISTNET)
	if err != nil {
		// Log and handle the error
		return nil, &CrawlError{"Read message after sending Version", err}
	}

	switch msg := msg.(type) {
	case *wire.MsgVersion:
		// The message is a pointer to a MsgVersion struct.
		if config.debug {
			log.Printf("%s - Remote version: %v\n", ip, msg.ProtocolVersion)
		}
	default:
		return nil, &CrawlError{"Did not receive expected Version message from remote client", errors.New("")}
	}

	// FIXME - update twistee client version with what they just said

	// send verack command
	msgverack := wire.NewMsgVerAck()

	err = wire.WriteMessage(conn, msgverack, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"writing message VerAck", err}
	}

	// second message received should be verack
	msg, _, err = wire.ReadMessage(conn, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"reading expected Ver Ack from remote client", err}
	}

	switch msg.(type) {
	case *wire.MsgVerAck:
		if config.debug {
			log.Printf("%s - received Version Ack\n", ip)
		}
	default:
		return nil, &CrawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")}
	}

	// send getaddr command
	msgGetAddr := wire.NewMsgGetAddr()

	err = wire.WriteMessage(conn, msgGetAddr, PVER, TWISTNET)
	if err != nil {
		return nil, &CrawlError{"writing Addr message to remote client", err}
	}

	c := 0
	dowhile := true
	for dowhile == true {

		// Using the Bitcoin lib for the Twister Net means it does not understand some
		// of the commands and will error. We can ignore these as we are only
		// interested in the addr message and its content.
		msgaddr, _, _ := wire.ReadMessage(conn, PVER, TWISTNET)
		if msgaddr != nil {
			switch msg := msgaddr.(type) {
			case *wire.MsgAddr:
				// received the addrt message so return the result
				if config.debug {
					log.Printf("%s - received valid addr message\n", ip)
				}
				dowhile = false
				return msg.AddrList, nil
			default:
				if config.debug {
					log.Printf("%s - ignoring message - %v\n", ip, msg.Command())
				}
			}
		}
		// if we get more than 25 messages before the addr we asked for then give up on this client
		if c++; c >= 25 {
			dowhile = false
		}
	}

	// received too many messages before requested Addr
	return nil, &CrawlError{"message loop - did not receive remote addresses in first 25 messages from remote client", errors.New("")}
}

/*

 */
<|endoftext|>


<commit_before>package user

import (
	"appengine"
	"appengine/datastore"
	"bytes"
	"encoding/json"
	"net/http"
	"strings"
	"techtraits.com/klaxon/router"
	"techtraits.com/log"

)

func init() {
	router.Register("/rest/user/", router.GET, nil, nil, getUsers)
	router.Register("/rest/user/{user_id}/", router.GET, nil, nil, getUser)
	router.Register("/rest/user/", router.POST, []string{"application/json"}, nil, postUser)
}

func getUsers(request router.Request) {

	context := appengine.NewContext(request.HttpRequest)
	query := datastore.NewQuery(USER_KEY)

	var users []User
	_, err := query.GetAll(context, &users)

	if err != nil && strings.Contains(err.Error(), "no such entity") {
		log.Error("Error retriving user: %v", err)
		http.Error(request.ResponseWriter, "User not found", http.StatusNotFound)
	} else if err != nil {
		log.Error("Error retriving user: %v", err)
		http.Error(request.ResponseWriter, err.Error(), http.StatusInternalServerError)
	} else {
		//Empty out password has before seding user
		for _, user := range users {
    		go func(userObj User) {
        		userObj.PasswordHash = ""
    		}(user)
		}

		var userBytes, _ = json.Marshal(users)
		var respBuffer bytes.Buffer
		json.Indent(&respBuffer, userBytes, "", "	")
		respBuffer.WriteTo(request.ResponseWriter)
	}

}

func postUser(request router.Request) {
	var user User
	context := appengine.NewContext(request.HttpRequest)
	content := make([]byte, request.HttpRequest.ContentLength)
	request.HttpRequest.Body.Read(content)
	err := json.Unmarshal(content, &user)
	if err != nil {
		log.Info("error: %v", err)
		http.Error(request.ResponseWriter, err.Error(), http.StatusInternalServerError)
	}
	 _, err = datastore.Put(context, datastore.NewKey(context, USER_KEY, user.UserName, 0, nil), &user)
    if err != nil {
		log.Info("error: %v", err)
		http.Error(request.ResponseWriter, err.Error(), http.StatusInternalServerError)
	}

}

func getUser(request router.Request) {

	context := appengine.NewContext(request.HttpRequest)
	var user User
	err := datastore.Get(context, datastore.NewKey(context, USER_KEY, request.PathParams["user_id"], 0, nil), &user)
	if err != nil && strings.Contains(err.Error(), "no such entity") {
		log.Error("Error retriving user: %v", err)
		http.Error(request.ResponseWriter, "User not found", http.StatusNotFound)
	} else if err != nil {
		log.Error("Error retriving user: %v", err)
		http.Error(request.ResponseWriter, err.Error(), http.StatusInternalServerError)
	} else {
		//Empty out password has before seding user
		user.PasswordHash = ""
		var userBytes, _ = json.Marshal(user)
		var respBuffer bytes.Buffer
		json.Indent(&respBuffer, userBytes, "", "	")
		respBuffer.WriteTo(request.ResponseWriter)
	}
}
<commit_msg>Fixing formatting<commit_after>techtraits.com/klaxon/rest/user/user_resource.go
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         string            `json:"date"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}

	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := formatItem(db.Info(k), repo, name)

		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						return formatItem(info, repo, name)
					}
				}
			}
		}
	}
	return ListItem{}
}

func formatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	item := ListItem{
		ID:           info["id"],
		Date:         info["date"],
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}
<commit_msg>Added date for item info<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         string            `json:"upload-date-formated"`
	Timestamp    string            `json:"upload-date-timestamp,omitempty"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}

	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := formatItem(db.Info(k), repo, name)

		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						return formatItem(info, repo, name)
					}
				}
			}
		}
	}
	return ListItem{}
}

func formatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	date, _ := time.Parse(time.RFC3339Nano, info["date"])
	timestamp := strconv.FormatInt(date.Unix(), 10)

	item := ListItem{
		ID:           info["id"],
		Date:         info["date"],
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
		Timestamp:    timestamp,
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}
<|endoftext|>


<commit_before>package irc

import (
	"bufio"
	"io"
)

type Decoder struct {
	scanner *bufio.Scanner
}

func NewDecoder(r io.Reader) *Decoder {
	scanner := bufio.NewScanner(r)
	return &Decoder{scanner}
}

func (d *Decoder) Decode(msg *Msg) (err error) {

	for d.scanner.Scan() {
		msg.Reset()
		msg.Data = d.scanner.Bytes()[:]
		err = msg.PeekCmd()
		if err != nil {
			return err
		}
		break
	}

	if d.scanner.Err() != nil {
		return d.scanner.Err()
	}

	return nil
}
<commit_msg>add decoder lock<commit_after>package irc

import (
	"bufio"
	"io"
	"sync"
)

type Decoder struct {
	scanner *bufio.Scanner
	*sync.Mutex
}

func NewDecoder(r io.Reader) *Decoder {
	scanner := bufio.NewScanner(r)
	return &Decoder{scanner, &sync.Mutex{}}
}

func (d *Decoder) Decode(msg *Msg) (err error) {
	d.Lock()
	defer d.Unlock()

	for d.scanner.Scan() {
		msg.Reset()
		msg.Data = d.scanner.Bytes()[:]
		err = msg.PeekCmd()
		if err != nil {
			return err
		}
		break
	}

	if d.scanner.Err() != nil {
		return d.scanner.Err()
	}

	return nil
}
<|endoftext|>


<commit_before>package future

import (
	"time"
)

// FuelRechargeRate is number of units recharged per second.
const FuelRechargeRate = float32(1.0 / 3.0) // 1 unit every 3 seconds

// Fuel calculates new value after elap delta time interval. 0.0 to 10.0
func Fuel(initial float32, elap time.Duration) float32 {
	fuel := initial + FuelRechargeRate*float32(int64(elap))/float32(time.Second)
	if fuel > 10 {
		fuel = 10
	}
	return fuel
}

// CannonX calculates new value after elap delta time interval. 0.0 to 1.0
func CannonX(initial float32, rate float32, elap time.Duration) (float32, float32) {
	x := initial + rate*float32(int64(elap))/float32(time.Second)
	switch {
	case x < 0:
		x = -x
		rate = -rate
	case x > 1:
		x = 2 - x
		rate = -rate
	}
	return x, rate
}

// MissileY calculates new value after elap delta time interval. 0.0 to 1.0
func MissileY(initial float32, rate float32, elap time.Duration) float32 {
	y := initial + rate*float32(int64(elap))/float32(time.Second)
	if y > 1 {
		y = 1
	}
	return y
}
<commit_msg>Clean-up.<commit_after>package future

import (
	"time"
)

// FuelRechargeRate is number of units recharged per second.
const FuelRechargeRate = float32(1.0 / 3.0) // 1 unit every 3 seconds

// Fuel calculates new value after elap delta time interval. 0.0 to 10.0
func Fuel(initial float32, elap time.Duration) float32 {
	fuel := initial + FuelRechargeRate*float32(elap)/float32(time.Second)
	if fuel > 10 {
		fuel = 10
	}
	return fuel
}

// CannonX calculates new value after elap delta time interval. 0.0 to 1.0
func CannonX(initial float32, rate float32, elap time.Duration) (float32, float32) {
	x := initial + rate*float32(elap)/float32(time.Second)
	switch {
	case x < 0:
		x = -x
		rate = -rate
	case x > 1:
		x = 2 - x
		rate = -rate
	}
	return x, rate
}

// MissileY calculates new value after elap delta time interval. 0.0 to 1.0
func MissileY(initial float32, rate float32, elap time.Duration) float32 {
	y := initial + rate*float32(elap)/float32(time.Second)
	if y > 1 {
		y = 1
	}
	return y
}
<|endoftext|>