<commit_before>package records

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"github.com/mesosphere/mesos-dns/logging"
	"github.com/miekg/dns"
)

// Config holds mesos dns configuration
type Config struct {

	// Mesos master(s): a list of IP:port pairs for one or more Mesos masters
	Masters []string

	// Zookeeper: a single Zk url
	Zk string

	// Refresh frequency: the frequency in seconds of regenerating records (default 60)
	RefreshSeconds int

	// TTL: the TTL value used for SRV and A records (default 60)
	TTL int32

	// Resolver port: port used to listen for slave requests (default 53)
	Port int

	//  Domain: name of the domain used (default "mesos", ie .mesos domain)
	Domain string

	// DNS server: IP address of the DNS server for forwarded accesses
	Resolvers []string

	// Timeout is the default connect/read/write timeout for outbound
	// queries
	Timeout int

	// File is the location of the config.json file
	File string

	// SOA record fields (see http://tools.ietf.org/html/rfc1035#page-18)
	SOAMname   string // primary name server
	SOARname   string // email of admin esponsible
	SOASerial  uint32 // initial version number (incremented on refresh)
	SOARefresh uint32 // refresh interval
	SOARetry   uint32 // retry interval
	SOAExpire  uint32 // expiration time
	SOAMinttl  uint32 // minimum TTL

	// Value of RecursionAvailable for responses in Mesos domain
	RecurseOn bool

	// ListenAddr is the server listener address
	Listener string

	// NOTE(tsenart): HTTPPort, DNSOn and HTTPOn have defined JSON keys for
	// backwards compatibility with external API clients.
	HTTPPort int `json:"HttpPort"`

	// Enable serving DSN and HTTP requests
	DNSOn  bool `json:"DnsOn"`
	HTTPOn bool `json:"HttpOn"`

	// Enable replies for external requests
	ExternalOn bool

	// EnforceRFC952 will enforce an older, more strict set of rules for DNS labels
	EnforceRFC952 bool
}

// SetConfig instantiates a Config struct read in from config.json
func SetConfig(cjson string) Config {
	c, err := readConfig(cjson)
	if err != nil {
		logging.Error.Fatal(err)
	}
	// validate and complete configuration file
	if !c.DNSOn && !c.HTTPOn {
		logging.Error.Fatalf("Either DNS or HTTP server should be on")
	}
	if len(c.Masters) == 0 && c.Zk == "" {
		logging.Error.Fatalf("specify mesos masters or zookeeper in config.json")
	}
	if err = validateMasters(c.Masters); err != nil {
		logging.Error.Fatalf("Masters validation failed: %v", err)
	}

	if c.ExternalOn {
		if len(c.Resolvers) == 0 {
			c.Resolvers = GetLocalDNS()
		}
		if err = validateResolvers(c.Resolvers); err != nil {
			logging.Error.Fatalf("Resovlers validation failed: %v", err)
		}
	}

	c.Domain = strings.ToLower(c.Domain)

	// SOA record fields
	c.SOARname = strings.TrimRight(strings.Replace(c.SOARname, "@", ".", -1), ".") + "."
	c.SOAMname = strings.TrimRight(c.SOAMname, ".") + "."
	c.SOASerial = uint32(time.Now().Unix())

	// print configuration file
	logging.Verbose.Println("Mesos-DNS configuration:")
	logging.Verbose.Println("   - Masters: " + strings.Join(c.Masters, ", "))
	logging.Verbose.Println("   - Zookeeper: ", c.Zk)
	logging.Verbose.Println("   - RefreshSeconds: ", c.RefreshSeconds)
	logging.Verbose.Println("   - Domain: " + c.Domain)
	logging.Verbose.Println("   - Listener: " + c.Listener)
	logging.Verbose.Println("   - Port: ", c.Port)
	logging.Verbose.Println("   - DnsOn: ", c.DNSOn)
	logging.Verbose.Println("   - TTL: ", c.TTL)
	logging.Verbose.Println("   - Timeout: ", c.Timeout)
	logging.Verbose.Println("   - Resolvers: " + strings.Join(c.Resolvers, ", "))
	logging.Verbose.Println("   - ExternalOn: ", c.ExternalOn)
	logging.Verbose.Println("   - SOAMname: " + c.SOAMname)
	logging.Verbose.Println("   - SOARname: " + c.SOARname)
	logging.Verbose.Println("   - SOASerial: ", c.SOASerial)
	logging.Verbose.Println("   - SOARefresh: ", c.SOARefresh)
	logging.Verbose.Println("   - SOARetry: ", c.SOARetry)
	logging.Verbose.Println("   - SOAExpire: ", c.SOAExpire)
	logging.Verbose.Println("   - SOAExpire: ", c.SOAMinttl)
	logging.Verbose.Println("   - RecurseOn: ", c.RecurseOn)
	logging.Verbose.Println("   - HttpPort: ", c.HTTPPort)
	logging.Verbose.Println("   - HttpOn: ", c.HTTPOn)
	logging.Verbose.Println("   - ConfigFile: ", c.File)
	logging.Verbose.Println("   - EnforceRFC952: ", c.EnforceRFC952)

	return *c
}

func readConfig(file string) (*Config, error) {
	c := Config{
		RefreshSeconds: 60,
		TTL:            60,
		Domain:         "mesos",
		Port:           53,
		Timeout:        5,
		SOARname:       "root.ns1.mesos",
		SOAMname:       "ns1.mesos",
		SOARefresh:     60,
		SOARetry:       600,
		SOAExpire:      86400,
		SOAMinttl:      60,
		Resolvers:      []string{"8.8.8.8"},
		Listener:       "0.0.0.0",
		HTTPPort:       8123,
		DNSOn:          true,
		HTTPOn:         true,
		ExternalOn:     true,
		RecurseOn:      true,
	}

	usr, err := user.Current()
	if err != nil {
		return nil, fmt.Errorf("couldn't retrieve current user: %v", err)
	}

	c.File, err = filepath.Abs(strings.Replace(file, "~/", usr.HomeDir+"/", 1))
	if err != nil {
		return nil, fmt.Errorf("cannot find configuration file")
	} else if bs, err := ioutil.ReadFile(c.File); err != nil {
		return nil, fmt.Errorf("missing configuration file: %q", c.File)
	} else if err = json.Unmarshal(bs, &c); err != nil {
		return nil, fmt.Errorf("failed to unmarshal config file: %q", c.File)
	}

	return &c, nil
}

// GetLocalDNS returns the first nameserver in /etc/resolv.conf
// Used for non-Mesos queries.
func GetLocalDNS() []string {
	conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		logging.Error.Fatalf("%v", err)
	}

	return nonLocalAddies(conf.Servers)
}

// Returns non-local nameserver entries
func nonLocalAddies(cservers []string) []string {
	bad := localAddies()

	good := []string{}

	for i := 0; i < len(cservers); i++ {
		local := false
		for x := 0; x < len(bad); x++ {
			if cservers[i] == bad[x] {
				local = true
			}
		}

		if !local {
			good = append(good, cservers[i])
		}
	}

	return good
}

// Returns an array of local ipv4 addresses
func localAddies() []string {
	addies, err := net.InterfaceAddrs()
	if err != nil {
		logging.Error.Println(err)
	}

	bad := []string{}

	for i := 0; i < len(addies); i++ {
		ip, _, err := net.ParseCIDR(addies[i].String())
		if err != nil {
			logging.Error.Println(err)
		}
		t4 := ip.To4()
		if t4 != nil {
			bad = append(bad, t4.String())
		}
	}

	return bad
}
<commit_msg>workaround for when Current returns nil<commit_after>package records

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os/user"
	"path/filepath"
	"strings"
	"time"

	"github.com/mesosphere/mesos-dns/logging"
	"github.com/miekg/dns"
)

// Config holds mesos dns configuration
type Config struct {

	// Mesos master(s): a list of IP:port pairs for one or more Mesos masters
	Masters []string

	// Zookeeper: a single Zk url
	Zk string

	// Refresh frequency: the frequency in seconds of regenerating records (default 60)
	RefreshSeconds int

	// TTL: the TTL value used for SRV and A records (default 60)
	TTL int32

	// Resolver port: port used to listen for slave requests (default 53)
	Port int

	//  Domain: name of the domain used (default "mesos", ie .mesos domain)
	Domain string

	// DNS server: IP address of the DNS server for forwarded accesses
	Resolvers []string

	// Timeout is the default connect/read/write timeout for outbound
	// queries
	Timeout int

	// File is the location of the config.json file
	File string

	// SOA record fields (see http://tools.ietf.org/html/rfc1035#page-18)
	SOAMname   string // primary name server
	SOARname   string // email of admin esponsible
	SOASerial  uint32 // initial version number (incremented on refresh)
	SOARefresh uint32 // refresh interval
	SOARetry   uint32 // retry interval
	SOAExpire  uint32 // expiration time
	SOAMinttl  uint32 // minimum TTL

	// Value of RecursionAvailable for responses in Mesos domain
	RecurseOn bool

	// ListenAddr is the server listener address
	Listener string

	// NOTE(tsenart): HTTPPort, DNSOn and HTTPOn have defined JSON keys for
	// backwards compatibility with external API clients.
	HTTPPort int `json:"HttpPort"`

	// Enable serving DSN and HTTP requests
	DNSOn  bool `json:"DnsOn"`
	HTTPOn bool `json:"HttpOn"`

	// Enable replies for external requests
	ExternalOn bool

	// EnforceRFC952 will enforce an older, more strict set of rules for DNS labels
	EnforceRFC952 bool
}

// SetConfig instantiates a Config struct read in from config.json
func SetConfig(cjson string) Config {
	c, err := readConfig(cjson)
	if err != nil {
		logging.Error.Fatal(err)
	}
	// validate and complete configuration file
	if !c.DNSOn && !c.HTTPOn {
		logging.Error.Fatalf("Either DNS or HTTP server should be on")
	}
	if len(c.Masters) == 0 && c.Zk == "" {
		logging.Error.Fatalf("specify mesos masters or zookeeper in config.json")
	}
	if err = validateMasters(c.Masters); err != nil {
		logging.Error.Fatalf("Masters validation failed: %v", err)
	}

	if c.ExternalOn {
		if len(c.Resolvers) == 0 {
			c.Resolvers = GetLocalDNS()
		}
		if err = validateResolvers(c.Resolvers); err != nil {
			logging.Error.Fatalf("Resovlers validation failed: %v", err)
		}
	}

	c.Domain = strings.ToLower(c.Domain)

	// SOA record fields
	c.SOARname = strings.TrimRight(strings.Replace(c.SOARname, "@", ".", -1), ".") + "."
	c.SOAMname = strings.TrimRight(c.SOAMname, ".") + "."
	c.SOASerial = uint32(time.Now().Unix())

	// print configuration file
	logging.Verbose.Println("Mesos-DNS configuration:")
	logging.Verbose.Println("   - Masters: " + strings.Join(c.Masters, ", "))
	logging.Verbose.Println("   - Zookeeper: ", c.Zk)
	logging.Verbose.Println("   - RefreshSeconds: ", c.RefreshSeconds)
	logging.Verbose.Println("   - Domain: " + c.Domain)
	logging.Verbose.Println("   - Listener: " + c.Listener)
	logging.Verbose.Println("   - Port: ", c.Port)
	logging.Verbose.Println("   - DnsOn: ", c.DNSOn)
	logging.Verbose.Println("   - TTL: ", c.TTL)
	logging.Verbose.Println("   - Timeout: ", c.Timeout)
	logging.Verbose.Println("   - Resolvers: " + strings.Join(c.Resolvers, ", "))
	logging.Verbose.Println("   - ExternalOn: ", c.ExternalOn)
	logging.Verbose.Println("   - SOAMname: " + c.SOAMname)
	logging.Verbose.Println("   - SOARname: " + c.SOARname)
	logging.Verbose.Println("   - SOASerial: ", c.SOASerial)
	logging.Verbose.Println("   - SOARefresh: ", c.SOARefresh)
	logging.Verbose.Println("   - SOARetry: ", c.SOARetry)
	logging.Verbose.Println("   - SOAExpire: ", c.SOAExpire)
	logging.Verbose.Println("   - SOAExpire: ", c.SOAMinttl)
	logging.Verbose.Println("   - RecurseOn: ", c.RecurseOn)
	logging.Verbose.Println("   - HttpPort: ", c.HTTPPort)
	logging.Verbose.Println("   - HttpOn: ", c.HTTPOn)
	logging.Verbose.Println("   - ConfigFile: ", c.File)
	logging.Verbose.Println("   - EnforceRFC952: ", c.EnforceRFC952)

	return *c
}

func readConfig(file string) (*Config, error) {
	c := Config{
		RefreshSeconds: 60,
		TTL:            60,
		Domain:         "mesos",
		Port:           53,
		Timeout:        5,
		SOARname:       "root.ns1.mesos",
		SOAMname:       "ns1.mesos",
		SOARefresh:     60,
		SOARetry:       600,
		SOAExpire:      86400,
		SOAMinttl:      60,
		Resolvers:      []string{"8.8.8.8"},
		Listener:       "0.0.0.0",
		HTTPPort:       8123,
		DNSOn:          true,
		HTTPOn:         true,
		ExternalOn:     true,
		RecurseOn:      true,
	}

	workingDir := "."
	usr, err := user.Current()
	if err != nil {
		logging.Error.Println("Failed to determine current user, translating ~/ to ./, error was", err)
	} else {
		workingDir = usr.HomeDir
	}

	c.File, err = filepath.Abs(strings.Replace(file, "~/", workingDir+"/", 1))
	if err != nil {
		return nil, fmt.Errorf("cannot find configuration file")
	} else if bs, err := ioutil.ReadFile(c.File); err != nil {
		return nil, fmt.Errorf("missing configuration file: %q", c.File)
	} else if err = json.Unmarshal(bs, &c); err != nil {
		return nil, fmt.Errorf("failed to unmarshal config file: %q", c.File)
	}

	return &c, nil
}

// GetLocalDNS returns the first nameserver in /etc/resolv.conf
// Used for non-Mesos queries.
func GetLocalDNS() []string {
	conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		logging.Error.Fatalf("%v", err)
	}

	return nonLocalAddies(conf.Servers)
}

// Returns non-local nameserver entries
func nonLocalAddies(cservers []string) []string {
	bad := localAddies()

	good := []string{}

	for i := 0; i < len(cservers); i++ {
		local := false
		for x := 0; x < len(bad); x++ {
			if cservers[i] == bad[x] {
				local = true
			}
		}

		if !local {
			good = append(good, cservers[i])
		}
	}

	return good
}

// Returns an array of local ipv4 addresses
func localAddies() []string {
	addies, err := net.InterfaceAddrs()
	if err != nil {
		logging.Error.Println(err)
	}

	bad := []string{}

	for i := 0; i < len(addies); i++ {
		ip, _, err := net.ParseCIDR(addies[i].String())
		if err != nil {
			logging.Error.Println(err)
		}
		t4 := ip.To4()
		if t4 != nil {
			bad = append(bad, t4.String())
		}
	}

	return bad
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

const (
	VERSION_MASK = 0xffff0000
	VERSION_1    = 0x80010000
)

type TProtocol interface {
	WriteMessageBegin(name string, typeId TMessageType, seqid int32) error
	WriteMessageEnd() error
	WriteStructBegin(name string) error
	WriteStructEnd() error
	WriteFieldBegin(name string, typeId TType, id int16) error
	WriteFieldEnd() error
	WriteFieldStop() error
	WriteMapBegin(keyType TType, valueType TType, size int) error
	WriteMapEnd() error
	WriteListBegin(elemType TType, size int) error
	WriteListEnd() error
	WriteSetBegin(elemType TType, size int) error
	WriteSetEnd() error
	WriteBool(value bool) error
	WriteByte(value byte) error
	WriteI16(value int16) error
	WriteI32(value int32) error
	WriteI64(value int64) error
	WriteDouble(value float64) error
	WriteString(value string) error
	WriteBinary(value []byte) error

	ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error)
	ReadMessageEnd() error
	ReadStructBegin() (name string, err error)
	ReadStructEnd() error
	ReadFieldBegin() (name string, typeId TType, id int16, err error)
	ReadFieldEnd() error
	ReadMapBegin() (keyType TType, valueType TType, size int, err error)
	ReadMapEnd() error
	ReadListBegin() (elemType TType, size int, err error)
	ReadListEnd() error
	ReadSetBegin() (elemType TType, size int, err error)
	ReadSetEnd() error
	ReadBool() (value bool, err error)
	ReadByte() (value byte, err error)
	ReadI16() (value int16, err error)
	ReadI32() (value int32, err error)
	ReadI64() (value int64, err error)
	ReadDouble() (value float64, err error)
	ReadString() (value string, err error)
	ReadBinary() (value []byte, err error)

	Skip(fieldType TType) (err error)
	Flush() (err error)

	Transport() TTransport
}

// The maximum recursive depth the skip() function will traverse
var MaxSkipDepth = 1<<31 - 1

// Skips over the next data element from the provided input TProtocol object.
func SkipDefaultDepth(prot TProtocol, typeId TType) (err error) {
	return Skip(prot, typeId, MaxSkipDepth)
}

// Skips over the next data element from the provided input TProtocol object.
func Skip(self TProtocol, fieldType TType, maxDepth int) (err error) {
	switch fieldType {
	case STOP:
		return
	case BOOL:
		_, err = self.ReadBool()
		return
	case BYTE:
		_, err = self.ReadByte()
		return
	case I16:
		_, err = self.ReadI16()
		return
	case I32:
		_, err = self.ReadI32()
		return
	case I64:
		_, err = self.ReadI64()
		return
	case DOUBLE:
		_, err = self.ReadDouble()
		return
	case STRING:
		_, err = self.ReadString()
		return
	case STRUCT:
		if _, err = self.ReadStructBegin(); err != nil {
			return err
		}
		for {
			_, typeId, _, _ := self.ReadFieldBegin()
			if typeId == STOP {
				break
			}
			Skip(self, typeId, maxDepth-1)
			self.ReadFieldEnd()
		}
		return self.ReadStructEnd()
	case MAP:
		keyType, valueType, size, err := self.ReadMapBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			Skip(self, keyType, maxDepth-1)
			self.Skip(valueType)
		}
		return self.ReadMapEnd()
	case SET:
		elemType, size, err := self.ReadSetBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			Skip(self, elemType, maxDepth-1)
		}
		return self.ReadSetEnd()
	case LIST:
		elemType, size, err := self.ReadListBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			Skip(self, elemType, maxDepth-1)
		}
		return self.ReadListEnd()
	}
	return nil
}
<commit_msg>THRIFT-3236 MaxSkipDepth never checked Client: Go Patch: Jens Geyer<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package thrift

import (
	"errors"
)

const (
	VERSION_MASK = 0xffff0000
	VERSION_1    = 0x80010000
)

type TProtocol interface {
	WriteMessageBegin(name string, typeId TMessageType, seqid int32) error
	WriteMessageEnd() error
	WriteStructBegin(name string) error
	WriteStructEnd() error
	WriteFieldBegin(name string, typeId TType, id int16) error
	WriteFieldEnd() error
	WriteFieldStop() error
	WriteMapBegin(keyType TType, valueType TType, size int) error
	WriteMapEnd() error
	WriteListBegin(elemType TType, size int) error
	WriteListEnd() error
	WriteSetBegin(elemType TType, size int) error
	WriteSetEnd() error
	WriteBool(value bool) error
	WriteByte(value byte) error
	WriteI16(value int16) error
	WriteI32(value int32) error
	WriteI64(value int64) error
	WriteDouble(value float64) error
	WriteString(value string) error
	WriteBinary(value []byte) error

	ReadMessageBegin() (name string, typeId TMessageType, seqid int32, err error)
	ReadMessageEnd() error
	ReadStructBegin() (name string, err error)
	ReadStructEnd() error
	ReadFieldBegin() (name string, typeId TType, id int16, err error)
	ReadFieldEnd() error
	ReadMapBegin() (keyType TType, valueType TType, size int, err error)
	ReadMapEnd() error
	ReadListBegin() (elemType TType, size int, err error)
	ReadListEnd() error
	ReadSetBegin() (elemType TType, size int, err error)
	ReadSetEnd() error
	ReadBool() (value bool, err error)
	ReadByte() (value byte, err error)
	ReadI16() (value int16, err error)
	ReadI32() (value int32, err error)
	ReadI64() (value int64, err error)
	ReadDouble() (value float64, err error)
	ReadString() (value string, err error)
	ReadBinary() (value []byte, err error)

	Skip(fieldType TType) (err error)
	Flush() (err error)

	Transport() TTransport
}

// The maximum recursive depth the skip() function will traverse
const DEFAULT_RECURSION_DEPTH = 64

// Skips over the next data element from the provided input TProtocol object.
func SkipDefaultDepth(prot TProtocol, typeId TType) (err error) {
	return Skip(prot, typeId, DEFAULT_RECURSION_DEPTH)
}

// Skips over the next data element from the provided input TProtocol object.
func Skip(self TProtocol, fieldType TType, maxDepth int) (err error) {

    if maxDepth <= 0 {
		return NewTProtocolExceptionWithType( DEPTH_LIMIT, errors.New("Depth limit exceeded"))
	}

	switch fieldType {
	case STOP:
		return
	case BOOL:
		_, err = self.ReadBool()
		return
	case BYTE:
		_, err = self.ReadByte()
		return
	case I16:
		_, err = self.ReadI16()
		return
	case I32:
		_, err = self.ReadI32()
		return
	case I64:
		_, err = self.ReadI64()
		return
	case DOUBLE:
		_, err = self.ReadDouble()
		return
	case STRING:
		_, err = self.ReadString()
		return
	case STRUCT:
		if _, err = self.ReadStructBegin(); err != nil {
			return err
		}
		for {
			_, typeId, _, _ := self.ReadFieldBegin()
			if typeId == STOP {
				break
			}
			err := Skip(self, typeId, maxDepth-1)
			if err != nil {
				return err
			}
			self.ReadFieldEnd()
		}
		return self.ReadStructEnd()
	case MAP:
		keyType, valueType, size, err := self.ReadMapBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			err := Skip(self, keyType, maxDepth-1)
			if err != nil {
				return err
			}
			self.Skip(valueType)
		}
		return self.ReadMapEnd()
	case SET:
		elemType, size, err := self.ReadSetBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			err := Skip(self, elemType, maxDepth-1)
			if err != nil {
				return err
			}
		}
		return self.ReadSetEnd()
	case LIST:
		elemType, size, err := self.ReadListBegin()
		if err != nil {
			return err
		}
		for i := 0; i < size; i++ {
			err := Skip(self, elemType, maxDepth-1)
			if err != nil {
				return err
			}
		}
		return self.ReadListEnd()
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
        "path/filepath"
        "os"
        "math"

)

func main() {
        var sizeRange [64]int
        var maxSize int = 0

        if len(os.Args) < 2 {
                fmt.Println("No path specified")
                os.Exit(1)
        }

        _, err := os.Stat(os.Args[1])
        if os.IsNotExist(err) {
                fmt.Printf("%s does not exist\n", os.Args[1])
                os.Exit(1)
        }

        filepath.Walk(
                os.Args[1],
                func(path string, f os.FileInfo, err error) error {
                        if !f.IsDir() {
                                tmp := int(findSize(f.Size(), 0))
                                if maxSize < tmp { maxSize = tmp }
                                sizeRange[tmp]++
                        }
                        return nil
                },
        )

        formatTable(sizeRange, maxSize)
}

func findSize(number int64, exp float64) float64 {
        if number < (1024*int64(math.Pow(2, exp))) || number == 64 {
                return exp
        }

        return findSize(number, exp+1)
}

func formatTable(sizeRange [64]int, maxSize int) {
        for i := 0; i <= maxSize; i++ {
                fmt.Printf("# <%6s: %d\n", formatNum(math.Pow(2,float64(i)), 0), sizeRange[i])
        }
}

func formatNum(size float64, unit int) string {
        units := []string{"KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
        if size >= 1024 {
                return formatNum(size / 1024, unit+1)
        }

        return fmt.Sprintf("%d%s", int(size), units[unit])

}<commit_msg>Switch calculation of unit index<commit_after>package main

import (
  "fmt"
  "path/filepath"
  "os"
  "math"

)

func main() {
  var sizeRange [64]int
  var maxSize int = 0

  if len(os.Args) < 2 {
    fmt.Println("No path specified")
    os.Exit(1)
  }

  _, err := os.Stat(os.Args[1])
  if os.IsNotExist(err) {
    fmt.Printf("%s does not exist\n", os.Args[1])
    os.Exit(1)
  }

  filepath.Walk(
    os.Args[1],
    func(path string, f os.FileInfo, err error) error {
      if !f.IsDir() {
        tmp := int(findSize(f.Size(), 0))
        if maxSize < tmp { maxSize = tmp }
        sizeRange[tmp]++
      }
      return nil
    },
  )

  formatTable(sizeRange, maxSize)
}

func findSize(number int64, exp float64) float64 {
  if number < (1024*int64(math.Pow(2, exp))) || number == 64 {
    return exp
  }

  return findSize(number, exp+1)
}

func formatTable(sizeRange [64]int, maxSize int) {
  units := []string{"KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}

  for i := 0; i <= maxSize; i++ {
    size := math.Pow(2,float64(i))
    index := math.Floor(math.Log(size) / math.Log(1024))

    fmt.Printf(
      "# <%4d%s: %d\n",
      int(size/math.Pow(1024,index)),
      units[int(index)],
      sizeRange[i],
    )
  }
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"fmt"
	"github.com/HouzuoGuo/laitos/daemon/common"
	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/misc"
	"net/http"
	"path"
	"sort"
	"strings"
)

const HandleGitlabPage = `<!doctype html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>Gitlab browser</title>

    <form action="#" method="get">

            Shortcut name: <input type="password" name="shortcut" value="%s" />

            Path: <input type="text" name="path" value="%s" />
            <input type="submit" name="submit" value="Go"/>

            Download file from current path: <input type="text" name="file" value="%s" />
            <input type="submit" name="submit" value="Download"/>

%s

    </form>

` // Gitlab browser content

const (
	GitlabAPITimeoutSec = 110  // Timeout for outgoing API calls made to gitlab
	GitlabMaxObjects    = 4000 // GitlabMaxObjects is the maximum number of objects to list when browsing a git repository.
)

// Browse gitlab repositories and download repository files.
type HandleGitlabBrowser struct {
	PrivateToken string            `json:"PrivateToken"` // Gitlab user private token
	Projects     map[string]string `json:"Projects"`     // Project shortcut name VS "gitlab project ID"
	Recipients   []string          `json:"Recipients"`   // Recipients of notification emails
	MailClient   inet.MailClient   `json:"-"`            // MTA that delivers file download notification email

	logger misc.Logger
}

func (lab *HandleGitlabBrowser) Initialise(logger misc.Logger, _ *common.CommandProcessor) error {
	lab.logger = logger
	return nil
}

// An element of gitlab API "/repository/tree" response array.
type GitlabTreeObject struct {
	ID       string `json:"id"`
	Name     string `json:"name"`
	Type     string `json:"type"`
	FullPath string `json:"path"`
}

/*
Call gitlab API to find out what directories and files are located under that path.
Directory names come with suffix forward-slash.
*/
func (lab *HandleGitlabBrowser) ListGitObjects(projectID string, paths string, maxEntries int) (dirs []string, fileNameID map[string]string, err error) {
	/*
		If there is a leading slash when browsing a directory, the result will definitely be empty. The user most likely
		wants to browse the directory's file list, therefore get rid of the leading slash.
		If present, a trailing slash does not matter.
	*/
	if len(paths) > 1 && paths[0] == '/' {
		paths = paths[1:]
	}
	dirs = make([]string, 0, 8)
	fileNameID = make(map[string]string)
	resp, err := inet.DoHTTP(inet.HTTPRequest{
		Header:     map[string][]string{"PRIVATE-TOKEN": {lab.PrivateToken}},
		TimeoutSec: GitlabAPITimeoutSec,
	}, "https://gitlab.com/handler/v4/projects/%s/repository/tree?ref=master&recursive=false&per_page=%s&path=%s", projectID, maxEntries, paths)
	if err != nil {
		return
	} else if err = resp.Non2xxToError(); err != nil {
		return
	}
	var objects []GitlabTreeObject
	if err = json.Unmarshal(resp.Body, &objects); err != nil {
		return
	}
	for _, obj := range objects {
		if obj.Type == "blob" {
			fileNameID[obj.Name] = obj.ID
		} else {
			dirs = append(dirs, obj.Name+"/")
		}
	}
	sort.Strings(dirs)
	return
}

// Call gitlab API to download a file form git project.
func (lab *HandleGitlabBrowser) DownloadGitBlob(clientIP, projectID string, paths string, fileName string) (content []byte, err error) {
	resp, err := inet.DoHTTP(inet.HTTPRequest{
		Header:     map[string][]string{"PRIVATE-TOKEN": {lab.PrivateToken}},
		TimeoutSec: GitlabAPITimeoutSec,
	}, "https://gitlab.com/handler/v4/projects/%s/repository/files/%s/raw?ref=master", projectID, path.Join(paths, fileName))
	if err != nil {
		return
	} else if err = resp.Non2xxToError(); err != nil {
		return
	}
	content = resp.Body
	if lab.Recipients != nil && len(lab.Recipients) > 0 && lab.MailClient.IsConfigured() {
		go func() {
			subject := inet.OutgoingMailSubjectKeyword + "-gitlab-download-" + fileName
			if err := lab.MailClient.Send(subject, fmt.Sprintf("File \"%s\" has been downloaded by %s", paths+fileName, clientIP), lab.Recipients...); err != nil {
				lab.logger.Warningf("DownloadGitBlob", "", err, "failed to send notification for file \"%s\"", fileName)
			}
		}()
	}
	return
}

func (lab *HandleGitlabBrowser) Handle(w http.ResponseWriter, r *http.Request) {
	shortcutName := strings.TrimSpace(r.FormValue("shortcut"))
	browsePath := r.FormValue("path")
	fileName := strings.TrimSpace(r.FormValue("file"))
	submitAction := r.FormValue("submit")

	NoCache(w)
	if !WarnIfNoHTTPS(r, w) {
		return
	}
	switch submitAction {
	case "Go":
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		projectID, found := lab.Projects[shortcutName]
		if !found {
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "(cannot find shortcut name)")))
			return
		}
		dirs, fileNames, err := lab.ListGitObjects(projectID, browsePath, GitlabMaxObjects)
		if err != nil {
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Error: "+err.Error())))
			return
		}
		// Directory names are already sorted
		contentList := strings.Join(dirs, "\n")
		// Sort file names
		sortedFiles := make([]string, 0, len(fileNames))
		for fileName := range fileNames {
			sortedFiles = append(sortedFiles, fileName)
		}
		sort.Strings(sortedFiles)
		contentList += "\n\n"
		for _, fileName := range sortedFiles {
			contentList += fmt.Sprintf("%s\n", fileName)
		}
		w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, contentList)))
	case "Download":
		projectID, found := lab.Projects[shortcutName]
		if !found {
			w.Header().Set("Content-Type", "text/html; charset=utf-8")
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "(cannot find shortcut name)")))
			return
		}
		content, err := lab.DownloadGitBlob(GetRealClientIP(r), projectID, browsePath, fileName)
		if err != nil {
			w.Header().Set("Content-Type", "text/html; charset=utf-8")
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Error: "+err.Error())))
			return
		}
		w.Header().Set("Content-Type", "application/octet-stream")
		w.Header().Set("Content-Disposition", "attachment; filename="+fileName)
		w.Write(content)
	default:
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Enter path to browse or blob ID to download")))
	}
}

func (_ *HandleGitlabBrowser) GetRateLimitFactor() int {
	return 5
}

func (lab *HandleGitlabBrowser) SelfTest() error {
	errs := make([]error, 0, 0)
	for shortcut, projectID := range lab.Projects {
		if _, _, err := lab.ListGitObjects(projectID, "/", 3); err != nil {
			errs = append(errs, fmt.Errorf("project %s(%s) - %v", shortcut, projectID, err))
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("HandleGitlabBrowser encountered errors: %+v", errs)
}
<commit_msg>revert accidentally renamed api endpoint for gitlab<commit_after>package handler

import (
	"encoding/json"
	"fmt"
	"github.com/HouzuoGuo/laitos/daemon/common"
	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/misc"
	"net/http"
	"path"
	"sort"
	"strings"
)

const HandleGitlabPage = `<!doctype html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <title>Gitlab browser</title>

    <form action="#" method="get">

            Shortcut name: <input type="password" name="shortcut" value="%s" />

            Path: <input type="text" name="path" value="%s" />
            <input type="submit" name="submit" value="Go"/>

            Download file from current path: <input type="text" name="file" value="%s" />
            <input type="submit" name="submit" value="Download"/>

%s

    </form>

` // Gitlab browser content

const (
	GitlabAPITimeoutSec = 110  // Timeout for outgoing API calls made to gitlab
	GitlabMaxObjects    = 4000 // GitlabMaxObjects is the maximum number of objects to list when browsing a git repository.
)

// Browse gitlab repositories and download repository files.
type HandleGitlabBrowser struct {
	PrivateToken string            `json:"PrivateToken"` // Gitlab user private token
	Projects     map[string]string `json:"Projects"`     // Project shortcut name VS "gitlab project ID"
	Recipients   []string          `json:"Recipients"`   // Recipients of notification emails
	MailClient   inet.MailClient   `json:"-"`            // MTA that delivers file download notification email

	logger misc.Logger
}

func (lab *HandleGitlabBrowser) Initialise(logger misc.Logger, _ *common.CommandProcessor) error {
	lab.logger = logger
	return nil
}

// An element of gitlab API "/repository/tree" response array.
type GitlabTreeObject struct {
	ID       string `json:"id"`
	Name     string `json:"name"`
	Type     string `json:"type"`
	FullPath string `json:"path"`
}

/*
Call gitlab API to find out what directories and files are located under that path.
Directory names come with suffix forward-slash.
*/
func (lab *HandleGitlabBrowser) ListGitObjects(projectID string, paths string, maxEntries int) (dirs []string, fileNameID map[string]string, err error) {
	/*
		If there is a leading slash when browsing a directory, the result will definitely be empty. The user most likely
		wants to browse the directory's file list, therefore get rid of the leading slash.
		If present, a trailing slash does not matter.
	*/
	if len(paths) > 1 && paths[0] == '/' {
		paths = paths[1:]
	}
	dirs = make([]string, 0, 8)
	fileNameID = make(map[string]string)
	resp, err := inet.DoHTTP(inet.HTTPRequest{
		Header:     map[string][]string{"PRIVATE-TOKEN": {lab.PrivateToken}},
		TimeoutSec: GitlabAPITimeoutSec,
	}, "https://gitlab.com/api/v4/projects/%s/repository/tree?ref=master&recursive=false&per_page=%s&path=%s", projectID, maxEntries, paths)
	if err != nil {
		return
	} else if err = resp.Non2xxToError(); err != nil {
		return
	}
	var objects []GitlabTreeObject
	if err = json.Unmarshal(resp.Body, &objects); err != nil {
		return
	}
	for _, obj := range objects {
		if obj.Type == "blob" {
			fileNameID[obj.Name] = obj.ID
		} else {
			dirs = append(dirs, obj.Name+"/")
		}
	}
	sort.Strings(dirs)
	return
}

// Call gitlab API to download a file form git project.
func (lab *HandleGitlabBrowser) DownloadGitBlob(clientIP, projectID string, paths string, fileName string) (content []byte, err error) {
	resp, err := inet.DoHTTP(inet.HTTPRequest{
		Header:     map[string][]string{"PRIVATE-TOKEN": {lab.PrivateToken}},
		TimeoutSec: GitlabAPITimeoutSec,
	}, "https://gitlab.com/api/v4/projects/%s/repository/files/%s/raw?ref=master", projectID, path.Join(paths, fileName))
	if err != nil {
		return
	} else if err = resp.Non2xxToError(); err != nil {
		return
	}
	content = resp.Body
	if lab.Recipients != nil && len(lab.Recipients) > 0 && lab.MailClient.IsConfigured() {
		go func() {
			subject := inet.OutgoingMailSubjectKeyword + "-gitlab-download-" + fileName
			if err := lab.MailClient.Send(subject, fmt.Sprintf("File \"%s\" has been downloaded by %s", paths+fileName, clientIP), lab.Recipients...); err != nil {
				lab.logger.Warningf("DownloadGitBlob", "", err, "failed to send notification for file \"%s\"", fileName)
			}
		}()
	}
	return
}

func (lab *HandleGitlabBrowser) Handle(w http.ResponseWriter, r *http.Request) {
	shortcutName := strings.TrimSpace(r.FormValue("shortcut"))
	browsePath := r.FormValue("path")
	fileName := strings.TrimSpace(r.FormValue("file"))
	submitAction := r.FormValue("submit")

	NoCache(w)
	if !WarnIfNoHTTPS(r, w) {
		return
	}
	switch submitAction {
	case "Go":
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		projectID, found := lab.Projects[shortcutName]
		if !found {
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "(cannot find shortcut name)")))
			return
		}
		dirs, fileNames, err := lab.ListGitObjects(projectID, browsePath, GitlabMaxObjects)
		if err != nil {
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Error: "+err.Error())))
			return
		}
		// Directory names are already sorted
		contentList := strings.Join(dirs, "\n")
		// Sort file names
		sortedFiles := make([]string, 0, len(fileNames))
		for fileName := range fileNames {
			sortedFiles = append(sortedFiles, fileName)
		}
		sort.Strings(sortedFiles)
		contentList += "\n\n"
		for _, fileName := range sortedFiles {
			contentList += fmt.Sprintf("%s\n", fileName)
		}
		w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, contentList)))
	case "Download":
		projectID, found := lab.Projects[shortcutName]
		if !found {
			w.Header().Set("Content-Type", "text/html; charset=utf-8")
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "(cannot find shortcut name)")))
			return
		}
		content, err := lab.DownloadGitBlob(GetRealClientIP(r), projectID, browsePath, fileName)
		if err != nil {
			w.Header().Set("Content-Type", "text/html; charset=utf-8")
			w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Error: "+err.Error())))
			return
		}
		w.Header().Set("Content-Type", "application/octet-stream")
		w.Header().Set("Content-Disposition", "attachment; filename="+fileName)
		w.Write(content)
	default:
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write([]byte(fmt.Sprintf(HandleGitlabPage, shortcutName, browsePath, fileName, "Enter path to browse or blob ID to download")))
	}
}

func (_ *HandleGitlabBrowser) GetRateLimitFactor() int {
	return 5
}

func (lab *HandleGitlabBrowser) SelfTest() error {
	errs := make([]error, 0, 0)
	for shortcut, projectID := range lab.Projects {
		if _, _, err := lab.ListGitObjects(projectID, "/", 3); err != nil {
			errs = append(errs, fmt.Errorf("project %s(%s) - %v", shortcut, projectID, err))
		}
	}
	if len(errs) == 0 {
		return nil
	}
	return fmt.Errorf("HandleGitlabBrowser encountered errors: %+v", errs)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package zipfs file provides an implementation of the FileSystem
// interface based on the contents of a .zip file.
//
// Assumptions:
//
// - The file paths stored in the zip file must use a slash ('/') as path
//   separator; and they must be relative (i.e., they must not start with
//   a '/' - this is usually the case if the file was created w/o special
//   options).
// - The zip file system treats the file paths found in the zip internally
//   like absolute paths w/o a leading '/'; i.e., the paths are considered
//   relative to the root of the file system.
// - All path arguments to file system methods must be absolute paths.
package zipfs // import "golang.org/x/tools/godoc/vfs/zipfs"

import (
	"archive/zip"
	"fmt"
	"go/build"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"golang.org/x/tools/godoc/vfs"
)

// zipFI is the zip-file based implementation of FileInfo
type zipFI struct {
	name string    // directory-local name
	file *zip.File // nil for a directory
}

func (fi zipFI) Name() string {
	return fi.name
}

func (fi zipFI) Size() int64 {
	if f := fi.file; f != nil {
		return int64(f.UncompressedSize)
	}
	return 0 // directory
}

func (fi zipFI) ModTime() time.Time {
	if f := fi.file; f != nil {
		return f.ModTime()
	}
	return time.Time{} // directory has no modified time entry
}

func (fi zipFI) Mode() os.FileMode {
	if fi.file == nil {
		// Unix directories typically are executable, hence 555.
		return os.ModeDir | 0555
	}
	return 0444
}

func (fi zipFI) IsDir() bool {
	return fi.file == nil
}

func (fi zipFI) Sys() interface{} {
	return nil
}

// zipFS is the zip-file based implementation of FileSystem
type zipFS struct {
	*zip.ReadCloser
	list zipList
	name string
}

func (fs *zipFS) String() string {
	return "zip(" + fs.name + ")"
}

func (fs *zipFS) RootType(abspath string) vfs.RootType {
	var t vfs.RootType
	switch {
	case exists(path.Join(vfs.GOROOT, abspath)):
		t = vfs.RootTypeGoRoot
	case isGoPath(abspath):
		t = vfs.RootTypeGoPath
	}
	return t
}

func isGoPath(abspath string) bool {
	for _, p := range filepath.SplitList(build.Default.GOPATH) {
		if exists(path.Join(p, abspath)) {
			return true
		}
	}
	return false
}

func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func (fs *zipFS) Close() error {
	fs.list = nil
	return fs.ReadCloser.Close()
}

func zipPath(name string) (string, error) {
	name = path.Clean(name)
	if !path.IsAbs(name) {
		return "", fmt.Errorf("stat: not an absolute path: %s", name)
	}
	return name[1:], nil // strip leading '/'
}

func isRoot(abspath string) bool {
	return path.Clean(abspath) == "/"
}

func (fs *zipFS) stat(abspath string) (int, zipFI, error) {
	if isRoot(abspath) {
		return 0, zipFI{
			name: "",
			file: nil,
		}, nil
	}
	zippath, err := zipPath(abspath)
	if err != nil {
		return 0, zipFI{}, err
	}
	i, exact := fs.list.lookup(zippath)
	if i < 0 {
		// zippath has leading '/' stripped - print it explicitly
		return -1, zipFI{}, &os.PathError{Path: "/" + zippath, Err: os.ErrNotExist}
	}
	_, name := path.Split(zippath)
	var file *zip.File
	if exact {
		file = fs.list[i] // exact match found - must be a file
	}
	return i, zipFI{name, file}, nil
}

func (fs *zipFS) Open(abspath string) (vfs.ReadSeekCloser, error) {
	_, fi, err := fs.stat(abspath)
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", abspath)
	}
	r, err := fi.file.Open()
	if err != nil {
		return nil, err
	}
	return &zipSeek{fi.file, r}, nil
}

type zipSeek struct {
	file *zip.File
	io.ReadCloser
}

func (f *zipSeek) Seek(offset int64, whence int) (int64, error) {
	if whence == 0 && offset == 0 {
		r, err := f.file.Open()
		if err != nil {
			return 0, err
		}
		f.Close()
		f.ReadCloser = r
		return 0, nil
	}
	return 0, fmt.Errorf("unsupported Seek in %s", f.file.Name)
}

func (fs *zipFS) Lstat(abspath string) (os.FileInfo, error) {
	_, fi, err := fs.stat(abspath)
	return fi, err
}

func (fs *zipFS) Stat(abspath string) (os.FileInfo, error) {
	_, fi, err := fs.stat(abspath)
	return fi, err
}

func (fs *zipFS) ReadDir(abspath string) ([]os.FileInfo, error) {
	i, fi, err := fs.stat(abspath)
	if err != nil {
		return nil, err
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("ReadDir: %s is not a directory", abspath)
	}

	var list []os.FileInfo

	// make dirname the prefix that file names must start with to be considered
	// in this directory. we must special case the root directory because, per
	// the spec of this package, zip file entries MUST NOT start with /, so we
	// should not append /, as we would in every other case.
	var dirname string
	if isRoot(abspath) {
		dirname = ""
	} else {
		zippath, err := zipPath(abspath)
		if err != nil {
			return nil, err
		}
		dirname = zippath + "/"
	}
	prevname := ""
	for _, e := range fs.list[i:] {
		if !strings.HasPrefix(e.Name, dirname) {
			break // not in the same directory anymore
		}
		name := e.Name[len(dirname):] // local name
		file := e
		if i := strings.IndexRune(name, '/'); i >= 0 {
			// We infer directories from files in subdirectories.
			// If we have x/y, return a directory entry for x.
			name = name[0:i] // keep local directory name only
			file = nil
		}
		// If we have x/y and x/z, don't return two directory entries for x.
		// TODO(gri): It should be possible to do this more efficiently
		// by determining the (fs.list) range of local directory entries
		// (via two binary searches).
		if name != prevname {
			list = append(list, zipFI{name, file})
			prevname = name
		}
	}

	return list, nil
}

func New(rc *zip.ReadCloser, name string) vfs.FileSystem {
	list := make(zipList, len(rc.File))
	copy(list, rc.File) // sort a copy of rc.File
	sort.Sort(list)
	return &zipFS{rc, list, name}
}

type zipList []*zip.File

// zipList implements sort.Interface
func (z zipList) Len() int           { return len(z) }
func (z zipList) Less(i, j int) bool { return z[i].Name < z[j].Name }
func (z zipList) Swap(i, j int)      { z[i], z[j] = z[j], z[i] }

// lookup returns the smallest index of an entry with an exact match
// for name, or an inexact match starting with name/. If there is no
// such entry, the result is -1, false.
func (z zipList) lookup(name string) (index int, exact bool) {
	// look for exact match first (name comes before name/ in z)
	i := sort.Search(len(z), func(i int) bool {
		return name <= z[i].Name
	})
	if i >= len(z) {
		return -1, false
	}
	// 0 <= i < len(z)
	if z[i].Name == name {
		return i, true
	}

	// look for inexact match (must be in z[i:], if present)
	z = z[i:]
	name += "/"
	j := sort.Search(len(z), func(i int) bool {
		return name <= z[i].Name
	})
	if j >= len(z) {
		return -1, false
	}
	// 0 <= j < len(z)
	if strings.HasPrefix(z[j].Name, name) {
		return i + j, false
	}

	return -1, false
}
<commit_msg>godoc/vfs/zipfs: fix godoc formatting<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package zipfs file provides an implementation of the FileSystem
// interface based on the contents of a .zip file.
//
// Assumptions:
//
//  - The file paths stored in the zip file must use a slash ('/') as path
//    separator; and they must be relative (i.e., they must not start with
//    a '/' - this is usually the case if the file was created w/o special
//    options).
//  - The zip file system treats the file paths found in the zip internally
//    like absolute paths w/o a leading '/'; i.e., the paths are considered
//    relative to the root of the file system.
//  - All path arguments to file system methods must be absolute paths.
package zipfs // import "golang.org/x/tools/godoc/vfs/zipfs"

import (
	"archive/zip"
	"fmt"
	"go/build"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"golang.org/x/tools/godoc/vfs"
)

// zipFI is the zip-file based implementation of FileInfo
type zipFI struct {
	name string    // directory-local name
	file *zip.File // nil for a directory
}

func (fi zipFI) Name() string {
	return fi.name
}

func (fi zipFI) Size() int64 {
	if f := fi.file; f != nil {
		return int64(f.UncompressedSize)
	}
	return 0 // directory
}

func (fi zipFI) ModTime() time.Time {
	if f := fi.file; f != nil {
		return f.ModTime()
	}
	return time.Time{} // directory has no modified time entry
}

func (fi zipFI) Mode() os.FileMode {
	if fi.file == nil {
		// Unix directories typically are executable, hence 555.
		return os.ModeDir | 0555
	}
	return 0444
}

func (fi zipFI) IsDir() bool {
	return fi.file == nil
}

func (fi zipFI) Sys() interface{} {
	return nil
}

// zipFS is the zip-file based implementation of FileSystem
type zipFS struct {
	*zip.ReadCloser
	list zipList
	name string
}

func (fs *zipFS) String() string {
	return "zip(" + fs.name + ")"
}

func (fs *zipFS) RootType(abspath string) vfs.RootType {
	var t vfs.RootType
	switch {
	case exists(path.Join(vfs.GOROOT, abspath)):
		t = vfs.RootTypeGoRoot
	case isGoPath(abspath):
		t = vfs.RootTypeGoPath
	}
	return t
}

func isGoPath(abspath string) bool {
	for _, p := range filepath.SplitList(build.Default.GOPATH) {
		if exists(path.Join(p, abspath)) {
			return true
		}
	}
	return false
}

func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func (fs *zipFS) Close() error {
	fs.list = nil
	return fs.ReadCloser.Close()
}

func zipPath(name string) (string, error) {
	name = path.Clean(name)
	if !path.IsAbs(name) {
		return "", fmt.Errorf("stat: not an absolute path: %s", name)
	}
	return name[1:], nil // strip leading '/'
}

func isRoot(abspath string) bool {
	return path.Clean(abspath) == "/"
}

func (fs *zipFS) stat(abspath string) (int, zipFI, error) {
	if isRoot(abspath) {
		return 0, zipFI{
			name: "",
			file: nil,
		}, nil
	}
	zippath, err := zipPath(abspath)
	if err != nil {
		return 0, zipFI{}, err
	}
	i, exact := fs.list.lookup(zippath)
	if i < 0 {
		// zippath has leading '/' stripped - print it explicitly
		return -1, zipFI{}, &os.PathError{Path: "/" + zippath, Err: os.ErrNotExist}
	}
	_, name := path.Split(zippath)
	var file *zip.File
	if exact {
		file = fs.list[i] // exact match found - must be a file
	}
	return i, zipFI{name, file}, nil
}

func (fs *zipFS) Open(abspath string) (vfs.ReadSeekCloser, error) {
	_, fi, err := fs.stat(abspath)
	if err != nil {
		return nil, err
	}
	if fi.IsDir() {
		return nil, fmt.Errorf("Open: %s is a directory", abspath)
	}
	r, err := fi.file.Open()
	if err != nil {
		return nil, err
	}
	return &zipSeek{fi.file, r}, nil
}

type zipSeek struct {
	file *zip.File
	io.ReadCloser
}

func (f *zipSeek) Seek(offset int64, whence int) (int64, error) {
	if whence == 0 && offset == 0 {
		r, err := f.file.Open()
		if err != nil {
			return 0, err
		}
		f.Close()
		f.ReadCloser = r
		return 0, nil
	}
	return 0, fmt.Errorf("unsupported Seek in %s", f.file.Name)
}

func (fs *zipFS) Lstat(abspath string) (os.FileInfo, error) {
	_, fi, err := fs.stat(abspath)
	return fi, err
}

func (fs *zipFS) Stat(abspath string) (os.FileInfo, error) {
	_, fi, err := fs.stat(abspath)
	return fi, err
}

func (fs *zipFS) ReadDir(abspath string) ([]os.FileInfo, error) {
	i, fi, err := fs.stat(abspath)
	if err != nil {
		return nil, err
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("ReadDir: %s is not a directory", abspath)
	}

	var list []os.FileInfo

	// make dirname the prefix that file names must start with to be considered
	// in this directory. we must special case the root directory because, per
	// the spec of this package, zip file entries MUST NOT start with /, so we
	// should not append /, as we would in every other case.
	var dirname string
	if isRoot(abspath) {
		dirname = ""
	} else {
		zippath, err := zipPath(abspath)
		if err != nil {
			return nil, err
		}
		dirname = zippath + "/"
	}
	prevname := ""
	for _, e := range fs.list[i:] {
		if !strings.HasPrefix(e.Name, dirname) {
			break // not in the same directory anymore
		}
		name := e.Name[len(dirname):] // local name
		file := e
		if i := strings.IndexRune(name, '/'); i >= 0 {
			// We infer directories from files in subdirectories.
			// If we have x/y, return a directory entry for x.
			name = name[0:i] // keep local directory name only
			file = nil
		}
		// If we have x/y and x/z, don't return two directory entries for x.
		// TODO(gri): It should be possible to do this more efficiently
		// by determining the (fs.list) range of local directory entries
		// (via two binary searches).
		if name != prevname {
			list = append(list, zipFI{name, file})
			prevname = name
		}
	}

	return list, nil
}

func New(rc *zip.ReadCloser, name string) vfs.FileSystem {
	list := make(zipList, len(rc.File))
	copy(list, rc.File) // sort a copy of rc.File
	sort.Sort(list)
	return &zipFS{rc, list, name}
}

type zipList []*zip.File

// zipList implements sort.Interface
func (z zipList) Len() int           { return len(z) }
func (z zipList) Less(i, j int) bool { return z[i].Name < z[j].Name }
func (z zipList) Swap(i, j int)      { z[i], z[j] = z[j], z[i] }

// lookup returns the smallest index of an entry with an exact match
// for name, or an inexact match starting with name/. If there is no
// such entry, the result is -1, false.
func (z zipList) lookup(name string) (index int, exact bool) {
	// look for exact match first (name comes before name/ in z)
	i := sort.Search(len(z), func(i int) bool {
		return name <= z[i].Name
	})
	if i >= len(z) {
		return -1, false
	}
	// 0 <= i < len(z)
	if z[i].Name == name {
		return i, true
	}

	// look for inexact match (must be in z[i:], if present)
	z = z[i:]
	name += "/"
	j := sort.Search(len(z), func(i int) bool {
		return name <= z[i].Name
	})
	if j >= len(z) {
		return -1, false
	}
	// 0 <= j < len(z)
	if strings.HasPrefix(z[j].Name, name) {
		return i + j, false
	}

	return -1, false
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build buildlet

// The buildlet is an HTTP server that untars content to disk and runs
// commands it has untarred, streaming their output back over HTTP.
// It is part of Go's continuous build system.
//
// This program intentionally allows remote code execution, and
// provides no security of its own. It is assumed that any user uses
// it with an appropriately-configured firewall between their VM
// instances.
package main // import "golang.org/x/tools/dashboard/buildlet"

import (
	"archive/tar"
	"compress/gzip"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"google.golang.org/cloud/compute/metadata"
)

var (
	scratchDir = flag.String("scratchdir", "", "Temporary directory to use. The contents of this directory may be deleted at any time. If empty, TempDir is used to create one.")
	listenAddr = flag.String("listen", defaultListenAddr(), "address to listen on. Warning: this service is inherently insecure and offers no protection of its own. Do not expose this port to the world.")
)

func defaultListenAddr() string {
	if runtime.GOOS == "darwin" {
		// Darwin will never run on GCE, so let's always
		// listen on a high port (so we don't need to be
		// root).
		return ":5936"
	}
	if !metadata.OnGCE() {
		return "localhost:5936"
	}
	// In production, default to port 80 or 443, depending on
	// whether TLS is configured.
	if metadataValue("tls-cert") != "" {
		return ":443"
	}
	return ":80"
}

func main() {
	flag.Parse()
	if !metadata.OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") {
		log.Printf("** WARNING ***  This server is unsafe and offers no security. Be careful.")
	}
	if runtime.GOOS == "plan9" {
		// Plan 9 is too slow on GCE, so stop running run.rc after the basics.
		// See https://golang.org/cl/2522 and https://golang.org/issue/9491
		// TODO(bradfitz): once the buildlet has environment variable support,
		// the coordinator can send this in, and this variable can be part of
		// the build configuration struct instead of hard-coded here.
		// But no need for environment variables quite yet.
		os.Setenv("GOTESTONLY", "std")
	}

	if *scratchDir == "" {
		dir, err := ioutil.TempDir("", "buildlet-scatch")
		if err != nil {
			log.Fatalf("error creating scratchdir with ioutil.TempDir: %v", err)
		}
		*scratchDir = dir
	}
	if _, err := os.Lstat(*scratchDir); err != nil {
		log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
	}
	http.HandleFunc("/", handleRoot)

	password := metadataValue("password")
	http.Handle("/writetgz", requirePassword{http.HandlerFunc(handleWriteTGZ), password})
	http.Handle("/exec", requirePassword{http.HandlerFunc(handleExec), password})
	// TODO: removeall

	tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key")
	if (tlsCert == "") != (tlsKey == "") {
		log.Fatalf("tls-cert and tls-key must both be supplied, or neither.")
	}

	log.Printf("Listening on %s ...", *listenAddr)
	ln, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		log.Fatalf("Failed to listen on %s: %v", *listenAddr, err)
	}
	ln = tcpKeepAliveListener{ln.(*net.TCPListener)}

	var srv http.Server
	if tlsCert != "" {
		cert, err := tls.X509KeyPair([]byte(tlsCert), []byte(tlsKey))
		if err != nil {
			log.Fatalf("TLS cert error: %v", err)
		}
		tlsConf := &tls.Config{
			Certificates: []tls.Certificate{cert},
		}
		ln = tls.NewListener(ln, tlsConf)
	}

	log.Fatalf("Serve: %v", srv.Serve(ln))
}

// metadataValue returns the GCE metadata instance value for the given key.
//
// If not running on GCE, it falls back to using environment variables
// for local development.
func metadataValue(key string) string {
	// The common case:
	if metadata.OnGCE() {
		v, err := metadata.InstanceAttributeValue(key)
		if err != nil {
			log.Fatalf("metadata.InstanceAttributeValue(%q): %v", key, err)
		}
		return v
	}

	// Else let developers use environment variables to fake
	// metadata keys, for local testing.
	envKey := "GCEMETA_" + strings.Replace(key, "-", "_", -1)
	v := os.Getenv(envKey)
	// Respect curl-style '@' prefix to mean the rest is a filename.
	if strings.HasPrefix(v, "@") {
		slurp, err := ioutil.ReadFile(v[1:])
		if err != nil {
			log.Fatalf("Error reading file for GCEMETA_%v: %v", key, err)
		}
		return string(slurp)
	}
	if v == "" {
		log.Printf("Warning: not running on GCE, and no %v environment variable defined", envKey)
	}
	return v
}

// tcpKeepAliveListener is a net.Listener that sets TCP keep-alive
// timeouts on accepted connections.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH)
}

func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
	if r.Method != "PUT" {
		http.Error(w, "requires PUT method", http.StatusBadRequest)
		return
	}
	err := untar(r.Body, *scratchDir)
	if err != nil {
		status := http.StatusInternalServerError
		if he, ok := err.(httpStatuser); ok {
			status = he.httpStatus()
		}
		http.Error(w, err.Error(), status)
		return
	}
	io.WriteString(w, "OK")
}

// untar reads the gzip-compressed tar file from r and writes it into dir.
func untar(r io.Reader, dir string) error {
	zr, err := gzip.NewReader(r)
	if err != nil {
		return badRequest("requires gzip-compressed body: " + err.Error())
	}
	tr := tar.NewReader(zr)
	for {
		f, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("tar reading error: %v", err)
			return badRequest("tar error: " + err.Error())
		}
		if !validRelPath(f.Name) {
			return badRequest(fmt.Sprintf("tar file contained invalid name %q", f.Name))
		}
		rel := filepath.FromSlash(f.Name)
		abs := filepath.Join(dir, rel)

		fi := f.FileInfo()
		mode := fi.Mode()
		switch {
		case mode.IsRegular():
			// Make the directory. This is redundant because it should
			// already be made by a directory entry in the tar
			// beforehand. Thus, don't check for errors; the next
			// write will fail with the same error.
			os.MkdirAll(filepath.Dir(abs), 0755)
			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
			if err != nil {
				return err
			}
			n, err := io.Copy(wf, tr)
			if closeErr := wf.Close(); closeErr != nil && err == nil {
				err = closeErr
			}
			if err != nil {
				return fmt.Errorf("error writing to %s: %v", abs, err)
			}
			if n != f.Size {
				return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
			}
			log.Printf("wrote %s", abs)
		case mode.IsDir():
			if err := os.MkdirAll(abs, 0755); err != nil {
				return err
			}
		default:
			return badRequest(fmt.Sprintf("tar file entry %s contained unsupported file type %v", f.Name, mode))
		}
	}
	return nil
}

// Process-State is an HTTP Trailer set in the /exec handler to "ok"
// on success, or os.ProcessState.String() on failure.
const hdrProcessState = "Process-State"

func handleExec(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "requires POST method", http.StatusBadRequest)
		return
	}
	if r.ProtoMajor*10+r.ProtoMinor < 11 {
		// We need trailers, only available in HTTP/1.1 or HTTP/2.
		http.Error(w, "HTTP/1.1 or higher required", http.StatusBadRequest)
		return
	}

	w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it

	cmdPath := r.FormValue("cmd") // required
	if !validRelPath(cmdPath) {
		http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
		return
	}
	if f, ok := w.(http.Flusher); ok {
		f.Flush()
	}

	absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
	cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
	cmd.Dir = filepath.Dir(absCmd)
	cmdOutput := &flushWriter{w: w}
	cmd.Stdout = cmdOutput
	cmd.Stderr = cmdOutput
	err := cmd.Run()
	state := "ok"
	if err != nil {
		if ps := cmd.ProcessState; ps != nil {
			state = ps.String()
		} else {
			state = err.Error()
		}
	}
	w.Header().Set(hdrProcessState, state)
	log.Printf("Run = %s", state)
}

// flushWriter is an io.Writer wrapper that writes to w and
// Flushes the output immediately, if w is an http.Flusher.
type flushWriter struct {
	mu sync.Mutex
	w  http.ResponseWriter
}

func (hw *flushWriter) Write(p []byte) (n int, err error) {
	hw.mu.Lock()
	defer hw.mu.Unlock()
	n, err = hw.w.Write(p)
	if f, ok := hw.w.(http.Flusher); ok {
		f.Flush()
	}
	return
}

func validRelPath(p string) bool {
	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
		return false
	}
	return true
}

type httpStatuser interface {
	error
	httpStatus() int
}

type httpError struct {
	statusCode int
	msg        string
}

func (he httpError) Error() string   { return he.msg }
func (he httpError) httpStatus() int { return he.statusCode }

func badRequest(msg string) error {
	return httpError{http.StatusBadRequest, msg}
}

// requirePassword is an http.Handler auth wrapper that enforces a
// HTTP Basic password. The username is ignored.
type requirePassword struct {
	h        http.Handler
	password string // empty means no password
}

func (h requirePassword) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	_, gotPass, _ := r.BasicAuth()
	if h.password != "" && h.password != gotPass {
		http.Error(w, "invalid password", http.StatusForbidden)
		return
	}
	h.h.ServeHTTP(w, r)
}
<commit_msg>dashboard/buildlet: fix start-up crash when TLS attributes aren't set<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build buildlet

// The buildlet is an HTTP server that untars content to disk and runs
// commands it has untarred, streaming their output back over HTTP.
// It is part of Go's continuous build system.
//
// This program intentionally allows remote code execution, and
// provides no security of its own. It is assumed that any user uses
// it with an appropriately-configured firewall between their VM
// instances.
package main // import "golang.org/x/tools/dashboard/buildlet"

import (
	"archive/tar"
	"compress/gzip"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"google.golang.org/cloud/compute/metadata"
)

var (
	scratchDir = flag.String("scratchdir", "", "Temporary directory to use. The contents of this directory may be deleted at any time. If empty, TempDir is used to create one.")
	listenAddr = flag.String("listen", defaultListenAddr(), "address to listen on. Warning: this service is inherently insecure and offers no protection of its own. Do not expose this port to the world.")
)

func defaultListenAddr() string {
	if runtime.GOOS == "darwin" {
		// Darwin will never run on GCE, so let's always
		// listen on a high port (so we don't need to be
		// root).
		return ":5936"
	}
	if !metadata.OnGCE() {
		return "localhost:5936"
	}
	// In production, default to port 80 or 443, depending on
	// whether TLS is configured.
	if metadataValue("tls-cert") != "" {
		return ":443"
	}
	return ":80"
}

func main() {
	flag.Parse()
	if !metadata.OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") {
		log.Printf("** WARNING ***  This server is unsafe and offers no security. Be careful.")
	}
	if runtime.GOOS == "plan9" {
		// Plan 9 is too slow on GCE, so stop running run.rc after the basics.
		// See https://golang.org/cl/2522 and https://golang.org/issue/9491
		// TODO(bradfitz): once the buildlet has environment variable support,
		// the coordinator can send this in, and this variable can be part of
		// the build configuration struct instead of hard-coded here.
		// But no need for environment variables quite yet.
		os.Setenv("GOTESTONLY", "std")
	}

	if *scratchDir == "" {
		dir, err := ioutil.TempDir("", "buildlet-scatch")
		if err != nil {
			log.Fatalf("error creating scratchdir with ioutil.TempDir: %v", err)
		}
		*scratchDir = dir
	}
	if _, err := os.Lstat(*scratchDir); err != nil {
		log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
	}
	http.HandleFunc("/", handleRoot)

	password := metadataValue("password")
	http.Handle("/writetgz", requirePassword{http.HandlerFunc(handleWriteTGZ), password})
	http.Handle("/exec", requirePassword{http.HandlerFunc(handleExec), password})
	// TODO: removeall

	tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key")
	if (tlsCert == "") != (tlsKey == "") {
		log.Fatalf("tls-cert and tls-key must both be supplied, or neither.")
	}

	log.Printf("Listening on %s ...", *listenAddr)
	ln, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		log.Fatalf("Failed to listen on %s: %v", *listenAddr, err)
	}
	ln = tcpKeepAliveListener{ln.(*net.TCPListener)}

	var srv http.Server
	if tlsCert != "" {
		cert, err := tls.X509KeyPair([]byte(tlsCert), []byte(tlsKey))
		if err != nil {
			log.Fatalf("TLS cert error: %v", err)
		}
		tlsConf := &tls.Config{
			Certificates: []tls.Certificate{cert},
		}
		ln = tls.NewListener(ln, tlsConf)
	}

	log.Fatalf("Serve: %v", srv.Serve(ln))
}

// metadataValue returns the GCE metadata instance value for the given key.
// If the metadata is not defined, the returned string is empty.
//
// If not running on GCE, it falls back to using environment variables
// for local development.
func metadataValue(key string) string {
	// The common case:
	if metadata.OnGCE() {
		v, err := metadata.InstanceAttributeValue(key)
		if _, notDefined := err.(metadata.NotDefinedError); notDefined {
			return ""
		}
		if err != nil {
			log.Fatalf("metadata.InstanceAttributeValue(%q): %v", key, err)
		}
		return v
	}

	// Else let developers use environment variables to fake
	// metadata keys, for local testing.
	envKey := "GCEMETA_" + strings.Replace(key, "-", "_", -1)
	v := os.Getenv(envKey)
	// Respect curl-style '@' prefix to mean the rest is a filename.
	if strings.HasPrefix(v, "@") {
		slurp, err := ioutil.ReadFile(v[1:])
		if err != nil {
			log.Fatalf("Error reading file for GCEMETA_%v: %v", key, err)
		}
		return string(slurp)
	}
	if v == "" {
		log.Printf("Warning: not running on GCE, and no %v environment variable defined", envKey)
	}
	return v
}

// tcpKeepAliveListener is a net.Listener that sets TCP keep-alive
// timeouts on accepted connections.
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(3 * time.Minute)
	return tc, nil
}

func handleRoot(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH)
}

func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
	if r.Method != "PUT" {
		http.Error(w, "requires PUT method", http.StatusBadRequest)
		return
	}
	err := untar(r.Body, *scratchDir)
	if err != nil {
		status := http.StatusInternalServerError
		if he, ok := err.(httpStatuser); ok {
			status = he.httpStatus()
		}
		http.Error(w, err.Error(), status)
		return
	}
	io.WriteString(w, "OK")
}

// untar reads the gzip-compressed tar file from r and writes it into dir.
func untar(r io.Reader, dir string) error {
	zr, err := gzip.NewReader(r)
	if err != nil {
		return badRequest("requires gzip-compressed body: " + err.Error())
	}
	tr := tar.NewReader(zr)
	for {
		f, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Printf("tar reading error: %v", err)
			return badRequest("tar error: " + err.Error())
		}
		if !validRelPath(f.Name) {
			return badRequest(fmt.Sprintf("tar file contained invalid name %q", f.Name))
		}
		rel := filepath.FromSlash(f.Name)
		abs := filepath.Join(dir, rel)

		fi := f.FileInfo()
		mode := fi.Mode()
		switch {
		case mode.IsRegular():
			// Make the directory. This is redundant because it should
			// already be made by a directory entry in the tar
			// beforehand. Thus, don't check for errors; the next
			// write will fail with the same error.
			os.MkdirAll(filepath.Dir(abs), 0755)
			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
			if err != nil {
				return err
			}
			n, err := io.Copy(wf, tr)
			if closeErr := wf.Close(); closeErr != nil && err == nil {
				err = closeErr
			}
			if err != nil {
				return fmt.Errorf("error writing to %s: %v", abs, err)
			}
			if n != f.Size {
				return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
			}
			log.Printf("wrote %s", abs)
		case mode.IsDir():
			if err := os.MkdirAll(abs, 0755); err != nil {
				return err
			}
		default:
			return badRequest(fmt.Sprintf("tar file entry %s contained unsupported file type %v", f.Name, mode))
		}
	}
	return nil
}

// Process-State is an HTTP Trailer set in the /exec handler to "ok"
// on success, or os.ProcessState.String() on failure.
const hdrProcessState = "Process-State"

func handleExec(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		http.Error(w, "requires POST method", http.StatusBadRequest)
		return
	}
	if r.ProtoMajor*10+r.ProtoMinor < 11 {
		// We need trailers, only available in HTTP/1.1 or HTTP/2.
		http.Error(w, "HTTP/1.1 or higher required", http.StatusBadRequest)
		return
	}

	w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it

	cmdPath := r.FormValue("cmd") // required
	if !validRelPath(cmdPath) {
		http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
		return
	}
	if f, ok := w.(http.Flusher); ok {
		f.Flush()
	}

	absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
	cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
	cmd.Dir = filepath.Dir(absCmd)
	cmdOutput := &flushWriter{w: w}
	cmd.Stdout = cmdOutput
	cmd.Stderr = cmdOutput
	err := cmd.Run()
	state := "ok"
	if err != nil {
		if ps := cmd.ProcessState; ps != nil {
			state = ps.String()
		} else {
			state = err.Error()
		}
	}
	w.Header().Set(hdrProcessState, state)
	log.Printf("Run = %s", state)
}

// flushWriter is an io.Writer wrapper that writes to w and
// Flushes the output immediately, if w is an http.Flusher.
type flushWriter struct {
	mu sync.Mutex
	w  http.ResponseWriter
}

func (hw *flushWriter) Write(p []byte) (n int, err error) {
	hw.mu.Lock()
	defer hw.mu.Unlock()
	n, err = hw.w.Write(p)
	if f, ok := hw.w.(http.Flusher); ok {
		f.Flush()
	}
	return
}

func validRelPath(p string) bool {
	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
		return false
	}
	return true
}

type httpStatuser interface {
	error
	httpStatus() int
}

type httpError struct {
	statusCode int
	msg        string
}

func (he httpError) Error() string   { return he.msg }
func (he httpError) httpStatus() int { return he.statusCode }

func badRequest(msg string) error {
	return httpError{http.StatusBadRequest, msg}
}

// requirePassword is an http.Handler auth wrapper that enforces a
// HTTP Basic password. The username is ignored.
type requirePassword struct {
	h        http.Handler
	password string // empty means no password
}

func (h requirePassword) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	_, gotPass, _ := r.BasicAuth()
	if h.password != "" && h.password != gotPass {
		http.Error(w, "invalid password", http.StatusForbidden)
		return
	}
	h.h.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/go-codec/codec"
	"github.com/keybase/kbfs/kbfsmd"
)

// PrevRevisionAndCount track the MD version of a previous revision of
// a dir entry, and how many revisions ago that was from the current
// revision.
type PrevRevisionAndCount struct {
	Revision kbfsmd.Revision `codec:"r"`
	Count    uint8           `codec:"c"`

	codec.UnknownFieldSetHandler
}

// minPrevRevisionSlotCounts defines the "min count" of each
// corresponding entry in a `PrevRevisions` slice.  The length of
// `minPrevRevisionSlotCounts` is the max length of a `PrevRevisions`
// slice.
var minPrevRevisionSlotCounts = [...]uint8{1, 5, 20, 50, 100}

// PrevRevisions tracks several previous versions of a file in order
// of descending revision number, starting with the most recent.
type PrevRevisions []PrevRevisionAndCount

// addRevision returns a copy of `pr` with a new immediately-previous
// revision added, with the existing entries moved or overwritten to
// accomodate the new entry, and with increased counts.  Any existing
// revisions smaller than or equal to minRev will be removed.
func (pr PrevRevisions) addRevision(
	r, minRev kbfsmd.Revision) (ret PrevRevisions) {
	newLength := len(pr)
	if newLength < len(minPrevRevisionSlotCounts) {
		newLength++
	}
	ret = make(PrevRevisions, newLength)
	copy(ret, pr)
	earliestGoodSlot := 0
	numDropped := 0

	for i, prc := range ret {
		if prc.Count == 255 {
			panic("Previous revision count is about to overflow")
		} else if prc.Revision >= r {
			if numDropped > 0 {
				panic("Revision too large after dropping one")
			}
			// The revision number is bigger than expected (e.g. it
			// was made on an unmerged branch)
			ret[i] = PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
			earliestGoodSlot = i + 1
			continue
		} else if prc.Revision <= minRev {
			// This revision is too old (or is empty), so remove it.
			ret[i] = PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
			numDropped++
			continue
		} else if numDropped > 0 {
			panic("Once we've dropped one, we should drop all the rest")
		}
		ret[i].Count++
	}

	// Cut out the revisions that are newer than `r` (e.g., because
	// they are from an unmerged branch).
	if earliestGoodSlot > 0 {
		ret = ret[earliestGoodSlot:]
	}

	// Drop revisions off the end that are too old.
	if numDropped == len(ret) {
		// Leave the first slot available for overwriting.
		ret = ret[:1]
	} else if numDropped > 1 {
		ret = ret[:len(ret)-(numDropped-1)]
	}

	for i := len(ret) - 1; i >= 1; i-- {
		toMove := ret[i-1]
		if ret[i].Count == 0 || toMove.Count >= minPrevRevisionSlotCounts[i] {
			ret[i] = toMove
			ret[i-1] = PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
		}
	}
	ret[0] = PrevRevisionAndCount{
		Revision: r,
		Count:    1,
	}
	return ret
}
<commit_msg>prev_revisions: rework shifting code for clarity<commit_after>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"github.com/keybase/go-codec/codec"
	"github.com/keybase/kbfs/kbfsmd"
)

// PrevRevisionAndCount track the MD version of a previous revision of
// a dir entry, and how many revisions ago that was from the current
// revision.
type PrevRevisionAndCount struct {
	Revision kbfsmd.Revision `codec:"r"`
	Count    uint8           `codec:"c"`

	codec.UnknownFieldSetHandler
}

// minPrevRevisionSlotCounts defines the "min count" of each
// corresponding entry in a `PrevRevisions` slice.  The length of
// `minPrevRevisionSlotCounts` is the max length of a `PrevRevisions`
// slice.
var minPrevRevisionSlotCounts = [...]uint8{1, 5, 20, 50, 100}

// PrevRevisions tracks several previous versions of a file in order
// of descending revision number, starting with the most recent.
type PrevRevisions []PrevRevisionAndCount

// addRevision returns a copy of `pr` with a new immediately-previous
// revision added, with the existing entries moved or overwritten to
// accomodate the new entry, and with increased counts.  Any existing
// revisions smaller than or equal to minRev will be removed.
func (pr PrevRevisions) addRevision(
	r, minRev kbfsmd.Revision) (ret PrevRevisions) {
	newLength := len(pr)
	if newLength < len(minPrevRevisionSlotCounts) {
		newLength++
	}
	ret = make(PrevRevisions, newLength)
	copy(ret, pr)
	earliestGoodSlot := 0
	numDropped := 0

	for i, prc := range ret {
		if prc.Count == 255 {
			panic("Previous revision count is about to overflow")
		} else if prc.Revision >= r {
			if numDropped > 0 {
				panic("Revision too large after dropping one")
			}
			// The revision number is bigger than expected (e.g. it
			// was made on an unmerged branch)
			ret[i] = PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
			earliestGoodSlot = i + 1
			continue
		} else if prc.Revision <= minRev {
			// This revision is too old (or is empty), so remove it.
			ret[i] = PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
			numDropped++
			continue
		} else if numDropped > 0 {
			panic("Once we've dropped one, we should drop all the rest")
		}
		ret[i].Count++
	}

	// Cut out the revisions that are newer than `r` (e.g., because
	// they are from an unmerged branch).
	if earliestGoodSlot > 0 {
		ret = ret[earliestGoodSlot:]
	}

	// Drop revisions off the end that are too old.
	if numDropped == len(ret) {
		// Leave the first slot available for overwriting.
		ret = ret[:1]
	} else if numDropped > 1 {
		ret = ret[:len(ret)-(numDropped-1)]
	}

	for i := len(ret) - 1; i >= 1; i-- {
		// Check if we can shift over the entry in slot i-1.
		if ret[i].Count == 0 || ret[i-1].Count >= minPrevRevisionSlotCounts[i] {
			ret[i], ret[i-1] = ret[i-1], PrevRevisionAndCount{
				Revision: kbfsmd.RevisionUninitialized,
				Count:    0,
			}
		}
	}
	ret[0] = PrevRevisionAndCount{
		Revision: r,
		Count:    1,
	}
	return ret
}
<|endoftext|>


<commit_before>package template_metrics

import (
	"fmt"
	metrics "github.com/yvasiyarov/go-metrics" // max,mean,min,stddev,percentile
	"html/template"                            // ToDo: how can I change this from outside?
	"io"
	"time"
)

// print infomation on each request
var Verbose = false

// a set of proxies
var proxyRegistry = make(map[string](*proxyTemplate))

type proxyTemplate struct {
	name     string
	original *template.Template
	timer    metrics.Timer
}

func newProxyTemplate(name string, template *template.Template) *proxyTemplate {
	return &proxyTemplate{
		name:     name,
		original: template,
		timer:    metrics.NewTimer(),
	}
}

//print the elapsed time on each request if Verbose flag is true
func (proxy *proxyTemplate) printVerbose(elapsedTime time.Duration) {
	fmt.Printf("time:%v\ttemplate:%s\telapsed:%f\n",
		time.Now(),
		proxy.original.Name(),
		elapsedTime.Seconds(),
	)
}

//measure the time
func (proxy *proxyTemplate) measure(startTime time.Time) {
	elapsedTime := time.Now().Sub(startTime)
	proxy.timer.Update(elapsedTime)
	if Verbose {
		proxy.printVerbose(elapsedTime)
	}
}

////// instrument functions

// instrument template.Execute
func (proxy *proxyTemplate) Execute(wr io.Writer, data interface{}) error {
	startTime := time.Now()
	error := proxy.original.Execute(wr, data)
	defer proxy.measure(startTime)
	return error
}

// instrucment template.ExecuteTemplate
func (proxy *proxyTemplate) ExecuteTemplate(wr io.Writer, name string, data interface{}) error {
	startTime := time.Now()
	error := proxy.original.ExecuteTemplate(wr, name, data)
	defer proxy.measure(startTime)
	return error
}

///// package functions

//Wrap  instrument template
func WrapTemplate(name string, template *template.Template) *proxyTemplate {
	proxy := newProxyTemplate(name, template)
	proxyRegistry[template.Name()] = proxy
	return proxy
}

//Print  print the metrics in each second
func Print() {
	go func() {
		time.Sleep(1000 * time.Millisecond)
		for {
			startTime := time.Now()
			for name, proxy := range proxyRegistry {
				timer := proxy.timer
				count := timer.Count()
				if count > 0 {
					fmt.Printf("time:%v\ttemplate:%s\tcount:%d\tmax:%f\tmean:%f\tmin:%f\tpercentile95:%f\n",
						time.Now(),
						name,
						timer.Count(),
						float64(timer.Max())/float64(time.Second),
						timer.Mean()/float64(time.Second),
						float64(timer.Min())/float64(time.Second),
						timer.Percentile(0.95)/float64(time.Second),
					)
					proxy.timer = metrics.NewTimer()
				}
			}
			elapsedTime := time.Now().Sub(startTime)
			time.Sleep(1000*time.Millisecond - elapsedTime)
		}
	}()
}
<commit_msg>template.Name() might be nil, do not use<commit_after>package template_metrics

import (
	"fmt"
	metrics "github.com/yvasiyarov/go-metrics" // max,mean,min,stddev,percentile
	"html/template"                            // ToDo: how can I change this from outside?
	"io"
	"time"
)

// print infomation on each request
var Verbose = false

// a set of proxies
var proxyRegistry = make(map[string](*proxyTemplate))

type proxyTemplate struct {
	name     string
	original *template.Template
	timer    metrics.Timer
}

func newProxyTemplate(name string, template *template.Template) *proxyTemplate {
	return &proxyTemplate{
		name:     name,
		original: template,
		timer:    metrics.NewTimer(),
	}
}

//print the elapsed time on each request if Verbose flag is true
func (proxy *proxyTemplate) printVerbose(elapsedTime time.Duration) {
	fmt.Printf("time:%v\ttemplate:%s\telapsed:%f\n",
		time.Now(),
		proxy.name,
		elapsedTime.Seconds(),
	)
}

//measure the time
func (proxy *proxyTemplate) measure(startTime time.Time) {
	elapsedTime := time.Now().Sub(startTime)
	proxy.timer.Update(elapsedTime)
	if Verbose {
		proxy.printVerbose(elapsedTime)
	}
}

////// instrument functions

// instrument template.Execute
func (proxy *proxyTemplate) Execute(wr io.Writer, data interface{}) error {
	startTime := time.Now()
	error := proxy.original.Execute(wr, data)
	defer proxy.measure(startTime)
	return error
}

// instrucment template.ExecuteTemplate
func (proxy *proxyTemplate) ExecuteTemplate(wr io.Writer, name string, data interface{}) error {
	startTime := time.Now()
	error := proxy.original.ExecuteTemplate(wr, name, data)
	defer proxy.measure(startTime)
	return error
}

///// package functions

//Wrap  instrument template
func WrapTemplate(name string, template *template.Template) *proxyTemplate {
	proxy := newProxyTemplate(name, template)
	proxyRegistry[name] = proxy
	return proxy
}

//Print  print the metrics in each second
func Print() {
	go func() {
		time.Sleep(1000 * time.Millisecond)
		for {
			startTime := time.Now()
			for name, proxy := range proxyRegistry {
				timer := proxy.timer
				count := timer.Count()
				if count > 0 {
					fmt.Printf("time:%v\ttemplate:%s\tcount:%d\tmax:%f\tmean:%f\tmin:%f\tpercentile95:%f\n",
						time.Now(),
						name,
						timer.Count(),
						float64(timer.Max())/float64(time.Second),
						timer.Mean()/float64(time.Second),
						float64(timer.Min())/float64(time.Second),
						timer.Percentile(0.95)/float64(time.Second),
					)
					proxy.timer = metrics.NewTimer()
				}
			}
			elapsedTime := time.Now().Sub(startTime)
			time.Sleep(1000*time.Millisecond - elapsedTime)
		}
	}()
}
<|endoftext|>


<commit_before>// Package analytic provides an exponential integrator for systems of ordinary
// differential equations modeling temperature of electronic systems.
//
// The initial system is
//
//         dQ'
//     C * -- + G * (Q - Qamb) = M * P
//         dt
//
//     Q = M**T * Q'
//
// where C is a diagonal matrix of the thermal capacitance; G is a symmetric
// matrix of the thermal conductance; Q' is a vector of the temperature of the
// thermal nodes; Q is a subset of Q corresponding to the processing elements;
// Qamb is a vector of the ambient temperature; P is a vector of the power
// consumption of the processing elements; and M is a diagonal matrix whose
// diagonal elements equal to unity that maps the power dissipation of the
// processing elements onto the thermal nodes.
//
// The transformed system is
//
//     dS
//     -- = A * S + B * P
//     dt
//
//     Q = B**T * S + Qamb
//
// where
//
//     S = D**(-1) * (Q' - Qamb),
//     A = -D * G * D,
//     B = D * M, and
//     D = C**(-½).
//
// The eigendecomposition of A, which is real and symmetric, is
//
//     A = U * diag(Λ) * U**T.
//
// Assuming the power dissipation from time 0 to time t is constant and is equal
// to P(0), the solution to the system at time t is as follows:
//
//     S(t) = E(t) * S(0) + F(t) * P(0)
//
// where
//
//     E(t) = exp(A * t) = U * diag(exp(λi * t)) * U**T and
//     F(t) = A**(-1) * (exp(A * t) - I) * B
//          = U * diag((exp(λi * t) - 1) / λi) * U**T * B.
package analytic
<commit_msg>temperature/analytic: fix the description<commit_after>// Package analytic provides an exponential integrator for systems of ordinary
// differential equations modeling temperature of electronic systems.
//
// The initial system is
//
//         dQ'
//     C * -- + G * (Q - Qamb) = M * P
//         dt
//
//     Q = M**T * Q'
//
// where C is a diagonal matrix of the thermal capacitance; G is a symmetric
// matrix of the thermal conductance; Q' is a vector of the temperature of the
// thermal nodes; Q is a subset of Q corresponding to the processing elements;
// Qamb is a vector of the ambient temperature; P is a vector of the power
// consumption of the processing elements; and M is a diagonal matrix whose
// diagonal elements equal to unity that maps the power dissipation of the
// processing elements onto the thermal nodes.
//
// The transformed system is
//
//     dS
//     -- = A * S + B * P
//     dt
//
//     Q = B**T * S + Qamb
//
// where
//
//     S = D**(-1) * (Q' - Qamb),
//     A = -D * G * D,
//     B = D * M, and
//     D = C**(-½).
//
// The eigendecomposition of A, which is real and symmetric, is
//
//     A = U * diag(Λ) * U**T.
//
// Assuming that the power dissipation from time 0 to time t is constant, the
// solution to the system at time t is as follows:
//
//     S(t) = E(t) * S(0) + F(t) * P(0)
//
// where
//
//     E(t) = exp(A * t) = U * diag(exp(λi * t)) * U**T and
//     F(t) = A**(-1) * (exp(A * t) - I) * B
//          = U * diag((exp(λi * t) - 1) / λi) * U**T * B.
package analytic
<|endoftext|>


<commit_before>package main

import (
	"os"
	"log"
	"database/sql"
	"github.com/mattn/go-sqlite3"
)

func main() {
	if len(os.Args) != 3 {
		usage()
	}
	dbfn := os.Args[1]
	query := os.Args[2]

	db, err := sql.Open("sqlite3", dbfn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	_ = sqlite3.ErrError

	var dummy string
	err = db.QueryRow("select name from sqlite_master where type='table' and name='bilm'").Scan(&dummy)
	if err == sql.ErrNoRows {
		log.Fatalf("%s does not appear to be a bilm index created using bilm_add", dbfn)
	}

	rows, err := db.Query("select message from bilm where message match ?", query)
	if err != nil {
		log.Fatal(err)
	}
	for rows.Next() {
		var message string
		if err := rows.Scan(&message); err != nil {
			log.Fatal(err)
		}
		os.Stdout.Write([]byte(message+"\n"))
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func usage() {
	log.Fatal("Usage: bilm_query /path/to/index \"the quick * fox\"")
}

<commit_msg>bilm_query: exit code 1 if no results found<commit_after>package main

import (
	"os"
	"log"
	"database/sql"
	"github.com/mattn/go-sqlite3"
)

func main() {
	os.Exit(main_impl());
}

func main_impl() int {
	if len(os.Args) != 3 {
		usage()
	}
	dbfn := os.Args[1]
	query := os.Args[2]

	db, err := sql.Open("sqlite3", dbfn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	_ = sqlite3.ErrError

	var dummy string
	err = db.QueryRow("select name from sqlite_master where type='table' and name='bilm'").Scan(&dummy)
	if err == sql.ErrNoRows {
		log.Fatalf("%s does not appear to be a bilm index created using bilm_add", dbfn)
	}

	rows, err := db.Query("select message from bilm where message match ?", query)
	if err != nil {
		log.Fatal(err)
	}
	exit_code := 1
	for rows.Next() {
		var message string
		if err := rows.Scan(&message); err != nil {
			log.Fatal(err)
		}
		os.Stdout.Write([]byte(message+"\n"))
		exit_code = 0
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	return exit_code
}

func usage() {
	log.Fatal("Usage: bilm_query /path/to/index \"the quick * fox\"")
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/sgoertzen/html2text"
	"golang.org/x/net/html"
	"log"
	"strings"
	"time"
)

type Organizer struct {
	Mailbox Mailbox
}

type Mailbox struct {
	Name string
}

type ItemId struct {
	Id        string `xml:"Id,attr"`
	ChangeKey string `xml:"ChangeKey,attr"`
}

type CalendarItem struct {
	ItemId         ItemId
	Subject        string
	DisplayCc      string
	DisplayTo      string
	Start          string
	End            string
	IsAllDayEvent  bool
	Location       string
	MyResponseType string
	Organizer      Organizer
	Body           Body
}

type Body struct {
	BodyType string `xml:"BodyType,attr"`
	Body     string `xml:",chardata"`
}

type Appointment struct {
	ItemId         string
	ChangeKey      string
	Subject        string
	Cc             string
	To             string
	Start          time.Time
	End            time.Time
	IsAllDayEvent  bool
	Location       string
	MyResponseType string
	Organizer      string
	Body           string
	BodyType       string
}

func ParseCalendarFolder(soap string) ItemId {
	decoder := xml.NewDecoder(bytes.NewBufferString(soap))
	var itemId ItemId

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "FolderId" {
				decoder.DecodeElement(&itemId, &se)
				break
			}
		}
	}
	return itemId
}

func ParseAppointments(soap string) []Appointment {
	decoder := xml.NewDecoder(bytes.NewBufferString(soap))

	appointments := make([]Appointment, 0)

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "CalendarItem" {
				var item CalendarItem
				decoder.DecodeElement(&item, &se)
				appointments = append(appointments, item.ToAppointment())
			}
		}
	}
	return appointments
}

func (c CalendarItem) ToAppointment() Appointment {
	app := Appointment{
		ItemId:         c.ItemId.Id,
		ChangeKey:      c.ItemId.ChangeKey,
		Subject:        c.Subject,
		Cc:             c.DisplayCc,
		To:             c.DisplayTo,
		IsAllDayEvent:  c.IsAllDayEvent,
		Location:       c.Location,
		MyResponseType: c.MyResponseType,
		Organizer:      c.Organizer.Mailbox.Name,
		Body:           c.Body.Body,
		BodyType:       c.Body.BodyType,
	}
	if len(c.Start) > 0 {
		t1, err := time.Parse(time.RFC3339, c.Start)
		if err != nil {
			log.Printf("Error while parsing time.  Time string is: ", c.Start, err)
		}
		app.Start = t1
	}

	if len(c.End) > 0 {
		t1, err := time.Parse(time.RFC3339, c.End)
		if err != nil {
			log.Printf("Error while parsing time.  Time string is: ", c.End, err)
		}
		app.End = t1
	}
	return app
}

func (a Appointment) String() string {
	return fmt.Sprintf("%s starting %d", a.Subject, a.Start)
}

func (a *Appointment) BuildDesc() string {
	desc := ""

	addField := func(field string, label string) {
		if len(field) > 0 {
			desc += label + " " + field + "\n"
		}
	}
	addField(a.Organizer, "Organizer:")
	addField(a.To, "To:")
	addField(a.Cc, "Cc:")
	addField(a.MyResponseType, "Response:")
	body := html2text.Textify(a.Body)
	desc += body
	return desc
}

func convertLinks(body string) string {
	r := strings.NewReader(body)
	doc, err := html.Parse(r)
	if err != nil {
		// ...
	}

	var breakers = make(map[string]bool)
	breakers["br"] = true
	breakers["div"] = true
	breakers["tr"] = true
	breakers["li"] = true

	var f func(*html.Node, *bytes.Buffer)
	f = func(n *html.Node, b *bytes.Buffer) {
		processChildren := true

		if n.Type == html.ElementNode && n.Data == "head" {
			return
		} else if n.Type == html.ElementNode && n.Data == "a" && n.FirstChild != nil {
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				f(c, b)
			}
			b.WriteString(fmt.Sprintf(" (link: %s)", n.Attr[0].Val))
			processChildren = false
		} else if n.Type == html.TextNode {
			b.WriteString(n.Data)
		}
		if processChildren {
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				f(c, b)
			}
		}
		if n.Type == html.ElementNode && breakers[n.Data] {
			b.WriteString("\n")
		}
	}
	var buffer bytes.Buffer
	f(doc, &buffer)
	return strings.TrimSpace(buffer.String())
}<commit_msg>Removing unused method<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"github.com/sgoertzen/html2text"
	"log"
	"time"
)

type Organizer struct {
	Mailbox Mailbox
}

type Mailbox struct {
	Name string
}

type ItemId struct {
	Id        string `xml:"Id,attr"`
	ChangeKey string `xml:"ChangeKey,attr"`
}

type CalendarItem struct {
	ItemId         ItemId
	Subject        string
	DisplayCc      string
	DisplayTo      string
	Start          string
	End            string
	IsAllDayEvent  bool
	Location       string
	MyResponseType string
	Organizer      Organizer
	Body           Body
}

type Body struct {
	BodyType string `xml:"BodyType,attr"`
	Body     string `xml:",chardata"`
}

type Appointment struct {
	ItemId         string
	ChangeKey      string
	Subject        string
	Cc             string
	To             string
	Start          time.Time
	End            time.Time
	IsAllDayEvent  bool
	Location       string
	MyResponseType string
	Organizer      string
	Body           string
	BodyType       string
}

func ParseCalendarFolder(soap string) ItemId {
	decoder := xml.NewDecoder(bytes.NewBufferString(soap))
	var itemId ItemId

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "FolderId" {
				decoder.DecodeElement(&itemId, &se)
				break
			}
		}
	}
	return itemId
}

func ParseAppointments(soap string) []Appointment {
	decoder := xml.NewDecoder(bytes.NewBufferString(soap))

	appointments := make([]Appointment, 0)

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "CalendarItem" {
				var item CalendarItem
				decoder.DecodeElement(&item, &se)
				appointments = append(appointments, item.ToAppointment())
			}
		}
	}
	return appointments
}

func (c CalendarItem) ToAppointment() Appointment {
	app := Appointment{
		ItemId:         c.ItemId.Id,
		ChangeKey:      c.ItemId.ChangeKey,
		Subject:        c.Subject,
		Cc:             c.DisplayCc,
		To:             c.DisplayTo,
		IsAllDayEvent:  c.IsAllDayEvent,
		Location:       c.Location,
		MyResponseType: c.MyResponseType,
		Organizer:      c.Organizer.Mailbox.Name,
		Body:           c.Body.Body,
		BodyType:       c.Body.BodyType,
	}
	if len(c.Start) > 0 {
		t1, err := time.Parse(time.RFC3339, c.Start)
		if err != nil {
			log.Printf("Error while parsing time.  Time string is: ", c.Start, err)
		}
		app.Start = t1
	}

	if len(c.End) > 0 {
		t1, err := time.Parse(time.RFC3339, c.End)
		if err != nil {
			log.Printf("Error while parsing time.  Time string is: ", c.End, err)
		}
		app.End = t1
	}
	return app
}

func (a Appointment) String() string {
	return fmt.Sprintf("%s starting %d", a.Subject, a.Start)
}

func (a *Appointment) BuildDesc() string {
	desc := ""

	addField := func(field string, label string) {
		if len(field) > 0 {
			desc += label + " " + field + "\n"
		}
	}
	addField(a.Organizer, "Organizer:")
	addField(a.To, "To:")
	addField(a.Cc, "Cc:")
	addField(a.MyResponseType, "Response:")
	body := html2text.Textify(a.Body)
	desc += body
	return desc
}<|endoftext|>