<commit_before>package magicnum

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"
)

const (
	Unknown    Format = iota // unknown format
	Gzip                     // Gzip compression format
	Tar                      // Tar format; normally used
	Tar1                     // Tar1 header format; normalizes to Tar
	Tar2                     // Tar1 header format; normalizes to Tar
	Zip                      // Zip archive
	ZipEmpty                 // Empty Zip Archive
	ZipSpanned               // Spanned Zip Archive
	Bzip2                    // Bzip2 compression
	LZH                      // LZH compression
	LZW                      // LZW compression
	LZ4                      // LZ4 compression
	RAR                      // RAR 5.0 and later compression
	RAROld                   // Rar pre 1.5 compression
)

// TODO: should Format be more specific? e.g. CompressionFormat, MediaFormat, etc.
type Format int

func (f Format) String() string {
	switch f {
	case Gzip:
		return "gzip"
	case Tar, Tar1, Tar2:
		return "tar"
	case Zip:
		return "zip"
	case ZipEmpty:
		return "empty zip archive"
	case ZipSpanned:
		return "spanned zip archive"
	case Bzip2:
		return "bzip2"
	case LZH:
		return "lzh"
	case LZW:
		return "lzw"
	case LZ4:
		return "lz4"
	case RAR:
		return "rar post 5.0"
	case RAROld:
		return "rar pre 1.5"
	}
	return "unknown"
}

// Ext returns the extension for the format. Formats may have more than one
// accepted extension; alternate extensiona are not supported.
func (f Format) Ext() string {
	switch f {
	case Gzip:
		return ".gz"
	case Tar, Tar1, Tar2:
		return ".tar"
	case Zip, ZipEmpty, ZipSpanned:
		return ".zip"
	case Bzip2:
		return ".bz2"
	case LZH:
		return ".lzh"
	case LZW:
		return ".Z"
	case LZ4:
		return "lz4"
	case RAR, RAROld:
		return ".rar"
	}
	return "unknown"
}

func FormatFromString(s string) Format {
	s = strings.ToLower(s)
	switch s {
	case "gzip", "gz":
		return Gzip
	case "tar":
		return Tar
	case "zip":
		return Zip
	case "bzip2", "bz2":
		return Bzip2
	case "lzh":
		return LZH
	case "lzw", "Z":
		return LZW
	case "lz4":
		return LZ4
	case "rar":
		return RAR
	}
	return Unknown
}

// Magic numbers for supported formats
var (
	headerGzip       = []byte{0x1f, 0x8b}
	headerTar1       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	headerTar2       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	headerZip        = []byte{0x50, 0x4b, 0x03, 0x04}
	headerZipEmpty   = []byte{0x50, 0x4b, 0x05, 0x06}
	headerZipSpanned = []byte{0x50, 0x4b, 0x07, 0x08}
	headerBzip2      = []byte{0x42, 0x5a, 0x68}
	headerLZH        = []byte{0x1F, 0xa0}
	headerLZW        = []byte{0x1F, 0x9d}
	headerLZ4        = []byte{0x18, 0x4d, 0x22, 0x04}
	headerRAR        = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00}
	headerRAROld     = []byte{0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00}
)

// GetFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
//
// For zips, this will also match on files with empty zip or spanned zip magic
// numbers.  If you need to distinguich between the various zip formats, use
// something else.
func GetFormat(r io.ReaderAt) (Format, error) {
	ok, err := IsLZ4(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZ4, nil
	}
	ok, err = IsGzip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Gzip, nil
	}
	ok, err = IsZip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Zip, nil
	}
	ok, err = IsBzip2(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Bzip2, nil
	}
	ok, err = IsLZW(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZW, nil
	}
	return Unknown, errors.New("unsupported format: input format is not known")
}

// IsBzip2 checks to see if the received reader's contents are in bzip2 format
// by checking the magic numbers.
func IsBzip2(r io.ReaderAt) (bool, error) {
	h := make([]byte, 3)
	// Read the first 3 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var hb [3]byte
	// check for bzip2
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &hb)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var mb [3]byte
	mbuf := bytes.NewBuffer(headerBzip2)
	err = binary.Read(mbuf, binary.BigEndian, &mb)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if hb == mb {
		return true, nil
	}
	return false, nil
}

// IsGzip checks to see if the received reader's contents are in gzip format
// by checking the magic numbers.
func IsGzip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Read the first 2 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for gzip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var m16 uint16
	mbuf := bytes.NewBuffer(headerGzip)
	err = binary.Read(mbuf, binary.BigEndian, &m16)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if h16 == m16 {
		return true, nil
	}
	return false, nil
}

// IsLZ4 checks to see if the received reader's contents are in LZ4 foramt by
// checking the magic numbers.
func IsLZ4(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for lz4
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZ4's magic number: %s", err)
	}
	var m32 uint32
	mbuf := bytes.NewBuffer(headerLZ4)
	err = binary.Read(mbuf, binary.BigEndian, &m32)
	if err != nil {
		return false, fmt.Errorf("error while converting LZ4 magic number for comparison: %s", err)
	}
	if h32 == m32 {
		return true, nil
	}
	return false, nil
}

// IsLZW checks to see if the received reader's contents are in LZ4 format by
// checking the magic numbers.
func IsLZW(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Reat the first 8 bytes since that's where most magic numbers are
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for lzw
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZW's magic number: %s", err)
	}
	var m16 uint16
	mbuf := bytes.NewBuffer(headerLZW)
	err = binary.Read(mbuf, binary.BigEndian, &m16)
	if err != nil {
		return false, fmt.Errorf("error while converting LZW magic number for comparison: %s", err)
	}
	if h16 == m16 {
		return true, nil
	}
	return false, nil
}

// IsZip checks to see if the received reader's contents are in zip format
// by checking the magic numbers. This wil match on zip, empty zip and spanned
// zip magic numbers. If you need to distinguish between those, use something
// else.
func IsZip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched zip's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(headerZip)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(headerZipEmpty)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(headerZipSpanned)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the spanned zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}
<commit_msg>remove formats for which there are no plans to support<commit_after>package magicnum

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"
)

const (
	Unknown    Format = iota // unknown format
	Gzip                     // Gzip compression format
	Tar                      // Tar format; normally used
	Tar1                     // Tar1 header format; normalizes to Tar
	Tar2                     // Tar1 header format; normalizes to Tar
	Zip                      // Zip archive
	ZipEmpty                 // Empty Zip Archive
	ZipSpanned               // Spanned Zip Archive
	Bzip2                    // Bzip2 compression
	LZW                      // LZW compression
	LZ4                      // LZ4 compression
)

// TODO: should Format be more specific? e.g. CompressionFormat, MediaFormat, etc.
type Format int

func (f Format) String() string {
	switch f {
	case Gzip:
		return "gzip"
	case Tar, Tar1, Tar2:
		return "tar"
	case Zip:
		return "zip"
	case ZipEmpty:
		return "empty zip archive"
	case ZipSpanned:
		return "spanned zip archive"
	case Bzip2:
		return "bzip2"
	case LZW:
		return "lzw"
	case LZ4:
		return "lz4"
	}
	return "unknown"
}

// Ext returns the extension for the format. Formats may have more than one
// accepted extension; alternate extensiona are not supported.
func (f Format) Ext() string {
	switch f {
	case Gzip:
		return ".gz"
	case Tar, Tar1, Tar2:
		return ".tar"
	case Zip, ZipEmpty, ZipSpanned:
		return ".zip"
	case Bzip2:
		return ".bz2"
	case LZW:
		return ".Z"
	case LZ4:
		return "lz4"
	}
	return "unknown"
}

func FormatFromString(s string) Format {
	s = strings.ToLower(s)
	switch s {
	case "gzip", "gz":
		return Gzip
	case "tar":
		return Tar
	case "zip":
		return Zip
	case "bzip2", "bz2":
		return Bzip2
	case "lzw", "Z":
		return LZW
	case "lz4":
		return LZ4
	}
	return Unknown
}

// Magic numbers for supported formats
var (
	headerGzip       = []byte{0x1f, 0x8b}
	headerTar1       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	headerTar2       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	headerZip        = []byte{0x50, 0x4b, 0x03, 0x04}
	headerZipEmpty   = []byte{0x50, 0x4b, 0x05, 0x06}
	headerZipSpanned = []byte{0x50, 0x4b, 0x07, 0x08}
	headerBzip2      = []byte{0x42, 0x5a, 0x68}
	headerLZW        = []byte{0x1F, 0x9d}
	headerLZ4        = []byte{0x18, 0x4d, 0x22, 0x04}
)

// GetFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
//
// For zips, this will also match on files with empty zip or spanned zip magic
// numbers.  If you need to distinguich between the various zip formats, use
// something else.
func GetFormat(r io.ReaderAt) (Format, error) {
	ok, err := IsLZ4(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZ4, nil
	}
	ok, err = IsGzip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Gzip, nil
	}
	ok, err = IsZip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Zip, nil
	}
	ok, err = IsBzip2(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Bzip2, nil
	}
	ok, err = IsLZW(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZW, nil
	}
	return Unknown, errors.New("unsupported format: input format is not known")
}

// IsBzip2 checks to see if the received reader's contents are in bzip2 format
// by checking the magic numbers.
func IsBzip2(r io.ReaderAt) (bool, error) {
	h := make([]byte, 3)
	// Read the first 3 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var hb [3]byte
	// check for bzip2
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &hb)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var mb [3]byte
	mbuf := bytes.NewBuffer(headerBzip2)
	err = binary.Read(mbuf, binary.BigEndian, &mb)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if hb == mb {
		return true, nil
	}
	return false, nil
}

// IsGzip checks to see if the received reader's contents are in gzip format
// by checking the magic numbers.
func IsGzip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Read the first 2 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for gzip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var m16 uint16
	mbuf := bytes.NewBuffer(headerGzip)
	err = binary.Read(mbuf, binary.BigEndian, &m16)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if h16 == m16 {
		return true, nil
	}
	return false, nil
}

// IsLZ4 checks to see if the received reader's contents are in LZ4 foramt by
// checking the magic numbers.
func IsLZ4(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for lz4
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZ4's magic number: %s", err)
	}
	var m32 uint32
	mbuf := bytes.NewBuffer(headerLZ4)
	err = binary.Read(mbuf, binary.BigEndian, &m32)
	if err != nil {
		return false, fmt.Errorf("error while converting LZ4 magic number for comparison: %s", err)
	}
	if h32 == m32 {
		return true, nil
	}
	return false, nil
}

// IsLZW checks to see if the received reader's contents are in LZ4 format by
// checking the magic numbers.
func IsLZW(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Reat the first 8 bytes since that's where most magic numbers are
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for lzw
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZW's magic number: %s", err)
	}
	var m16 uint16
	mbuf := bytes.NewBuffer(headerLZW)
	err = binary.Read(mbuf, binary.BigEndian, &m16)
	if err != nil {
		return false, fmt.Errorf("error while converting LZW magic number for comparison: %s", err)
	}
	if h16 == m16 {
		return true, nil
	}
	return false, nil
}

// IsZip checks to see if the received reader's contents are in zip format
// by checking the magic numbers. This wil match on zip, empty zip and spanned
// zip magic numbers. If you need to distinguish between those, use something
// else.
func IsZip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched zip's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(headerZip)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(headerZipEmpty)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(headerZipSpanned)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the spanned zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/user"
	"path"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/mitnk/goutils/encrypt"
)

var VERSION = "1.2.1"
var countConnected = 0
var KEY = getKey()

func main() {
	port := flag.String("p", "12345", "port")
	flag.Usage = func() {
		fmt.Printf("Usage of lightsocks v%s:\n", VERSION)
		fmt.Printf("lightsocks [flags]\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	remote, err := net.Listen("tcp", ":"+*port)
	if err != nil {
		fmt.Printf("net listen: %v\n", err)
		os.Exit(2)
	}
	defer remote.Close()
	info("lightsocks v%s", VERSION)
	info("listen on port %s", *port)

	for {
		local, err := remote.Accept()
		if err != nil {
			info("error when accept: %v", err)
			continue
		}
		go handleClient(local)
	}
}

func handleClient(local net.Conn) {
	defer local.Close()
	countConnected += 1
	defer func() {
		countConnected -= 1
	}()

	info("local connected: %v", local.RemoteAddr())
	buffer := make([]byte, 1)
	_, err := io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read first byte from local")
		return
	}
	buffer = make([]byte, buffer[0])
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read validation data from local")
		return
	}
	dataCheck, err := encrypt.Decrypt(buffer, KEY)
	if err != nil || !reflect.DeepEqual(KEY[8:16], dataCheck) {
		info("invalid local")
		return
	}

	buffer = make([]byte, 1)
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read size from local")
		return
	}

	buffer = make([]byte, buffer[0])
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read host from local")
		return
	}
	host, err := encrypt.Decrypt(buffer, KEY)
	if err != nil {
		info("ERROR: cannot decrypt host")
		return
	}

	buffer = make([]byte, 2)
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read port from local")
		return
	}
	port := binary.BigEndian.Uint16(buffer)

	url := net.JoinHostPort(string(host), strconv.Itoa(int(port)))
	server, err := net.Dial("tcp", url)
	if err != nil {
		info("ERROR: cannot dial to server %s", url)
		return
	}
	info("connected to server: %s", url)
	defer server.Close()

	ch_local := make(chan []byte)
	ch_server := make(chan DataInfo)
	go readDataFromLocal(ch_local, ch_server, local)
	go readDataFromServer(ch_server, server)

	shouldStop := false
	for {
		if shouldStop {
			break
		}

		select {
		case data := <-ch_local:
			if data == nil {
				local.Close()
				info("local closed %v", local.RemoteAddr())
				shouldStop = true
				break
			}
			server.Write(data)
		case di := <-ch_server:
			if di.data == nil {
				server.Close()
				info("server closed %v", server.RemoteAddr())
				break
			}
			buffer = encrypt.Encrypt(di.data[:di.size], KEY)
			b := make([]byte, 2)
			binary.BigEndian.PutUint16(b, uint16(len(buffer)))
			local.Write(b)
			local.Write(buffer)
		}
	}
}

func readDataFromServer(ch chan DataInfo, conn net.Conn) {
	for {
		data := make([]byte, 7000+rand.Intn(2000))
		n, err := conn.Read(data)
		if err != nil {
			ch <- DataInfo{nil, 0}
			return
		}
		ch <- DataInfo{data, n}
	}
}

func readDataFromLocal(ch chan []byte, ch2 chan DataInfo, conn net.Conn) {
	for {
		buffer := make([]byte, 2)
		_, err := io.ReadFull(conn, buffer)
		if err != nil {
			ch <- nil
			ch2 <- DataInfo{nil, 0}
			return
		}
		size := binary.BigEndian.Uint16(buffer)
		buffer = make([]byte, size)
		_, err = io.ReadFull(conn, buffer)
		if err != nil {
			ch <- nil
			return
		}
		data, err := encrypt.Decrypt(buffer, KEY)
		if err != nil {
			fmt.Printf("ERROR: cannot decrypt data from local.")
			ch <- nil
			return
		}
		ch <- data
	}
}

func getKey() []byte {
	usr, err := user.Current()
	if err != nil {
		fmt.Printf("user current: %v\n", err)
		os.Exit(2)
	}
	fileKey := path.Join(usr.HomeDir, ".lightsockskey")
	data, err := ioutil.ReadFile(fileKey)
	if err != nil {
		fmt.Printf("Failed to load key file: %v\n", err)
		os.Exit(1)
	}
	s := strings.TrimSpace(string(data))
	sum := sha256.Sum256([]byte(s))
	return sum[:]
}

func info(format string, a ...interface{}) {
	ts := time.Now().Format("2006-01-02 15:04:05")
	prefix := fmt.Sprintf("[%s][%d] ", ts, countConnected)
	fmt.Printf(prefix+format+"\n", a...)
}

type DataInfo struct {
	data []byte
	size int
}
<commit_msg>added timeout<commit_after>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"os/user"
	"path"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/mitnk/goutils/encrypt"
)

var VERSION = "1.3.0"
var countConnected = 0
var KEY = getKey()

func main() {
	port := flag.String("p", "12345", "port")
	flag.Usage = func() {
		fmt.Printf("Usage of lightsocks v%s:\n", VERSION)
		fmt.Printf("lightsocks [flags]\n")
		flag.PrintDefaults()
		os.Exit(0)
	}
	flag.Parse()
	remote, err := net.Listen("tcp", ":"+*port)
	if err != nil {
		fmt.Printf("net listen: %v\n", err)
		os.Exit(2)
	}
	defer remote.Close()
	info("lightsocks v%s", VERSION)
	info("listen on port %s", *port)

	for {
		local, err := remote.Accept()
		if err != nil {
			info("error when accept: %v", err)
			continue
		}
		go handleClient(local)
	}
}

func handleClient(local net.Conn) {
	defer local.Close()
	countConnected += 1
	defer func() {
		countConnected -= 1
	}()

	info("local connected: %v", local.RemoteAddr())
	buffer := make([]byte, 1)
	_, err := io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read first byte from local")
		return
	}
	buffer = make([]byte, buffer[0])
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read validation data from local")
		return
	}
	dataCheck, err := encrypt.Decrypt(buffer, KEY)
	if err != nil || !reflect.DeepEqual(KEY[8:16], dataCheck) {
		info("invalid local")
		return
	}

	buffer = make([]byte, 1)
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read size from local")
		return
	}

	buffer = make([]byte, buffer[0])
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read host from local")
		return
	}
	host, err := encrypt.Decrypt(buffer, KEY)
	if err != nil {
		info("ERROR: cannot decrypt host")
		return
	}

	buffer = make([]byte, 2)
	_, err = io.ReadFull(local, buffer)
	if err != nil {
		info("cannot read port from local")
		return
	}
	port := binary.BigEndian.Uint16(buffer)

	url := net.JoinHostPort(string(host), strconv.Itoa(int(port)))
	server, err := net.Dial("tcp", url)
	if err != nil {
		info("ERROR: cannot dial to server %s", url)
		return
	}
	info("connected to server: %s", url)
	defer server.Close()

	ch_local := make(chan []byte)
	ch_server := make(chan DataInfo)
	go readDataFromLocal(ch_local, ch_server, local)
	go readDataFromServer(ch_server, server)

	shouldStop := false
	for {
		if shouldStop {
			break
		}

		select {
		case data := <-ch_local:
			if data == nil {
				local.Close()
				info("local closed %v", local.RemoteAddr())
				shouldStop = true
				break
			}
			server.Write(data)
		case di := <-ch_server:
			if di.data == nil {
				server.Close()
				info("server closed %v", server.RemoteAddr())
				break
			}
			buffer = encrypt.Encrypt(di.data[:di.size], KEY)
			b := make([]byte, 2)
			binary.BigEndian.PutUint16(b, uint16(len(buffer)))
			local.Write(b)
			local.Write(buffer)
		case <-time.After(120 * time.Second):
			info("timeout on %s", url)
			return
		}
	}
}

func readDataFromServer(ch chan DataInfo, conn net.Conn) {
	for {
		data := make([]byte, 7000+rand.Intn(2000))
		n, err := conn.Read(data)
		if err != nil {
			ch <- DataInfo{nil, 0}
			return
		}
		ch <- DataInfo{data, n}
	}
}

func readDataFromLocal(ch chan []byte, ch2 chan DataInfo, conn net.Conn) {
	for {
		buffer := make([]byte, 2)
		_, err := io.ReadFull(conn, buffer)
		if err != nil {
			ch <- nil
			ch2 <- DataInfo{nil, 0}
			return
		}
		size := binary.BigEndian.Uint16(buffer)
		buffer = make([]byte, size)
		_, err = io.ReadFull(conn, buffer)
		if err != nil {
			ch <- nil
			return
		}
		data, err := encrypt.Decrypt(buffer, KEY)
		if err != nil {
			fmt.Printf("ERROR: cannot decrypt data from local.")
			ch <- nil
			return
		}
		ch <- data
	}
}

func getKey() []byte {
	usr, err := user.Current()
	if err != nil {
		fmt.Printf("user current: %v\n", err)
		os.Exit(2)
	}
	fileKey := path.Join(usr.HomeDir, ".lightsockskey")
	data, err := ioutil.ReadFile(fileKey)
	if err != nil {
		fmt.Printf("Failed to load key file: %v\n", err)
		os.Exit(1)
	}
	s := strings.TrimSpace(string(data))
	sum := sha256.Sum256([]byte(s))
	return sum[:]
}

func info(format string, a ...interface{}) {
	ts := time.Now().Format("2006-01-02 15:04:05")
	prefix := fmt.Sprintf("[%s][%d] ", ts, countConnected)
	fmt.Printf(prefix+format+"\n", a...)
}

type DataInfo struct {
	data []byte
	size int
}
<|endoftext|>


<commit_before>package mal

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
)

var (
	// mux is the HTTP request multiplexer used with the test server.
	mux *http.ServeMux

	// client is the MyAnimeList client being tested.
	client *Client

	// server is a test HTTP server used to provide mock API responses.
	server *httptest.Server
)

// setup sets up a test HTTP server along with a mal.Client that is
// configured to talk to that test server. Tests should register handlers on
// mux which provide mock responses for the API method being tested.
func setup() {
	// test server
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	// mal client configured to use test server
	client = NewClient()
	client.BaseURL, _ = url.Parse(server.URL)
}

// teardown closes the test HTTP server.
func teardown() {
	server.Close()
}

type urlValues map[string]string

func testURLValues(t *testing.T, r *http.Request, values urlValues) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}
	actual := r.URL.Query()
	if !reflect.DeepEqual(want, actual) {
		t.Errorf("URL Values = %v, want %v", actual, want)
	}
}

func testBasicAuth(t *testing.T, r *http.Request, usedWant bool, unameWant, passWant string) {
	uname, pass, used := r.BasicAuth()
	if used != usedWant {
		t.Errorf("BasicAuth used = %v, want %v", used, usedWant)
	}
	if uname != unameWant {
		t.Errorf("BasicAuth username = %v, want %v", uname, unameWant)
	}
	if pass != passWant {
		t.Errorf("BasicAuth password = %v, want %v", pass, passWant)
	}
}

func testUserAgent(t *testing.T, r *http.Request, want string) {
	agent := r.Header.Get("User-Agent")
	if want != agent {
		t.Errorf("User-Agent = %v, want %v", agent, want)
	}
}

func testMethod(t *testing.T, r *http.Request, want string) {
	if want != r.Method {
		t.Errorf("Request method = %v, want %v", r.Method, want)
	}
}

func testBody(t *testing.T, r *http.Request, want string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read body")
	}
	body := string(b)
	if body != want {
		t.Errorf("body = %v, want %v", body, want)
	}
}

func testContentType(t *testing.T, r *http.Request, want string) {
	ct := r.Header.Get("Content-Type")
	if ct != want {
		t.Errorf("Content-Type = %v, want %v", ct, want)
	}
}

func testID(t *testing.T, r *http.Request, want string) {
	idXML := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
	xml := idXML[len(idXML)-4:]
	if xml != ".xml" {
		t.Errorf("URL path %v does not end in .xml", r.URL.Path)
	}
	id := idXML[:len(idXML)-4]
	if id != want {
		t.Errorf("provided id = %v, want %v", id, want)
	}
}

func testFormValue(t *testing.T, r *http.Request, value, want string) {
	v := r.FormValue(value)
	if v != want {
		t.Errorf("form value %v = %v, want %v", value, v, want)
	}
}

func TestClient_NewClient(t *testing.T) {
	c := NewClient()

	if got, want := c.BaseURL.String(), defaultBaseURL; got != want {
		t.Errorf("NewClient.BaseURL = %v, want %v", got, want)
	}

	if got, want := c.UserAgent, defaultUserAgent; got != want {
		t.Errorf("NewClient.UserAgent = %v, want %v", got, want)
	}
}

func TestClient_NewRequest(t *testing.T) {
	c := NewClient()

	inURL, outURL := "/foo", defaultBaseURL+"foo"

	inData := &User{ID: 5, Username: "TestUser"}
	v := url.Values{}
	v.Set("data", "<user><id>5</id><username>TestUser</username></user>")
	urlEncData, _ := url.Parse(v.Encode())
	outData := urlEncData.Path

	req, _ := c.NewRequest("GET", inURL, inData)

	// test that the endpoint URL was correctly added to the base URL
	if got, want := req.URL.String(), outURL; got != want {
		t.Errorf("NewRequest(%q) URL =  %v, want %v", inURL, got, want)
	}

	// test that body was encoded to XML and then URL enconded as data=...
	body, _ := ioutil.ReadAll(req.Body)
	urlEncBody, _ := url.Parse(string(body)) // url.Path holds the URL decoded string
	if got, want := urlEncBody.Path, outData; got != want {
		t.Errorf("NewRequest(%+v) Body = '%v', want '%v'", inData, got, want)
	}

	testBasicAuth(t, req, false, "", "")
	testUserAgent(t, req, defaultUserAgent)

}

func TestClient_Do(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		Bar string `xml:"bar"`
	}

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if want := "GET"; r.Method != want {
			t.Errorf("request method = %v, want %v", r.Method, want)
		}
		fmt.Fprint(w, `<foo><bar>• foobar</bar></foo>`)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	body := new(foo)
	response, _ := client.Do(req, body)

	want := &foo{"• foobar"}
	if !reflect.DeepEqual(body, want) {
		t.Errorf("Do() response body = %v, want %v", body, want)
	}

	if want, got := "<foo><bar>• foobar</bar></foo>", string(response.Body); want != got {
		t.Errorf("Do() mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_Do_XML_invalid_entity(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		Bar string `xml:"bar"`
	}

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if want := "GET"; r.Method != want {
			t.Errorf("request method = %v, want %v", r.Method, want)
		}
		fmt.Fprint(w, `<foo><bar>&foo; bar</bar></foo>`)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	body := new(foo)
	response, err := client.Do(req, body)

	if err == nil {
		t.Errorf("Do() receiving XML with invalid entity should return err")
	}

	if response == nil {
		t.Errorf("Do() receiving XML with invalid entity should also return response")
	}

	if want, got := "<foo><bar>&foo; bar</bar></foo>", string(response.Body); want != got {
		t.Errorf("Do() receiving XML with invalid entity mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_Do_404(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "not found", http.StatusNotFound)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	response, err := client.Do(req, nil)

	if err == nil {
		t.Errorf("expected 404 err")
	}

	// http.Error seems to be adding a new line to the message.
	if want, got := "not found\n", string(response.Body); want != got {
		t.Errorf("Do() mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_post_invalid_ID(t *testing.T) {
	setup()
	defer teardown()

	client.SetCredentials("TestUser", "TestPass")
	client.SetUserAgent("TestAgent")

	mux.HandleFunc("/api/animelist/update/", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
		testID(t, r, "0")
		testBasicAuth(t, r, true, "TestUser", "TestPass")
		testUserAgent(t, r, "TestAgent")
		testContentType(t, r, "application/x-www-form-urlencoded")
		testFormValue(t, r, "data", "<entry><status>onhold</status></entry>")
		http.Error(w, "Invalid ID", http.StatusNotImplemented)
	})

	response, err := client.post("api/animelist/update/", 0, AnimeEntry{Status: "onhold"})

	if err == nil {
		t.Errorf("Anime.Update invalid ID should return err")
	}

	if response == nil {
		t.Errorf("Anime.Update invalid ID should return also return response")
	}
}

func TestClient_delete_invalid_ID(t *testing.T) {
	setup()
	defer teardown()

	client.SetCredentials("TestUser", "TestPass")
	client.SetUserAgent("TestAgent")

	mux.HandleFunc("/api/animelist/delete/", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
		testID(t, r, "0")
		testBasicAuth(t, r, true, "TestUser", "TestPass")
		testUserAgent(t, r, "TestAgent")
		http.Error(w, "Invalid ID", http.StatusNotImplemented)
	})

	response, err := client.delete("api/animelist/delete/", 0)

	if err == nil {
		t.Errorf("Anime.Delete invalid ID should return err")
	}

	if response == nil {
		t.Errorf("Anime.Delete invalid ID should return also return response")
	}
}

func TestClient_NewRequest_bad_endpoint(t *testing.T) {
	c := NewClient()
	inURL := "%foo"
	_, err := c.NewRequest("GET", inURL, nil)
	if err == nil {
		t.Errorf("NewRequest(%q) should return parse err", inURL)
	}
}

func TestClient_NewRequest_xml_encode_err(t *testing.T) {
	c := NewClient()
	in := func() {} // xml.Marshal cannot encode a func
	_, err := c.NewRequest("GET", "/foo", in)
	if err == nil {
		t.Errorf("NewRequest receiving a function as body should return XML encode err")
	}
}
<commit_msg>add tests for default endpoints<commit_after>package mal

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"reflect"
	"strings"
	"testing"
)

var (
	// mux is the HTTP request multiplexer used with the test server.
	mux *http.ServeMux

	// client is the MyAnimeList client being tested.
	client *Client

	// server is a test HTTP server used to provide mock API responses.
	server *httptest.Server
)

// setup sets up a test HTTP server along with a mal.Client that is
// configured to talk to that test server. Tests should register handlers on
// mux which provide mock responses for the API method being tested.
func setup() {
	// test server
	mux = http.NewServeMux()
	server = httptest.NewServer(mux)

	// mal client configured to use test server
	client = NewClient()
	client.BaseURL, _ = url.Parse(server.URL)
}

// teardown closes the test HTTP server.
func teardown() {
	server.Close()
}

type urlValues map[string]string

func testURLValues(t *testing.T, r *http.Request, values urlValues) {
	want := url.Values{}
	for k, v := range values {
		want.Add(k, v)
	}
	actual := r.URL.Query()
	if !reflect.DeepEqual(want, actual) {
		t.Errorf("URL Values = %v, want %v", actual, want)
	}
}

func testBasicAuth(t *testing.T, r *http.Request, usedWant bool, unameWant, passWant string) {
	uname, pass, used := r.BasicAuth()
	if used != usedWant {
		t.Errorf("BasicAuth used = %v, want %v", used, usedWant)
	}
	if uname != unameWant {
		t.Errorf("BasicAuth username = %v, want %v", uname, unameWant)
	}
	if pass != passWant {
		t.Errorf("BasicAuth password = %v, want %v", pass, passWant)
	}
}

func testUserAgent(t *testing.T, r *http.Request, want string) {
	agent := r.Header.Get("User-Agent")
	if want != agent {
		t.Errorf("User-Agent = %v, want %v", agent, want)
	}
}

func testMethod(t *testing.T, r *http.Request, want string) {
	if want != r.Method {
		t.Errorf("Request method = %v, want %v", r.Method, want)
	}
}

func testBody(t *testing.T, r *http.Request, want string) {
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		t.Errorf("Unable to read body")
	}
	body := string(b)
	if body != want {
		t.Errorf("body = %v, want %v", body, want)
	}
}

func testContentType(t *testing.T, r *http.Request, want string) {
	ct := r.Header.Get("Content-Type")
	if ct != want {
		t.Errorf("Content-Type = %v, want %v", ct, want)
	}
}

func testID(t *testing.T, r *http.Request, want string) {
	idXML := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
	xml := idXML[len(idXML)-4:]
	if xml != ".xml" {
		t.Errorf("URL path %v does not end in .xml", r.URL.Path)
	}
	id := idXML[:len(idXML)-4]
	if id != want {
		t.Errorf("provided id = %v, want %v", id, want)
	}
}

func testFormValue(t *testing.T, r *http.Request, value, want string) {
	v := r.FormValue(value)
	if v != want {
		t.Errorf("form value %v = %v, want %v", value, v, want)
	}
}

func TestClient_NewClient(t *testing.T) {
	c := NewClient()

	// test default base URL
	if got, want := c.BaseURL.String(), defaultBaseURL; got != want {
		t.Errorf("NewClient.BaseURL = %v, want %v", got, want)
	}

	// test default user agent
	if got, want := c.UserAgent, defaultUserAgent; got != want {
		t.Errorf("NewClient.UserAgent = %v, want %v", got, want)
	}

	// test account default endpoint
	if got, want := c.Account.Endpoint.String(), defaultAccountEndpoint; got != want {
		t.Errorf("NewClient.Account.Endpoint = %v, want %v", got, want)
	}

	// test anime default endpoints
	if got, want := c.Anime.AddEndpoint.String(), defaultAnimeAddEndpoint; got != want {
		t.Errorf("NewClient.Anime.Addpoint = %v, want %v", got, want)
	}
	if got, want := c.Anime.UpdateEndpoint.String(), defaultAnimeUpdateEndpoint; got != want {
		t.Errorf("NewClient.Anime.UpdateEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Anime.DeleteEndpoint.String(), defaultAnimeDeleteEndpoint; got != want {
		t.Errorf("NewClient.Anime.DeleteEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Anime.SearchEndpoint.String(), defaultAnimeSearchEndpoint; got != want {
		t.Errorf("NewClient.Anime.SearchEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Anime.ListEndpoint.String(), defaultListEndpoint; got != want {
		t.Errorf("NewClient.Anime.ListEndpoint = %v, want %v", got, want)
	}

	// test manga default endpoints
	if got, want := c.Manga.AddEndpoint.String(), defaultMangaAddEndpoint; got != want {
		t.Errorf("NewClient.Manga.Addpoint = %v, want %v", got, want)
	}
	if got, want := c.Manga.UpdateEndpoint.String(), defaultMangaUpdateEndpoint; got != want {
		t.Errorf("NewClient.Manga.UpdateEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Manga.DeleteEndpoint.String(), defaultMangaDeleteEndpoint; got != want {
		t.Errorf("NewClient.Manga.DeleteEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Manga.SearchEndpoint.String(), defaultMangaSearchEndpoint; got != want {
		t.Errorf("NewClient.Manga.SearchEndpoint = %v, want %v", got, want)
	}
	if got, want := c.Manga.ListEndpoint.String(), defaultListEndpoint; got != want {
		t.Errorf("NewClient.Manga.ListEndpoint = %v, want %v", got, want)
	}
}

func TestClient_NewRequest(t *testing.T) {
	c := NewClient()

	inURL, outURL := "/foo", defaultBaseURL+"foo"

	inData := &User{ID: 5, Username: "TestUser"}
	v := url.Values{}
	v.Set("data", "<user><id>5</id><username>TestUser</username></user>")
	urlEncData, _ := url.Parse(v.Encode())
	outData := urlEncData.Path

	req, _ := c.NewRequest("GET", inURL, inData)

	// test that the endpoint URL was correctly added to the base URL
	if got, want := req.URL.String(), outURL; got != want {
		t.Errorf("NewRequest(%q) URL =  %v, want %v", inURL, got, want)
	}

	// test that body was encoded to XML and then URL enconded as data=...
	body, _ := ioutil.ReadAll(req.Body)
	urlEncBody, _ := url.Parse(string(body)) // url.Path holds the URL decoded string
	if got, want := urlEncBody.Path, outData; got != want {
		t.Errorf("NewRequest(%+v) Body = '%v', want '%v'", inData, got, want)
	}

	testBasicAuth(t, req, false, "", "")
	testUserAgent(t, req, defaultUserAgent)

}

func TestClient_Do(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		Bar string `xml:"bar"`
	}

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if want := "GET"; r.Method != want {
			t.Errorf("request method = %v, want %v", r.Method, want)
		}
		fmt.Fprint(w, `<foo><bar>• foobar</bar></foo>`)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	body := new(foo)
	response, _ := client.Do(req, body)

	want := &foo{"• foobar"}
	if !reflect.DeepEqual(body, want) {
		t.Errorf("Do() response body = %v, want %v", body, want)
	}

	if want, got := "<foo><bar>• foobar</bar></foo>", string(response.Body); want != got {
		t.Errorf("Do() mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_Do_XML_invalid_entity(t *testing.T) {
	setup()
	defer teardown()

	type foo struct {
		Bar string `xml:"bar"`
	}

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if want := "GET"; r.Method != want {
			t.Errorf("request method = %v, want %v", r.Method, want)
		}
		fmt.Fprint(w, `<foo><bar>&foo; bar</bar></foo>`)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	body := new(foo)
	response, err := client.Do(req, body)

	if err == nil {
		t.Errorf("Do() receiving XML with invalid entity should return err")
	}

	if response == nil {
		t.Errorf("Do() receiving XML with invalid entity should also return response")
	}

	if want, got := "<foo><bar>&foo; bar</bar></foo>", string(response.Body); want != got {
		t.Errorf("Do() receiving XML with invalid entity mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_Do_404(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "not found", http.StatusNotFound)
	})

	req, _ := client.NewRequest("GET", "/", nil)

	response, err := client.Do(req, nil)

	if err == nil {
		t.Errorf("expected 404 err")
	}

	// http.Error seems to be adding a new line to the message.
	if want, got := "not found\n", string(response.Body); want != got {
		t.Errorf("Do() mal.Response.Body = %v, want %v", got, want)
	}
}

func TestClient_post_invalid_ID(t *testing.T) {
	setup()
	defer teardown()

	client.SetCredentials("TestUser", "TestPass")
	client.SetUserAgent("TestAgent")

	mux.HandleFunc("/api/animelist/update/", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "POST")
		testID(t, r, "0")
		testBasicAuth(t, r, true, "TestUser", "TestPass")
		testUserAgent(t, r, "TestAgent")
		testContentType(t, r, "application/x-www-form-urlencoded")
		testFormValue(t, r, "data", "<entry><status>onhold</status></entry>")
		http.Error(w, "Invalid ID", http.StatusNotImplemented)
	})

	response, err := client.post("api/animelist/update/", 0, AnimeEntry{Status: "onhold"})

	if err == nil {
		t.Errorf("Anime.Update invalid ID should return err")
	}

	if response == nil {
		t.Errorf("Anime.Update invalid ID should return also return response")
	}
}

func TestClient_delete_invalid_ID(t *testing.T) {
	setup()
	defer teardown()

	client.SetCredentials("TestUser", "TestPass")
	client.SetUserAgent("TestAgent")

	mux.HandleFunc("/api/animelist/delete/", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
		testID(t, r, "0")
		testBasicAuth(t, r, true, "TestUser", "TestPass")
		testUserAgent(t, r, "TestAgent")
		http.Error(w, "Invalid ID", http.StatusNotImplemented)
	})

	response, err := client.delete("api/animelist/delete/", 0)

	if err == nil {
		t.Errorf("Anime.Delete invalid ID should return err")
	}

	if response == nil {
		t.Errorf("Anime.Delete invalid ID should return also return response")
	}
}

func TestClient_NewRequest_bad_endpoint(t *testing.T) {
	c := NewClient()
	inURL := "%foo"
	_, err := c.NewRequest("GET", inURL, nil)
	if err == nil {
		t.Errorf("NewRequest(%q) should return parse err", inURL)
	}
}

func TestClient_NewRequest_xml_encode_err(t *testing.T) {
	c := NewClient()
	in := func() {} // xml.Marshal cannot encode a func
	_, err := c.NewRequest("GET", "/foo", in)
	if err == nil {
		t.Errorf("NewRequest receiving a function as body should return XML encode err")
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"go/types"
	"log"
	"sort"

	"github.com/kisielk/gotool"
	"golang.org/x/tools/go/loader"
)

var fset = token.NewFileSet()

func main() {
	flag.Parse()

	importPaths := gotool.ImportPaths(flag.Args())
	if len(importPaths) == 0 {
		return
	}

	var conf loader.Config
	conf.Fset = fset
	for _, importPath := range importPaths {
		conf.Import(importPath)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	var v visitor
	for _, pkg := range prog.InitialPackages() {
		v.pkg = pkg
		for _, file := range pkg.Files {
			ast.Walk(&v, file)
		}
	}
}

type visitor struct {
	pkg *loader.PackageInfo
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if str, ok := node.(*ast.StructType); ok {
		malign(node.Pos(), v.pkg.Types[str].Type.(*types.Struct))
	}
	return v
}

func malign(pos token.Pos, str *types.Struct) {
	wordSize := int64(8)
	maxAlign := int64(8)
	switch build.Default.GOARCH {
	case "386", "arm":
		wordSize, maxAlign = 4, 4
	case "amd64p32":
		wordSize = 4
	}

	s := gcSizes{wordSize, maxAlign}
	sz, opt := s.Sizeof(str), optimalSize(str, &s)
	if sz != opt {
		fmt.Printf("%s: struct of size %d could be %d\n", fset.Position(pos), sz, opt)
	}
}

func optimalSize(str *types.Struct, sizes *gcSizes) int64 {
	nf := str.NumFields()
	fields := make([]*types.Var, nf)
	alignofs := make([]int64, nf)
	sizeofs := make([]int64, nf)
	for i := 0; i < nf; i++ {
		fields[i] = str.Field(i)
		ft := fields[i].Type()
		alignofs[i] = sizes.Alignof(ft)
		sizeofs[i] = sizes.Sizeof(ft)
	}
	sort.Sort(&byAlignAndSize{fields, alignofs, sizeofs})
	return sizes.Sizeof(types.NewStruct(fields, nil))
}

type byAlignAndSize struct {
	fields   []*types.Var
	alignofs []int64
	sizeofs  []int64
}

func (s *byAlignAndSize) Len() int { return len(s.fields) }
func (s *byAlignAndSize) Swap(i, j int) {
	s.fields[i], s.fields[j] = s.fields[j], s.fields[i]
	s.alignofs[i], s.alignofs[j] = s.alignofs[j], s.alignofs[i]
	s.sizeofs[i], s.sizeofs[j] = s.sizeofs[j], s.sizeofs[i]
}

func (s *byAlignAndSize) Less(i, j int) bool {
	// Place zero sized objects before non-zero sized objects.
	if s.sizeofs[i] == 0 && s.sizeofs[j] != 0 {
		return true
	}
	if s.sizeofs[j] == 0 && s.sizeofs[i] != 0 {
		return false
	}

	// Next, place more tightly aligned objects before less tightly aligned objects.
	if s.alignofs[i] != s.alignofs[j] {
		return s.alignofs[i] > s.alignofs[j]
	}

	// Lastly, order by size.
	if s.sizeofs[i] != s.sizeofs[j] {
		return s.sizeofs[i] > s.sizeofs[j]
	}

	return false
}

// Code below based on go/types.StdSizes.

type gcSizes struct {
	WordSize int64
	MaxAlign int64
}

func (s *gcSizes) Alignof(T types.Type) int64 {
	// NOTE: On amd64, complex64 is 8 byte aligned,
	// even though float32 is only 4 byte aligned.

	// For arrays and structs, alignment is defined in terms
	// of alignment of the elements and fields, respectively.
	switch t := T.Underlying().(type) {
	case *types.Array:
		// spec: "For a variable x of array type: unsafe.Alignof(x)
		// is the same as unsafe.Alignof(x[0]), but at least 1."
		return s.Alignof(t.Elem())
	case *types.Struct:
		// spec: "For a variable x of struct type: unsafe.Alignof(x)
		// is the largest of the values unsafe.Alignof(x.f) for each
		// field f of x, but at least 1."
		max := int64(1)
		for i, nf := 0, t.NumFields(); i < nf; i++ {
			if a := s.Alignof(t.Field(i).Type()); a > max {
				max = a
			}
		}
		return max
	}
	a := s.Sizeof(T) // may be 0
	// spec: "For a variable x of any type: unsafe.Alignof(x) is at least 1."
	if a < 1 {
		return 1
	}
	if a > s.MaxAlign {
		return s.MaxAlign
	}
	return a
}

var basicSizes = [...]byte{
	types.Bool:       1,
	types.Int8:       1,
	types.Int16:      2,
	types.Int32:      4,
	types.Int64:      8,
	types.Uint8:      1,
	types.Uint16:     2,
	types.Uint32:     4,
	types.Uint64:     8,
	types.Float32:    4,
	types.Float64:    8,
	types.Complex64:  8,
	types.Complex128: 16,
}

func (s *gcSizes) Sizeof(T types.Type) int64 {
	switch t := T.Underlying().(type) {
	case *types.Basic:
		k := t.Kind()
		if int(k) < len(basicSizes) {
			if s := basicSizes[k]; s > 0 {
				return int64(s)
			}
		}
		if k == types.String {
			return s.WordSize * 2
		}
	case *types.Array:
		n := t.Len()
		if n == 0 {
			return 0
		}
		a := s.Alignof(t.Elem())
		z := s.Sizeof(t.Elem())
		return align(z, a)*(n-1) + z
	case *types.Slice:
		return s.WordSize * 3
	case *types.Struct:
		nf := t.NumFields()
		if nf == 0 {
			return 0
		}

		var o int64
		max := int64(1)
		for i := 0; i < nf; i++ {
			ft := t.Field(i).Type()
			a, sz := s.Alignof(ft), s.Sizeof(ft)
			if a > max {
				max = a
			}
			if i == nf-1 && sz == 0 && o != 0 {
				sz = 1
			}
			o = align(o, a) + sz
		}
		return align(o, max)
	case *types.Interface:
		return s.WordSize * 2
	}
	return s.WordSize // catch-all
}

// align returns the smallest y >= x such that y % a == 0.
func align(x, a int64) int64 {
	y := x + a - 1
	return y - y%a
}
<commit_msg>maligned: use ast.Inspect instead of ast.Walk<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"go/types"
	"log"
	"sort"

	"github.com/kisielk/gotool"
	"golang.org/x/tools/go/loader"
)

var fset = token.NewFileSet()

func main() {
	flag.Parse()

	importPaths := gotool.ImportPaths(flag.Args())
	if len(importPaths) == 0 {
		return
	}

	var conf loader.Config
	conf.Fset = fset
	for _, importPath := range importPaths {
		conf.Import(importPath)
	}
	prog, err := conf.Load()
	if err != nil {
		log.Fatal(err)
	}

	for _, pkg := range prog.InitialPackages() {
		for _, file := range pkg.Files {
			ast.Inspect(file, func(node ast.Node) bool {
				if s, ok := node.(*ast.StructType); ok {
					malign(node.Pos(), pkg.Types[s].Type.(*types.Struct))
				}
				return true
			})
		}
	}
}

func malign(pos token.Pos, str *types.Struct) {
	wordSize := int64(8)
	maxAlign := int64(8)
	switch build.Default.GOARCH {
	case "386", "arm":
		wordSize, maxAlign = 4, 4
	case "amd64p32":
		wordSize = 4
	}

	s := gcSizes{wordSize, maxAlign}
	sz, opt := s.Sizeof(str), optimalSize(str, &s)
	if sz != opt {
		fmt.Printf("%s: struct of size %d could be %d\n", fset.Position(pos), sz, opt)
	}
}

func optimalSize(str *types.Struct, sizes *gcSizes) int64 {
	nf := str.NumFields()
	fields := make([]*types.Var, nf)
	alignofs := make([]int64, nf)
	sizeofs := make([]int64, nf)
	for i := 0; i < nf; i++ {
		fields[i] = str.Field(i)
		ft := fields[i].Type()
		alignofs[i] = sizes.Alignof(ft)
		sizeofs[i] = sizes.Sizeof(ft)
	}
	sort.Sort(&byAlignAndSize{fields, alignofs, sizeofs})
	return sizes.Sizeof(types.NewStruct(fields, nil))
}

type byAlignAndSize struct {
	fields   []*types.Var
	alignofs []int64
	sizeofs  []int64
}

func (s *byAlignAndSize) Len() int { return len(s.fields) }
func (s *byAlignAndSize) Swap(i, j int) {
	s.fields[i], s.fields[j] = s.fields[j], s.fields[i]
	s.alignofs[i], s.alignofs[j] = s.alignofs[j], s.alignofs[i]
	s.sizeofs[i], s.sizeofs[j] = s.sizeofs[j], s.sizeofs[i]
}

func (s *byAlignAndSize) Less(i, j int) bool {
	// Place zero sized objects before non-zero sized objects.
	if s.sizeofs[i] == 0 && s.sizeofs[j] != 0 {
		return true
	}
	if s.sizeofs[j] == 0 && s.sizeofs[i] != 0 {
		return false
	}

	// Next, place more tightly aligned objects before less tightly aligned objects.
	if s.alignofs[i] != s.alignofs[j] {
		return s.alignofs[i] > s.alignofs[j]
	}

	// Lastly, order by size.
	if s.sizeofs[i] != s.sizeofs[j] {
		return s.sizeofs[i] > s.sizeofs[j]
	}

	return false
}

// Code below based on go/types.StdSizes.

type gcSizes struct {
	WordSize int64
	MaxAlign int64
}

func (s *gcSizes) Alignof(T types.Type) int64 {
	// NOTE: On amd64, complex64 is 8 byte aligned,
	// even though float32 is only 4 byte aligned.

	// For arrays and structs, alignment is defined in terms
	// of alignment of the elements and fields, respectively.
	switch t := T.Underlying().(type) {
	case *types.Array:
		// spec: "For a variable x of array type: unsafe.Alignof(x)
		// is the same as unsafe.Alignof(x[0]), but at least 1."
		return s.Alignof(t.Elem())
	case *types.Struct:
		// spec: "For a variable x of struct type: unsafe.Alignof(x)
		// is the largest of the values unsafe.Alignof(x.f) for each
		// field f of x, but at least 1."
		max := int64(1)
		for i, nf := 0, t.NumFields(); i < nf; i++ {
			if a := s.Alignof(t.Field(i).Type()); a > max {
				max = a
			}
		}
		return max
	}
	a := s.Sizeof(T) // may be 0
	// spec: "For a variable x of any type: unsafe.Alignof(x) is at least 1."
	if a < 1 {
		return 1
	}
	if a > s.MaxAlign {
		return s.MaxAlign
	}
	return a
}

var basicSizes = [...]byte{
	types.Bool:       1,
	types.Int8:       1,
	types.Int16:      2,
	types.Int32:      4,
	types.Int64:      8,
	types.Uint8:      1,
	types.Uint16:     2,
	types.Uint32:     4,
	types.Uint64:     8,
	types.Float32:    4,
	types.Float64:    8,
	types.Complex64:  8,
	types.Complex128: 16,
}

func (s *gcSizes) Sizeof(T types.Type) int64 {
	switch t := T.Underlying().(type) {
	case *types.Basic:
		k := t.Kind()
		if int(k) < len(basicSizes) {
			if s := basicSizes[k]; s > 0 {
				return int64(s)
			}
		}
		if k == types.String {
			return s.WordSize * 2
		}
	case *types.Array:
		n := t.Len()
		if n == 0 {
			return 0
		}
		a := s.Alignof(t.Elem())
		z := s.Sizeof(t.Elem())
		return align(z, a)*(n-1) + z
	case *types.Slice:
		return s.WordSize * 3
	case *types.Struct:
		nf := t.NumFields()
		if nf == 0 {
			return 0
		}

		var o int64
		max := int64(1)
		for i := 0; i < nf; i++ {
			ft := t.Field(i).Type()
			a, sz := s.Alignof(ft), s.Sizeof(ft)
			if a > max {
				max = a
			}
			if i == nf-1 && sz == 0 && o != 0 {
				sz = 1
			}
			o = align(o, a) + sz
		}
		return align(o, max)
	case *types.Interface:
		return s.WordSize * 2
	}
	return s.WordSize // catch-all
}

// align returns the smallest y >= x such that y % a == 0.
func align(x, a int64) int64 {
	y := x + a - 1
	return y - y%a
}
<|endoftext|>


<commit_before>package cuckoo_test

import (
	"cuckood"
	"fmt"
	"math/rand"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"sync"
	"testing"
	"time"
)

func TestSimple(t *testing.T) {
	c := cuckoo.New()
	c.Insert("hello", "world")
	v, ok := c.Get("hello")

	if !ok {
		t.Error("Get did not return successfully")
	}

	switch v := v.(type) {
	case string:
		if v != "world" {
			t.Error("Get returned wrong string")
		}
	default:
		t.Error("Get did not return a string")
	}
}

func TestMany(t *testing.T) {
	c := cuckoo.New()

	for i := 0; i < 1e3; i++ {
		j := rand.Int()
		c.Insert(strconv.Itoa(j), j)
		v, ok := c.Get(strconv.Itoa(j))
		if !ok {
			t.Error("Concurrent get failed")
		}
		if vj, ok := v.(int); !ok || j != vj {
			t.Error("Concurrent get did not return correct value")
		}
	}
}

type igtime struct {
	i      int
	insert time.Duration
	get    time.Duration
}

func TestConcurrent(t *testing.T) {
	runtime.GOMAXPROCS(4)
	c := cuckoo.New()

	ech := make(chan bool)
	errs := 0
	go func() {
		for range ech {
			errs++
		}
	}()

	os.Remove("results.log")
	res, _ := os.Create("results.log")
	tms := make(chan igtime)
	go func() {
		for tm := range tms {
			fmt.Fprintf(res, "%d %f %f\n", tm.i, tm.insert.Seconds(), tm.get.Seconds())
		}
		res.Close()
	}()

	var wg sync.WaitGroup
	ch := make(chan int)
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(wid int) {
			defer wg.Done()
			for i := range ch {
				tm := igtime{}

				start := time.Now()
				e := c.Insert(strconv.Itoa(i), i)
				tm.insert = time.Now().Sub(start)

				if e != nil {
					ech <- true
					continue
				}

				start = time.Now()
				v, ok := c.Get(strconv.Itoa(i))
				tm.get = time.Now().Sub(start)

				if !ok {
					t.Error("Concurrent get failed")
				}
				if vi, ok := v.(int); !ok || i != vi {
					t.Error("Concurrent get did not return correct value")
				}

				tm.i = i
				tms <- tm
			}
		}(i)
	}

	os.Remove("cpu.out")
	cpu, _ := os.Create("cpu.out")
	pprof.StartCPUProfile(cpu)
	for i := 0; i < 70e3; i++ {
		ch <- i

		if i%2e3 == 0 {
			fmt.Println(i)
		}
	}
	close(ch)
	wg.Wait()
	close(tms)

	fmt.Println("observed", errs, "insert errors")

	os.Remove("mem.out")
	mem, _ := os.Create("mem.out")
	pprof.WriteHeapProfile(mem)

	pprof.StopCPUProfile()
	cpu.Close()
}
<commit_msg>Test atomic get/set<commit_after>package cuckoo_test

import (
	"cuckood"
	"fmt"
	"math/rand"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"sync"
	"testing"
	"time"
)

func TestSimple(t *testing.T) {
	c := cuckoo.New()
	c.Insert("hello", "world")
	v, ok := c.Get("hello")

	if !ok {
		t.Error("Get did not return successfully")
	}

	switch v := v.(type) {
	case string:
		if v != "world" {
			t.Error("Get returned wrong string")
		}
	default:
		t.Error("Get did not return a string")
	}
}

func TestMany(t *testing.T) {
	c := cuckoo.New()

	for i := 0; i < 1e3; i++ {
		j := rand.Int()
		c.Insert(strconv.Itoa(j), j)
		v, ok := c.Get(strconv.Itoa(j))
		if !ok {
			t.Error("Concurrent get failed")
		}
		if vj, ok := v.(int); !ok || j != vj {
			t.Error("Concurrent get did not return correct value")
		}
	}
}

type igtime struct {
	i      int
	insert time.Duration
	get    time.Duration
}

func TestConcurrent(t *testing.T) {
	runtime.GOMAXPROCS(4)
	c := cuckoo.New()

	ech := make(chan bool)
	errs := 0
	go func() {
		for range ech {
			errs++
		}
	}()

	os.Remove("results.log")
	res, _ := os.Create("results.log")
	tms := make(chan igtime)
	go func() {
		for tm := range tms {
			fmt.Fprintf(res, "%d %f %f\n", tm.i, tm.insert.Seconds(), tm.get.Seconds())
		}
		res.Close()
	}()

	var wg sync.WaitGroup
	ch := make(chan int)
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(wid int) {
			defer wg.Done()
			for i := range ch {
				tm := igtime{}

				start := time.Now()
				e := c.Insert(strconv.Itoa(i), i)
				tm.insert = time.Now().Sub(start)

				if e != nil {
					ech <- true
					continue
				}

				start = time.Now()
				v, ok := c.Get(strconv.Itoa(i))
				tm.get = time.Now().Sub(start)

				if !ok {
					t.Error("Concurrent get failed")
				}
				if vi, ok := v.(int); !ok || i != vi {
					t.Error("Concurrent get did not return correct value")
				}

				tm.i = i
				tms <- tm
			}
		}(i)
	}

	os.Remove("cpu.out")
	cpu, _ := os.Create("cpu.out")
	pprof.StartCPUProfile(cpu)
	for i := 0; i < 70e3; i++ {
		ch <- i

		if i%2e3 == 0 {
			fmt.Println(i)
		}
	}
	close(ch)
	wg.Wait()
	close(tms)

	fmt.Println("observed", errs, "insert errors")

	os.Remove("mem.out")
	mem, _ := os.Create("mem.out")
	pprof.WriteHeapProfile(mem)

	pprof.StopCPUProfile()
	cpu.Close()
}

func TestSameKey(t *testing.T) {
	runtime.GOMAXPROCS(4)
	c := cuckoo.New()

	get := func() {
		v, ok := c.Get("a")
		if !ok {
			t.Error("key lost")
		}
		if vi, ok := v.(int); ok {
			if vi != 1 && vi != 2 {
				t.Error("value is not one of the inserted values")
			}
		} else {
			t.Error("value has wrong type")
		}
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 1e5; i++ {
			c.Insert("a", 2)
			get()
		}
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 1e5; i++ {
			c.Insert("a", 1)
			get()
		}
	}()
	wg.Wait()
}
<|endoftext|>


<commit_before>package hamt_test

import (
	"log"
	"math/rand"
	"testing"
	"time"
)

func BenchmarkMapGet(b *testing.B) {
	log.Printf("BenchmarkMapGet: b.N=%d", b.N)

	for i := 0; i < b.N; i++ {
		var j = int(rand.Int31()) % numHugeKvs
		var s = keyStrings[j]
		var val, ok = LookupMap[s]
		if !ok {
			b.Fatalf("LookupMap[%s] not ok", s)
		}
		if val != j {
			b.Fatalf("val,%v != %v", val, j)
		}
	}
}

func BenchmarkMapPut(b *testing.B) {
	log.Printf("BenchmarkMapPut: b.N=%d", b.N)

	var m = make(map[string]int, b.N)
	var s = "aaa"
	for i := 0; i < b.N; i++ {
		m[s] = i
		s = Inc(s)
	}
}

func BenchmarkMapDel(b *testing.B) {
	log.Printf("BenchmarkMapDel: b.N=%d", b.N)

	StartTime["BenchmarkMapDel:rebuildDeleteMap"] = time.Now()

	rebuildDeleteMap(keyStrings)

	RunTime["build BenchmarkMapDel:rebuildDeleteMap"] = time.Since(StartTime["BenchmarkMapDel:rebuildDeleteMap"])

	b.ResetTimer()

	s := "aaa"
	for i := 0; i < b.N; i++ {
		delete(DeleteMap, s)
		s = Inc(s)
	}

	if len(DeleteMap) == 0 {
		b.Fatal("len(DeleteMap) == 0")
	}
}
<commit_msg>moved rebuildDeleteMap() into map_test.go<commit_after>package hamt_test

import (
	"log"
	"math/rand"
	"testing"
	"time"
)

func rebuildDeleteMap(strs []string) {
	for val, str := range strs {
		_, exists := DeleteMap[str]
		if exists {
			break
		}

		DeleteMap[str] = val
	}
}

func BenchmarkMapGet(b *testing.B) {
	log.Printf("BenchmarkMapGet: b.N=%d", b.N)

	for i := 0; i < b.N; i++ {
		var j = int(rand.Int31()) % numHugeKvs
		var s = keyStrings[j]
		var val, ok = LookupMap[s]
		if !ok {
			b.Fatalf("LookupMap[%s] not ok", s)
		}
		if val != j {
			b.Fatalf("val,%v != %v", val, j)
		}
	}
}

func BenchmarkMapPut(b *testing.B) {
	log.Printf("BenchmarkMapPut: b.N=%d", b.N)

	var m = make(map[string]int, b.N)
	var s = "aaa"
	for i := 0; i < b.N; i++ {
		m[s] = i
		s = Inc(s)
	}
}

func BenchmarkMapDel(b *testing.B) {
	log.Printf("BenchmarkMapDel: b.N=%d", b.N)

	StartTime["BenchmarkMapDel:rebuildDeleteMap"] = time.Now()

	rebuildDeleteMap(keyStrings)

	RunTime["build BenchmarkMapDel:rebuildDeleteMap"] = time.Since(StartTime["BenchmarkMapDel:rebuildDeleteMap"])

	b.ResetTimer()

	s := "aaa"
	for i := 0; i < b.N; i++ {
		delete(DeleteMap, s)
		s = Inc(s)
	}

	if len(DeleteMap) == 0 {
		b.Fatal("len(DeleteMap) == 0")
	}
}
<|endoftext|>


<commit_before>package gokeepasslib

import (
	"bytes"
	"compress/gzip"
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"reflect"
)

type Decoder struct {
	r io.Reader
}

func (d *Decoder) Decode(db *Database) error {
	if err := d.readSignature(db); err != nil {
		return err
	}

	if err := d.readHeaders(db); err != nil {
		return err
	}

	if err := d.readData(db); err != nil {
		return err
	}

	return nil
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

func (d *Decoder) readSignature(db *Database) error {
	sig := new(Signature)
	if err := binary.Read(d.r, binary.LittleEndian, sig); err != nil {
		return err
	}

	if sig.BaseSignature != [...]byte{0x03, 0xd9, 0xa2, 0x9a} {
		return errors.New("BaseSignature not valid")
	}
	if sig.VersionSignature != [...]byte{0x67, 0xfb, 0x4b, 0xb5} {
		return errors.New("VersionSignature not valid")
	}

	db.signature = *sig
	return nil
}

func (d *Decoder) readHeaders(db *Database) error {
	headers := new(Headers)

	for {
		var fieldID byte
		if err := binary.Read(d.r, binary.LittleEndian, &fieldID); err != nil {
			return err
		}

		var fieldLength [2]byte
		if err := binary.Read(d.r, binary.LittleEndian, &fieldLength); err != nil {
			return err
		}

		var fieldData = make([]byte, binary.LittleEndian.Uint16(fieldLength[:]))
		if err := binary.Read(d.r, binary.LittleEndian, &fieldData); err != nil {
			return err
		}

		switch fieldID {
		case 1:
			headers.Comment = fieldData
		case 2:
			headers.CipherID = fieldData
		case 3:
			headers.CompressionFlags = binary.LittleEndian.Uint32(fieldData)
		case 4:
			headers.MasterSeed = fieldData
		case 5:
			headers.TransformSeed = fieldData
		case 6:
			data := binary.LittleEndian.Uint32(fieldData[4:8])*
				(1<<16)*(1<<16) +
				binary.LittleEndian.Uint32(fieldData[0:4])
			headers.TransformRounds = data
		case 7:
			headers.EncryptionIV = fieldData
		case 8:
			headers.ProtectedStreamKey = fieldData
		case 9:
			headers.StreamStartBytes = fieldData
		case 10:
			headers.InnerRandomStreamID = fieldData
		}

		if fieldID == 0 {
			break
		}
	}

	db.headers = *headers
	return nil
}

func (d *Decoder) readData(db *Database) error {
	masterKey, nil := d.buildMasterKey(db)

	block, err := aes.NewCipher(masterKey)
	if err != nil {
		return err
	}

	in, err := ioutil.ReadAll(d.r)
	if err != nil {
		return err
	}

	mode := cipher.NewCBCDecrypter(block, db.headers.EncryptionIV)
	decrypted := make([]byte, len(in))
	mode.CryptBlocks(decrypted, in)

	startBytes := db.headers.StreamStartBytes
	if !reflect.DeepEqual(decrypted[0:len(startBytes)], startBytes) {
		return errors.New("Database integrity check failed")
	}
	decrypted = decrypted[len(startBytes):]

	// TODO some more decryption here

	b := bytes.NewBuffer(decrypted)
	r, err := gzip.NewReader(b)
	if err != nil {
		return err
	}
	defer r.Close()
	result := []byte{}
	r.Read(result)

	fmt.Println(result)

	return nil
}

func (d *Decoder) buildMasterKey(db *Database) ([]byte, error) {
	masterKey := make([]byte, 32)
	copy(masterKey, db.credentials.Key)

	tmp := sha256.Sum256(masterKey)
	masterKey = tmp[:]

	block, err := aes.NewCipher(db.headers.TransformSeed)
	if err != nil {
		return nil, err
	}

	// http://crypto.stackexchange.com/questions/21048/can-i-simulate-iterated-aes-ecb-with-other-block-cipher-modes
	for i := uint32(0); i < 6000; i++ {
		result := make([]byte, 16)
		crypter := cipher.NewCBCEncrypter(block, result)
		crypter.CryptBlocks(masterKey[:16], masterKey[:16])
		crypter = cipher.NewCBCEncrypter(block, result)
		crypter.CryptBlocks(masterKey[16:], masterKey[16:])
	}

	tmp = sha256.Sum256(masterKey)
	masterKey = tmp[:]

	masterKey = append(db.headers.MasterSeed, masterKey...)
	masterHash := sha256.Sum256(masterKey)
	masterKey = masterHash[:]

	return masterKey, nil
}
<commit_msg>First version of hash checking<commit_after>package gokeepasslib

import (
	"bytes"
	"compress/gzip"
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha256"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"reflect"
)

type Decoder struct {
	r io.Reader
}

func (d *Decoder) Decode(db *Database) error {
	if err := d.readSignature(db); err != nil {
		return err
	}

	if err := d.readHeaders(db); err != nil {
		return err
	}

	if err := d.readData(db); err != nil {
		return err
	}

	return nil
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

func (d *Decoder) readSignature(db *Database) error {
	sig := new(Signature)
	if err := binary.Read(d.r, binary.LittleEndian, sig); err != nil {
		return err
	}

	if sig.BaseSignature != [...]byte{0x03, 0xd9, 0xa2, 0x9a} {
		return errors.New("BaseSignature not valid")
	}
	if sig.VersionSignature != [...]byte{0x67, 0xfb, 0x4b, 0xb5} {
		return errors.New("VersionSignature not valid")
	}

	db.signature = *sig
	return nil
}

func (d *Decoder) readHeaders(db *Database) error {
	headers := new(Headers)

	for {
		var fieldID byte
		if err := binary.Read(d.r, binary.LittleEndian, &fieldID); err != nil {
			return err
		}

		var fieldLength [2]byte
		if err := binary.Read(d.r, binary.LittleEndian, &fieldLength); err != nil {
			return err
		}

		var fieldData = make([]byte, binary.LittleEndian.Uint16(fieldLength[:]))
		if err := binary.Read(d.r, binary.LittleEndian, &fieldData); err != nil {
			return err
		}

		switch fieldID {
		case 1:
			headers.Comment = fieldData
		case 2:
			headers.CipherID = fieldData
		case 3:
			headers.CompressionFlags = binary.LittleEndian.Uint32(fieldData)
		case 4:
			headers.MasterSeed = fieldData
		case 5:
			headers.TransformSeed = fieldData
		case 6:
			data := binary.LittleEndian.Uint32(fieldData[4:8])*
				(1<<16)*(1<<16) +
				binary.LittleEndian.Uint32(fieldData[0:4])
			headers.TransformRounds = data
		case 7:
			headers.EncryptionIV = fieldData
		case 8:
			headers.ProtectedStreamKey = fieldData
		case 9:
			headers.StreamStartBytes = fieldData
		case 10:
			headers.InnerRandomStreamID = fieldData
		}

		if fieldID == 0 {
			break
		}
	}

	db.headers = *headers
	return nil
}

func (d *Decoder) readData(db *Database) error {
	masterKey, nil := d.buildMasterKey(db)

	block, err := aes.NewCipher(masterKey)
	if err != nil {
		return err
	}

	in, err := ioutil.ReadAll(d.r)
	if err != nil {
		return err
	}

	mode := cipher.NewCBCDecrypter(block, db.headers.EncryptionIV)
	decrypted := make([]byte, len(in))
	mode.CryptBlocks(decrypted, in)

	startBytes := db.headers.StreamStartBytes
	if !reflect.DeepEqual(decrypted[0:len(startBytes)], startBytes) {
		return errors.New("Database integrity check failed")
	}
	decrypted = decrypted[len(startBytes):]

	zippedBody, err := d.checkHashBlocks(decrypted)
	if err != nil {
		return err
	}

	b := bytes.NewBuffer(zippedBody)
	r, err := gzip.NewReader(b)
	if err != nil {
		return err
	}
	defer r.Close()
	result := []byte{}
	r.Read(result)

	fmt.Println(result)

	return nil
}

func (d *Decoder) buildMasterKey(db *Database) ([]byte, error) {
	masterKey := make([]byte, 32)
	copy(masterKey, db.credentials.Key)

	tmp := sha256.Sum256(masterKey)
	masterKey = tmp[:]

	block, err := aes.NewCipher(db.headers.TransformSeed)
	if err != nil {
		return nil, err
	}

	// http://crypto.stackexchange.com/questions/21048/can-i-simulate-iterated-aes-ecb-with-other-block-cipher-modes
	for i := uint32(0); i < 6000; i++ {
		result := make([]byte, 16)
		crypter := cipher.NewCBCEncrypter(block, result)
		crypter.CryptBlocks(masterKey[:16], masterKey[:16])
		crypter = cipher.NewCBCEncrypter(block, result)
		crypter.CryptBlocks(masterKey[16:], masterKey[16:])
	}

	tmp = sha256.Sum256(masterKey)
	masterKey = tmp[:]

	masterKey = append(db.headers.MasterSeed, masterKey...)
	masterHash := sha256.Sum256(masterKey)
	masterKey = masterHash[:]

	return masterKey, nil
}

func (d *Decoder) checkHashBlocks(hashedBody []byte) ([]byte, error) {
	result := make([]byte, 0)

	for len(hashedBody) > 0 {
		fmt.Println(len(hashedBody))
		index := binary.LittleEndian.Uint32(hashedBody[:4])
		hashedBody = hashedBody[4:]
		blockHash := hashedBody[:32]
		hashedBody = hashedBody[32:]
		blockLength := binary.LittleEndian.Uint32(hashedBody[:4])
		hashedBody = hashedBody[4:]

		if blockLength > 0 {
			blockData := hashedBody[:blockLength]
			hashedBody = hashedBody[blockLength:]
			calculatedHash := sha256.Sum256(blockData)

			if !reflect.DeepEqual(calculatedHash[:], blockHash[:]) {
				return nil, fmt.Errorf("Hash mismatch. Database seems to be corrupt at index %d", index)
			} else {
				result = append(result, blockData...)
			}
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Jason Goecke
// messages.go

package wit

import (
	"encoding/json"
	"net/url"
)

// Represents a Wit message (https://wit.ai/docs/api#toc_3)
type Message struct {
	MsgId   string  `json:"msg_id"`
	MsgBody string  `json:"msg_body"`
	Outcome Outcome `json:"outcome"`
}

// Represents the outcome portion of a Wit message
type Outcome struct {
	Intent     string        `json:"intent"`
	Entities   MessageEntity `json:"entities"`
	Confidence float32       `json:"confidence"`
}

// Represents the entity portion of a Wit message
type MessageEntity struct {
	Metric   Metric     `json:"metric"`
	Datetime []Datetime `json:"datetime"`
}

// Represents the metric portion of a Wit message
type Metric struct {
	Value string `json:"value"`
	Body  string `json:"value"`
}

// Represents the datetime portion of a Wit message
type Datetime struct {
	Value DatetimeValue `json:"value"`
	Body  string        `json:"body"`
}

// Represents the datetime value portion of a Wit message
type DatetimeValue struct {
	From string `json:"from"`
	To   string `json:"to"`
}

// Represents a request to process a message
type MessageRequest struct {
	File, Query, MsgId, ContentType string
	FileContents                    []byte
	// Are context and Meta necessary anymore?
	// Context     Context
	// Meta        map[string]interface{}
}

// Represents the context portion of the message request
type Context struct {
	ReferenceTime string `json:"reference_time"`
	Timezone      string `json:"timezone"`
}

// Lists an already existing message (https://wit.ai/docs/api#toc_11)
//
//		result, err := client.Messages("ba0fcf60-44d3-4499-877e-c8d65c239730")
func (client *WitClient) Messages(id string) (*Message, error) {
	result, err := get(client.ApiBase + "/messages/" + id)
	if err != nil {
		return nil, err
	}
	message, err := parseMessage(result)
	return message, nil
}

// Requests processing of a text message (https://wit.ai/docs/api#toc_3)
//
//		result, err := client.Message(request)
func (client *WitClient) Message(request *MessageRequest) (*Message, error) {
	result, err := get(client.ApiBase + "/message?q=" + url.QueryEscape(request.Query))
	if err != nil {
		return nil, err
	}
	message, _ := parseMessage(result)
	return message, nil
}

// Requests processing of an audio message (https://wit.ai/docs/api#toc_8)
//
// 		request := &MessageRequest{}
// 		request.File = "./audio_sample/helloWorld.wav"
//		request.FileContents = data
//		request.ContentType = "audio/wav;rate=8000"
// 		message, err := client.AudioMessage(request)
func (client *WitClient) AudioMessage(request *MessageRequest) (*Message, error) {
	result, err := postFile(client.ApiBase+"/speech", request)
	if err != nil {
	}
	message, err := parseMessage(result)
	if err != nil {
		return nil, err
	}
	return message, nil
}

// Parses the JSON into a Message
//
//		message, err := parseMessage([]byte(data))
func parseMessage(data []byte) (*Message, error) {
	message := &Message{}
	err := json.Unmarshal(data, message)
	if err != nil {
		return nil, err
	}
	return message, nil
}
<commit_msg>Fixed a bug where the wrong error message was bubbling up.<commit_after>// Copyright (c) 2014 Jason Goecke
// messages.go

package wit

import (
	"encoding/json"
	"net/url"
)

// Represents a Wit message (https://wit.ai/docs/api#toc_3)
type Message struct {
	MsgId   string  `json:"msg_id"`
	MsgBody string  `json:"msg_body"`
	Outcome Outcome `json:"outcome"`
}

// Represents the outcome portion of a Wit message
type Outcome struct {
	Intent     string        `json:"intent"`
	Entities   MessageEntity `json:"entities"`
	Confidence float32       `json:"confidence"`
}

// Represents the entity portion of a Wit message
type MessageEntity struct {
	Metric   Metric     `json:"metric"`
	Datetime []Datetime `json:"datetime"`
}

// Represents the metric portion of a Wit message
type Metric struct {
	Value string `json:"value"`
	Body  string `json:"value"`
}

// Represents the datetime portion of a Wit message
type Datetime struct {
	Value DatetimeValue `json:"value"`
	Body  string        `json:"body"`
}

// Represents the datetime value portion of a Wit message
type DatetimeValue struct {
	From string `json:"from"`
	To   string `json:"to"`
}

// Represents a request to process a message
type MessageRequest struct {
	File, Query, MsgId, ContentType string
	FileContents                    []byte
	// Are context and Meta necessary anymore?
	// Context     Context
	// Meta        map[string]interface{}
}

// Represents the context portion of the message request
type Context struct {
	ReferenceTime string `json:"reference_time"`
	Timezone      string `json:"timezone"`
}

// Lists an already existing message (https://wit.ai/docs/api#toc_11)
//
//		result, err := client.Messages("ba0fcf60-44d3-4499-877e-c8d65c239730")
func (client *WitClient) Messages(id string) (*Message, error) {
	result, err := get(client.ApiBase + "/messages/" + id)
	if err != nil {
		return nil, err
	}
	message, err := parseMessage(result)
	return message, nil
}

// Requests processing of a text message (https://wit.ai/docs/api#toc_3)
//
//		result, err := client.Message(request)
func (client *WitClient) Message(request *MessageRequest) (*Message, error) {
	result, err := get(client.ApiBase + "/message?q=" + url.QueryEscape(request.Query))
	if err != nil {
		return nil, err
	}
	message, _ := parseMessage(result)
	return message, nil
}

// Requests processing of an audio message (https://wit.ai/docs/api#toc_8)
//
// 		request := &MessageRequest{}
// 		request.File = "./audio_sample/helloWorld.wav"
//		request.FileContents = data
//		request.ContentType = "audio/wav;rate=8000"
// 		message, err := client.AudioMessage(request)
func (client *WitClient) AudioMessage(request *MessageRequest) (*Message, error) {
	result, err := postFile(client.ApiBase+"/speech", request)
	if err != nil {
		return nil, err
	}
	message, err := parseMessage(result)
	if err != nil {
		return nil, err
	}
	return message, nil
}

// Parses the JSON into a Message
//
//		message, err := parseMessage([]byte(data))
func parseMessage(data []byte) (*Message, error) {
	message := &Message{}
	err := json.Unmarshal(data, message)
	if err != nil {
		return nil, err
	}
	return message, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sync"

	"gopnik"
)

const (
	PENDING    = 0
	INPROGRESS = 1
	DONE       = 2
)

type plan struct {
	bboxes []gopnik.TileCoord
	status []uint8
	cursor int
	mu     sync.Mutex
	cond   *sync.Cond
	condMu sync.Mutex
}

func newPlan(bboxes []gopnik.TileCoord) *plan {
	self := &plan{
		bboxes: bboxes,
		status: make([]uint8, len(bboxes)),
	}
	self.cond = sync.NewCond(&self.condMu)
	return self
}

func (self *plan) TotalTasks() int {
	return len(self.bboxes)
}

func (self *plan) DoneTasks() int {
	self.mu.Lock()
	defer self.mu.Unlock()

	result := 0
	for _, s := range self.status {
		if s == DONE {
			result++
		}
	}
	return result
}

func (self *plan) GetTask() *gopnik.TileCoord {
	self.mu.Lock()
	defer self.mu.Unlock()

	oldCursor := self.cursor
	allDone := true
	for {
		self.cursor++
		if self.cursor == len(self.bboxes) {
			self.cursor = 0
		}

		if self.status[self.cursor] == PENDING {
			bb := self.bboxes[self.cursor]
			self.status[self.cursor] = INPROGRESS
			return &bb
		}
		if self.status[self.cursor] != DONE {
			allDone = false
		}

		// On start point again
		if self.cursor == oldCursor {
			// Is plan complete?
			if allDone {
				return nil
			}
			// Waiting...
			log.Debug("Waiting for undone tasks...")
			self.condMu.Lock()
			self.mu.Unlock()
			self.cond.Wait()
			self.condMu.Unlock()
			self.mu.Lock()
			oldCursor = self.cursor
			allDone = true
		}
	}
}

func (self *plan) setStatus(coord gopnik.TileCoord, status uint8) error {
	for i, c := range self.bboxes {
		if coord.Equals(&c) {
			self.status[i] = status
			self.cond.Broadcast()
			return nil
		}
	}
	return fmt.Errorf("Can't find task %v", coord)
}

func (self *plan) FailTask(coord gopnik.TileCoord) error {
	return self.setStatus(coord, PENDING)
}

func (self *plan) DoneTask(coord gopnik.TileCoord) error {
	return self.setStatus(coord, DONE)
}
<commit_msg>fix race in prerender plan<commit_after>package main

import (
	"fmt"
	"sync"

	"gopnik"
)

const (
	PENDING    = 0
	INPROGRESS = 1
	DONE       = 2
)

type plan struct {
	bboxes []gopnik.TileCoord
	status []uint8
	cursor int
	mu     sync.Mutex
	cond   *sync.Cond
	condMu sync.Mutex
}

func newPlan(bboxes []gopnik.TileCoord) *plan {
	self := &plan{
		bboxes: bboxes,
		status: make([]uint8, len(bboxes)),
	}
	self.cond = sync.NewCond(&self.condMu)
	return self
}

func (self *plan) TotalTasks() int {
	return len(self.bboxes)
}

func (self *plan) DoneTasks() int {
	self.mu.Lock()
	defer self.mu.Unlock()

	result := 0
	for _, s := range self.status {
		if s == DONE {
			result++
		}
	}
	return result
}

func (self *plan) GetTask() *gopnik.TileCoord {
	self.mu.Lock()
	defer self.mu.Unlock()

	oldCursor := self.cursor
	allDone := true
	for {
		self.cursor++
		if self.cursor == len(self.bboxes) {
			self.cursor = 0
		}

		if self.status[self.cursor] == PENDING {
			bb := self.bboxes[self.cursor]
			self.status[self.cursor] = INPROGRESS
			return &bb
		}
		if self.status[self.cursor] != DONE {
			allDone = false
		}

		// On start point again
		if self.cursor == oldCursor {
			// Is plan complete?
			if allDone {
				return nil
			}
			// Waiting...
			log.Debug("Waiting for undone tasks...")
			self.condMu.Lock()
			self.mu.Unlock()
			self.cond.Wait()
			self.condMu.Unlock()
			self.mu.Lock()
			oldCursor = self.cursor
			allDone = true
		}
	}
}

func (self *plan) setStatus(coord gopnik.TileCoord, status uint8) error {
	self.mu.Lock()
	defer self.mu.Unlock()

	for i, c := range self.bboxes {
		if coord.Equals(&c) {
			self.status[i] = status
			self.cond.Broadcast()
			return nil
		}
	}
	return fmt.Errorf("Can't find task %v", coord)
}

func (self *plan) FailTask(coord gopnik.TileCoord) error {
	return self.setStatus(coord, PENDING)
}

func (self *plan) DoneTask(coord gopnik.TileCoord) error {
	return self.setStatus(coord, DONE)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

type licensesReply struct {
	Key      string `json:"key"`
	Name     string `json:"name"`
	URL      string `json:"url"`
	Featured bool   `json:"featured"`
}

type ownerReply struct {
	URL string `json:"url"`
}

type repositoryReply struct {
	License     licensesReply `json:"license"`
	CreatedAt   string        `json:"created_at"`
	Description string        `json:"description"`
	Owner       ownerReply    `json:"owner"`
}

type licenseReply struct {
	Body string `json:"body"`
}

type usersReply struct {
	Name string `json:"name"`
}

// To update, use:
// curl -s -H 'Accept: application/vnd.github.drax-preview+json' https://api.github.com/licenses | jq '.[].key'
// then compare with https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-specification
var githubLicenseToDebianLicense = map[string]string{
	//"agpl-3.0" (not in debian?)
	"apache-2.0":   "Apache-2.0",
	"artistic-2.0": "Artistic-2.0",
	"bsd-2-clause": "BSD-2-clause",
	"bsd-3-clause": "BSD-3-clause",
	"cc0-1.0":      "CC0-1.0",
	//"epl-1.0" (eclipse public license)
	"gpl-2.0":  "GPL-2.0", // TODO: is this GPL-2.0+?
	"gpl-3.0":  "GPL-3.0",
	"isc":      "ISC",
	"lgpl-2.1": "LGPL-2.1",
	"lgpl-3.0": "LGPL-3.0",
	//"mit" - expat?
	//"mpl-2.0" (only 1.1 is in debian)
	//"unlicense" (not in debian)
}

var debianLicenseText = map[string]string{
	"Apache-2.0": ` Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 .
 http://www.apache.org/licenses/LICENSE-2.0
 .
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 .
 On Debian systems, the complete text of the Apache version 2.0 license
 can be found in "/usr/share/common-licenses/Apache-2.0".
`,
}

func getLicenseForGopkg(gopkg string) (string, string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	// TODO: cache this reply
	req, err := http.NewRequest("GET", "https://api.github.com/repos/"+gopkg[len("github.com/"):], nil)
	if err != nil {
		return "", "", err
	}
	req.Header.Set("Accept", "application/vnd.github.drax-preview+json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()
	var r repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return "", "", err
	}
	if deblicense, ok := githubLicenseToDebianLicense[r.License.Key]; ok {
		fulltext := debianLicenseText[deblicense]
		if fulltext == "" {
			fulltext = "TODO"
		}
		return deblicense, fulltext, nil
	} else {
		return "TODO", "TODO", nil
	}
}

func getAuthorAndCopyrightForGopkg(gopkg string) (string, string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	resp, err := http.Get("https://api.github.com/repos/" + gopkg[len("github.com/"):])
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	var rr repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
		return "", "", err
	}

	creation, err := time.Parse("2006-01-02T15:04:05Z", rr.CreatedAt)
	if err != nil {
		return "", "", err
	}

	if strings.TrimSpace(rr.Owner.URL) == "" {
		return "", "", fmt.Errorf("Repository owner URL not present in API response")
	}

	resp, err = http.Get(rr.Owner.URL)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	var ur usersReply
	if err := json.NewDecoder(resp.Body).Decode(&ur); err != nil {
		return "", "", err
	}

	return ur.Name, creation.Format("2006") + " " + ur.Name, nil
}

func getDescriptionForGopkg(gopkg string) (string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	resp, err := http.Get("https://api.github.com/repos/" + gopkg[len("github.com/"):])
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var rr repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
		return "", err
	}

	return strings.TrimSpace(rr.Description), nil
}
<commit_msg>Set copyright to Google Inc. for github.com/google/*<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

type licensesReply struct {
	Key      string `json:"key"`
	Name     string `json:"name"`
	URL      string `json:"url"`
	Featured bool   `json:"featured"`
}

type ownerReply struct {
	URL string `json:"url"`
}

type repositoryReply struct {
	License     licensesReply `json:"license"`
	CreatedAt   string        `json:"created_at"`
	Description string        `json:"description"`
	Owner       ownerReply    `json:"owner"`
}

type licenseReply struct {
	Body string `json:"body"`
}

type usersReply struct {
	Name string `json:"name"`
}

// To update, use:
// curl -s -H 'Accept: application/vnd.github.drax-preview+json' https://api.github.com/licenses | jq '.[].key'
// then compare with https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-specification
var githubLicenseToDebianLicense = map[string]string{
	//"agpl-3.0" (not in debian?)
	"apache-2.0":   "Apache-2.0",
	"artistic-2.0": "Artistic-2.0",
	"bsd-2-clause": "BSD-2-clause",
	"bsd-3-clause": "BSD-3-clause",
	"cc0-1.0":      "CC0-1.0",
	//"epl-1.0" (eclipse public license)
	"gpl-2.0":  "GPL-2.0", // TODO: is this GPL-2.0+?
	"gpl-3.0":  "GPL-3.0",
	"isc":      "ISC",
	"lgpl-2.1": "LGPL-2.1",
	"lgpl-3.0": "LGPL-3.0",
	//"mit" - expat?
	//"mpl-2.0" (only 1.1 is in debian)
	//"unlicense" (not in debian)
}

var debianLicenseText = map[string]string{
	"Apache-2.0": ` Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 .
 http://www.apache.org/licenses/LICENSE-2.0
 .
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 .
 On Debian systems, the complete text of the Apache version 2.0 license
 can be found in "/usr/share/common-licenses/Apache-2.0".
`,
}

func getLicenseForGopkg(gopkg string) (string, string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	// TODO: cache this reply
	req, err := http.NewRequest("GET", "https://api.github.com/repos/"+gopkg[len("github.com/"):], nil)
	if err != nil {
		return "", "", err
	}
	req.Header.Set("Accept", "application/vnd.github.drax-preview+json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()
	var r repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return "", "", err
	}
	if deblicense, ok := githubLicenseToDebianLicense[r.License.Key]; ok {
		fulltext := debianLicenseText[deblicense]
		if fulltext == "" {
			fulltext = "TODO"
		}
		return deblicense, fulltext, nil
	} else {
		return "TODO", "TODO", nil
	}
}

func getAuthorAndCopyrightForGopkg(gopkg string) (string, string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	resp, err := http.Get("https://api.github.com/repos/" + gopkg[len("github.com/"):])
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	var rr repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
		return "", "", err
	}

	creation, err := time.Parse("2006-01-02T15:04:05Z", rr.CreatedAt)
	if err != nil {
		return "", "", err
	}

	if strings.TrimSpace(rr.Owner.URL) == "" {
		return "", "", fmt.Errorf("Repository owner URL not present in API response")
	}

	resp, err = http.Get(rr.Owner.URL)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	var ur usersReply
	if err := json.NewDecoder(resp.Body).Decode(&ur); err != nil {
		return "", "", err
	}

	copyright := creation.Format("2006") + " " + ur.Name
	if strings.HasPrefix(gopkg, "github.com/google/") {
		// As per https://opensource.google.com/docs/creating/, Google retains
		// the copyright for repositories underneath github.com/google/.
		copyright = creation.Format("2006") + " Google Inc."
	}

	return ur.Name, copyright, nil
}

func getDescriptionForGopkg(gopkg string) (string, error) {
	if !strings.HasPrefix(gopkg, "github.com/") {
		return "", fmt.Errorf("%q is not on GitHub", gopkg)
	}
	resp, err := http.Get("https://api.github.com/repos/" + gopkg[len("github.com/"):])
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var rr repositoryReply
	if err := json.NewDecoder(resp.Body).Decode(&rr); err != nil {
		return "", err
	}

	return strings.TrimSpace(rr.Description), nil
}
<|endoftext|>