<commit_before>package mcompress

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"
)

const (
	Unknown    Format = iota // unknown format
	Gzip                     // Gzip compression format
	Tar                      // Tar format; normally used
	Tar1                     // Tar1 magicnum format; normalizes to Tar
	Tar2                     // Tar1 magicnum format; normalizes to Tar
	Zip                      // Zip archive
	ZipEmpty                 // Empty Zip Archive
	ZipSpanned               // Spanned Zip Archive
	Bzip2                    // Bzip2 compression
	//LZW                      // LZW compression
	LZ4 // LZ4 compression
)

// Magic numbers for magicnum for compression and archive formats
var (
	magicnumGzip       = []byte{0x1f, 0x8b}
	magicnumTar1       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	magicnumTar2       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	magicnumZip        = []byte{0x50, 0x4b, 0x03, 0x04}
	magicnumZipEmpty   = []byte{0x50, 0x4b, 0x05, 0x06}
	magicnumZipSpanned = []byte{0x50, 0x4b, 0x07, 0x08}
	magicnumBzip2      = []byte{0x42, 0x5a, 0x68}
	//magicnumLZW        = []byte{0x1F, 0x9d}
	magicnumLZ4 = []byte{0x18, 0x4d, 0x22, 0x04}
)

// TODO: should Format be more specific? e.g. CompressionFormat, MediaFormat, etc.
type Format int

func (f Format) String() string {
	switch f {
	case Gzip:
		return "gzip"
	case Tar, Tar1, Tar2:
		return "tar"
	case Zip:
		return "zip"
	case ZipEmpty:
		return "empty zip archive"
	case ZipSpanned:
		return "spanned zip archive"
	case Bzip2:
		return "bzip2"
	//case LZW:
	//	return "lzw"
	case LZ4:
		return "lz4"
	}
	return "unknown"
}

// Ext returns the extension for the format. Formats may have more than one
// accepted extension; alternate extensiona are not supported.
func (f Format) Ext() string {
	switch f {
	case Gzip:
		return ".gz"
	case Tar, Tar1, Tar2:
		return ".tar"
	case Zip, ZipEmpty, ZipSpanned:
		return ".zip"
	case Bzip2:
		return ".bz2"
	//case LZW:
	//	return ".Z"
	case LZ4:
		return ".lz4"
	}
	return "unknown"
}

func FormatFromString(s string) Format {
	s = strings.ToLower(s)
	switch s {
	case "gzip", "gz":
		return Gzip
	case "tar":
		return Tar
	case "zip":
		return Zip
	case "bzip2", "bz2":
		return Bzip2
	//case "lzw", "Z":
	//	return LZW
	case "lz4":
		return LZ4
	}
	return Unknown
}

// ParseFormat takes a string and returns the format or unknown. Any compressed
// tar extensions are returned as the compression format and not tar.
//
// If the passed string starts with a '.', it is removed.
// All strings are lowercased
func ParseFormat(s string) Format {
	if s[0] == '.' {
		s = s[1:]
	}
	s = strings.ToLower(s)
	switch s {
	case "gzip", "tar.gz", "tgz":
		return Gzip
	case "tar":
		return Tar
	case "bz2", "tbz", "tb2", "tbz2", "tar.bz2":
		return Bzip2
	case "lz4", "tar.lz4", "tz4":
		return LZ4
	case "zip":
		return Zip
	}
	return Unknown
}

// GetFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
//
// For zips, this will also match on files with empty zip or spanned zip magic
// numbers.  If you need to distinguich between the various zip formats, use
// something else.
func GetFormat(r io.ReaderAt) (Format, error) {
	ok, err := IsLZ4(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZ4, nil
	}
	ok, err = IsGzip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Gzip, nil
	}
	ok, err = IsZip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Zip, nil
	}
	ok, err = IsTar(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Tar, nil
	}
	ok, err = IsBzip2(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Bzip2, nil
	}
	//ok, err = IsLZW(r)
	//if err != nil {
	//	return Unknown, err
	//}
	//if ok {
	//	return LZW, nil
	//}
	return Unknown, errors.New("unsupported format: input format is not known")
}

// IsBzip2 checks to see if the received reader's contents are in bzip2 format
// by checking the magic numbers.
func IsBzip2(r io.ReaderAt) (bool, error) {
	h := make([]byte, 3)
	// Read the first 3 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var hb [3]byte
	// check for bzip2
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &hb)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var cb [3]byte
	cbuf := bytes.NewBuffer(magicnumBzip2)
	err = binary.Read(cbuf, binary.BigEndian, &cb)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if hb == cb {
		return true, nil
	}
	return false, nil
}

// IsGzip checks to see if the received reader's contents are in gzip format
// by checking the magic numbers.
func IsGzip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Read the first 2 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for gzip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var c16 uint16
	cbuf := bytes.NewBuffer(magicnumGzip)
	err = binary.Read(cbuf, binary.BigEndian, &c16)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if h16 == c16 {
		return true, nil
	}
	return false, nil
}

// IsLZ4 checks to see if the received reader's contents are in LZ4 foramt by
// checking the magic numbers.
func IsLZ4(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for lz4
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZ4's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(magicnumLZ4)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting LZ4 magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}

// IsLZW checks to see if the received reader's contents are in LZ4 format by
// checking the magic numbers.
//
// TODO: unsupported until I have a better understanding of how to handle LZW
/*
func IsLZW(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Reat the first 8 bytes since that's where most magic numbers are
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for lzw
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZW's magic number: %s", err)
	}
	var c16 uint16
	cbuf := bytes.NewBuffer(magicnumLZW)
	err = binary.Read(cbuf, binary.BigEndian, &c16)
	if err != nil {
		return false, fmt.Errorf("error while converting LZW magic number for comparison: %s", err)
	}
	if h16 == c16 {
		return true, nil
	}
	return false, nil
}
*/

// IsTar checks to see if the received reader's contents are in the tar format
// by checking the magic numbers. This evaluates using both tar1 and tar2 magic
// numbers.
func IsTar(r io.ReaderAt) (bool, error) {
	h := make([]byte, 8)
	// Read the first 8 bytes at offset 257
	_, err := r.ReadAt(h, 257)
	if err != nil {
		return false, err
	}
	var h64 uint64
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h64)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched tar's magic number: %s", err)
	}
	var c64 uint64
	cbuf := bytes.NewBuffer(magicnumTar1)
	err = binary.Read(cbuf, binary.BigEndian, &c64)
	if err != nil {
		return false, fmt.Errorf("error while converting the tar magic number for comparison: %s", err)
	}
	if h64 == c64 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumTar2)
	err = binary.Read(cbuf, binary.BigEndian, &c64)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty tar magic number for comparison: %s", err)
	}
	if h64 == c64 {
		return true, nil
	}
	return false, nil
}

// IsZip checks to see if the received reader's contents are in the zip format
// by checking the magic numbers. This will match on zip, empty zip and spanned
// zip magic numbers. If you need to distinguish between those, use something
// else.
func IsZip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched zip's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(magicnumZip)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumZipEmpty)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumZipSpanned)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the spanned zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}
<commit_msg>fix magicnum var comment<commit_after>package mcompress

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"
)

const (
	Unknown    Format = iota // unknown format
	Gzip                     // Gzip compression format
	Tar                      // Tar format; normally used
	Tar1                     // Tar1 magicnum format; normalizes to Tar
	Tar2                     // Tar1 magicnum format; normalizes to Tar
	Zip                      // Zip archive
	ZipEmpty                 // Empty Zip Archive
	ZipSpanned               // Spanned Zip Archive
	Bzip2                    // Bzip2 compression
	//LZW                      // LZW compression
	LZ4 // LZ4 compression
)

// Magic numbers for compression and archive formats
var (
	magicnumGzip       = []byte{0x1f, 0x8b}
	magicnumTar1       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30} // offset: 257
	magicnumTar2       = []byte{0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x20, 0x00} // offset: 257
	magicnumZip        = []byte{0x50, 0x4b, 0x03, 0x04}
	magicnumZipEmpty   = []byte{0x50, 0x4b, 0x05, 0x06}
	magicnumZipSpanned = []byte{0x50, 0x4b, 0x07, 0x08}
	magicnumBzip2      = []byte{0x42, 0x5a, 0x68}
	//magicnumLZW        = []byte{0x1F, 0x9d}
	magicnumLZ4 = []byte{0x18, 0x4d, 0x22, 0x04}
)

// TODO: should Format be more specific? e.g. CompressionFormat, MediaFormat, etc.
type Format int

func (f Format) String() string {
	switch f {
	case Gzip:
		return "gzip"
	case Tar, Tar1, Tar2:
		return "tar"
	case Zip:
		return "zip"
	case ZipEmpty:
		return "empty zip archive"
	case ZipSpanned:
		return "spanned zip archive"
	case Bzip2:
		return "bzip2"
	//case LZW:
	//	return "lzw"
	case LZ4:
		return "lz4"
	}
	return "unknown"
}

// Ext returns the extension for the format. Formats may have more than one
// accepted extension; alternate extensiona are not supported.
func (f Format) Ext() string {
	switch f {
	case Gzip:
		return ".gz"
	case Tar, Tar1, Tar2:
		return ".tar"
	case Zip, ZipEmpty, ZipSpanned:
		return ".zip"
	case Bzip2:
		return ".bz2"
	//case LZW:
	//	return ".Z"
	case LZ4:
		return ".lz4"
	}
	return "unknown"
}

func FormatFromString(s string) Format {
	s = strings.ToLower(s)
	switch s {
	case "gzip", "gz":
		return Gzip
	case "tar":
		return Tar
	case "zip":
		return Zip
	case "bzip2", "bz2":
		return Bzip2
	//case "lzw", "Z":
	//	return LZW
	case "lz4":
		return LZ4
	}
	return Unknown
}

// ParseFormat takes a string and returns the format or unknown. Any compressed
// tar extensions are returned as the compression format and not tar.
//
// If the passed string starts with a '.', it is removed.
// All strings are lowercased
func ParseFormat(s string) Format {
	if s[0] == '.' {
		s = s[1:]
	}
	s = strings.ToLower(s)
	switch s {
	case "gzip", "tar.gz", "tgz":
		return Gzip
	case "tar":
		return Tar
	case "bz2", "tbz", "tb2", "tbz2", "tar.bz2":
		return Bzip2
	case "lz4", "tar.lz4", "tz4":
		return LZ4
	case "zip":
		return Zip
	}
	return Unknown
}

// GetFormat tries to match up the data in the Reader to a supported
// magic number, if a match isn't found, UnsupportedFmt is returned
//
// For zips, this will also match on files with empty zip or spanned zip magic
// numbers.  If you need to distinguich between the various zip formats, use
// something else.
func GetFormat(r io.ReaderAt) (Format, error) {
	ok, err := IsLZ4(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return LZ4, nil
	}
	ok, err = IsGzip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Gzip, nil
	}
	ok, err = IsZip(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Zip, nil
	}
	ok, err = IsTar(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Tar, nil
	}
	ok, err = IsBzip2(r)
	if err != nil {
		return Unknown, err
	}
	if ok {
		return Bzip2, nil
	}
	//ok, err = IsLZW(r)
	//if err != nil {
	//	return Unknown, err
	//}
	//if ok {
	//	return LZW, nil
	//}
	return Unknown, errors.New("unsupported format: input format is not known")
}

// IsBzip2 checks to see if the received reader's contents are in bzip2 format
// by checking the magic numbers.
func IsBzip2(r io.ReaderAt) (bool, error) {
	h := make([]byte, 3)
	// Read the first 3 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var hb [3]byte
	// check for bzip2
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &hb)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var cb [3]byte
	cbuf := bytes.NewBuffer(magicnumBzip2)
	err = binary.Read(cbuf, binary.BigEndian, &cb)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if hb == cb {
		return true, nil
	}
	return false, nil
}

// IsGzip checks to see if the received reader's contents are in gzip format
// by checking the magic numbers.
func IsGzip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Read the first 2 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for gzip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched bzip2's magic number: %s", err)
	}
	var c16 uint16
	cbuf := bytes.NewBuffer(magicnumGzip)
	err = binary.Read(cbuf, binary.BigEndian, &c16)
	if err != nil {
		return false, fmt.Errorf("error while converting bzip2 magic number for comparison: %s", err)
	}
	if h16 == c16 {
		return true, nil
	}
	return false, nil
}

// IsLZ4 checks to see if the received reader's contents are in LZ4 foramt by
// checking the magic numbers.
func IsLZ4(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for lz4
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZ4's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(magicnumLZ4)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting LZ4 magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}

// IsLZW checks to see if the received reader's contents are in LZ4 format by
// checking the magic numbers.
//
// TODO: unsupported until I have a better understanding of how to handle LZW
/*
func IsLZW(r io.ReaderAt) (bool, error) {
	h := make([]byte, 2)
	// Reat the first 8 bytes since that's where most magic numbers are
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h16 uint16
	// check for lzw
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.LittleEndian, &h16)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched LZW's magic number: %s", err)
	}
	var c16 uint16
	cbuf := bytes.NewBuffer(magicnumLZW)
	err = binary.Read(cbuf, binary.BigEndian, &c16)
	if err != nil {
		return false, fmt.Errorf("error while converting LZW magic number for comparison: %s", err)
	}
	if h16 == c16 {
		return true, nil
	}
	return false, nil
}
*/

// IsTar checks to see if the received reader's contents are in the tar format
// by checking the magic numbers. This evaluates using both tar1 and tar2 magic
// numbers.
func IsTar(r io.ReaderAt) (bool, error) {
	h := make([]byte, 8)
	// Read the first 8 bytes at offset 257
	_, err := r.ReadAt(h, 257)
	if err != nil {
		return false, err
	}
	var h64 uint64
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h64)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched tar's magic number: %s", err)
	}
	var c64 uint64
	cbuf := bytes.NewBuffer(magicnumTar1)
	err = binary.Read(cbuf, binary.BigEndian, &c64)
	if err != nil {
		return false, fmt.Errorf("error while converting the tar magic number for comparison: %s", err)
	}
	if h64 == c64 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumTar2)
	err = binary.Read(cbuf, binary.BigEndian, &c64)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty tar magic number for comparison: %s", err)
	}
	if h64 == c64 {
		return true, nil
	}
	return false, nil
}

// IsZip checks to see if the received reader's contents are in the zip format
// by checking the magic numbers. This will match on zip, empty zip and spanned
// zip magic numbers. If you need to distinguish between those, use something
// else.
func IsZip(r io.ReaderAt) (bool, error) {
	h := make([]byte, 4)
	// Read the first 4 bytes
	_, err := r.ReadAt(h, 0)
	if err != nil {
		return false, err
	}
	var h32 uint32
	// check for Zip
	hbuf := bytes.NewReader(h)
	err = binary.Read(hbuf, binary.BigEndian, &h32)
	if err != nil {
		return false, fmt.Errorf("error while checking if input matched zip's magic number: %s", err)
	}
	var c32 uint32
	cbuf := bytes.NewBuffer(magicnumZip)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumZipEmpty)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the empty zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	cbuf = bytes.NewBuffer(magicnumZipSpanned)
	err = binary.Read(cbuf, binary.BigEndian, &c32)
	if err != nil {
		return false, fmt.Errorf("error while converting the spanned zip magic number for comparison: %s", err)
	}
	if h32 == c32 {
		return true, nil
	}
	return false, nil
}
<|endoftext|>


<commit_before>package utils

import "sync"
import "errors"
import "fmt"

type MapWithRWMutex struct {
	mapInst map[interface{}]interface{}
	sync.RWMutex
}

func NewMapWithRWMutex() *MapWithRWMutex {
	return &MapWithRWMutex{
		mapInst: map[interface{}]interface{}{},
	}
}

func (p *MapWithRWMutex) Add(key interface{}, value interface{}) (err error) {
	p.Lock()
	defer p.Unlock()
	_, ok := p.mapInst[key]
	if ok {
		err = errors.New(fmt.Sprintf("Allready has :", key))
	} else {
		p.mapInst[key] = value
	}
	return
}

func (p *MapWithRWMutex) Get(key interface{}) (value interface{}, ok bool) {
	p.RLock()
	defer p.RUnlock()
	value, ok = p.mapInst[key]
	return
}

func (p *MapWithRWMutex) PopOne() (value interface{}, ok bool) {
	p.Lock()
	defer p.Unlock()
	if len(p.mapInst) > 0 {
		for k, v := range p.mapInst {
			value = v
			delete(p.mapInst, k)
			break
		}
		ok = true
	} else {
		ok = false
	}
	return
}

func (p *MapWithRWMutex) Remove(key interface{}) (err error) {
	// fmt.Println("call Remove begin")
	p.Lock()
	// fmt.Println("call Remove middle1")
	defer p.Unlock()
	// fmt.Println("call Remove middle2")
	_, ok := p.mapInst[key] //此时不能调用p.Has函数，否则就锁重入了，会卡住
	if ok {
		// fmt.Println("call Remove middle3")
		delete(p.mapInst, key)
		// fmt.Println("call Remove middle3.1")
	} else {
		// fmt.Println("call Remove middle4")
		err = errors.New(fmt.Sprint("Do not have:", key))
	}
	//fmt.Println("call Remove end")
	return
}

func (p *MapWithRWMutex) Has(key interface{}) bool {
	p.RLock()
	defer p.RUnlock()
	_, ok := p.mapInst[key]
	return ok
}

func (p *MapWithRWMutex) Len() int {
	return int(len(p.mapInst))
}

func (p *MapWithRWMutex) Clear() {
	p.Lock()
	defer p.Unlock()
	p.mapInst = map[interface{}]interface{}{}
}

func (p *MapWithRWMutex) IsEmpty() bool {
	if p.Len() == 0 {
		return true
	}
	return false
}

func (p *MapWithRWMutex) Items() (keys []interface{}, values []interface{}) {
	p.RLock()
	defer p.RUnlock()
	for key, value := range p.mapInst {
		keys = append(keys, key)
		values = append(values, value)
	}
	return
}

func (p *MapWithRWMutex) Keys() (keys []interface{}) {
	keys, _ = p.Items()
	return
}

func (p *MapWithRWMutex) Values() (values []interface{}) {
	_, values = p.Items()
	return
}
<commit_msg>add feature set<commit_after>package utils

import "sync"
import "errors"
import "fmt"

type MapWithRWMutex struct {
	mapInst map[interface{}]interface{}
	sync.RWMutex
}

func NewMapWithRWMutex() *MapWithRWMutex {
	return &MapWithRWMutex{
		mapInst: map[interface{}]interface{}{},
	}
}

//如果key已存在，就不会设置
func (p *MapWithRWMutex) Add(key interface{}, value interface{}) (err error) {
	p.Lock()
	defer p.Unlock()
	_, ok := p.mapInst[key]
	if ok {
		err = errors.New(fmt.Sprintf("Allready has :", key))
	} else {
		p.mapInst[key] = value
	}
	return
}

//key不存在就新增，key存在就覆盖
func (p *MapWithRWMutex) Set(key interface{}, value interface{}) {
	p.Lock()
	defer p.Unlock()
	p.mapInst[key] = value
}

func (p *MapWithRWMutex) Get(key interface{}) (value interface{}, ok bool) {
	p.RLock()
	defer p.RUnlock()
	value, ok = p.mapInst[key]
	return
}

func (p *MapWithRWMutex) PopOne() (value interface{}, ok bool) {
	p.Lock()
	defer p.Unlock()
	if len(p.mapInst) > 0 {
		for k, v := range p.mapInst {
			value = v
			delete(p.mapInst, k)
			break
		}
		ok = true
	} else {
		ok = false
	}
	return
}

func (p *MapWithRWMutex) Remove(key interface{}) (err error) {
	p.Lock()
	defer p.Unlock()
	_, ok := p.mapInst[key] //此时不能调用p.Has函数，否则就锁重入了，会卡住
	if ok {
		delete(p.mapInst, key)
	} else {
		err = errors.New(fmt.Sprint("Do not have:", key))
	}
	return
}

func (p *MapWithRWMutex) Has(key interface{}) bool {
	p.RLock()
	defer p.RUnlock()
	_, ok := p.mapInst[key]
	return ok
}

func (p *MapWithRWMutex) Len() int {
	return int(len(p.mapInst))
}

func (p *MapWithRWMutex) Clear() {
	p.Lock()
	defer p.Unlock()
	p.mapInst = map[interface{}]interface{}{}
}

func (p *MapWithRWMutex) IsEmpty() bool {
	if p.Len() == 0 {
		return true
	}
	return false
}

func (p *MapWithRWMutex) Items() (keys []interface{}, values []interface{}) {
	p.RLock()
	defer p.RUnlock()
	for key, value := range p.mapInst {
		keys = append(keys, key)
		values = append(values, value)
	}
	return
}

func (p *MapWithRWMutex) Keys() (keys []interface{}) {
	keys, _ = p.Items()
	return
}

func (p *MapWithRWMutex) Values() (values []interface{}) {
	_, values = p.Items()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprintf(os.Stderr,
		`%s is a wrapper script that installs dependencies and calls the extractor.

When LGTM_SRC is not set, the script installs dependencies as described below, and then invokes the
extractor in the working directory.

If LGTM_SRC is set, it checks for the presence of the files 'go.mod', 'Gopkg.toml', and
'glide.yaml' to determine how to install dependencies: if a 'Gopkg.toml' file is present, it uses
'dep ensure', if there is a 'glide.yaml' it uses 'glide install', and otherwise 'go get'.
Additionally, unless a 'go.mod' file is detected, it sets up a temporary GOPATH and moves all
source files into a folder corresponding to the package's import path before installing
dependencies.

This behavior can be further customized using environment variables: setting LGTM_INDEX_NEED_GOPATH
to 'false' disables the GOPATH set-up, LGTM_INDEX_BUILD_COMMAND can be set to a newline-separated
list of commands to run in order to install dependencies, and LGTM_INDEX_IMPORT_PATH can be used to override the package import path, which is otherwise inferred from the SEMMLE_REPO_URL environment
variable.
`,
		os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s\n", os.Args[0])
}

func getEnvGoVersion() string {
	gover, err := exec.Command("go", "version").CombinedOutput()
	if err != nil {
		log.Fatalf("Unable to run the go command, is it installed?\nError: %s", err.Error())
	}
	return strings.Fields(string(gover))[2]
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("Unable to stat %s: %s\n", filename, err.Error())
	}
	return err == nil
}

func getImportPath() (importpath string) {
	importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
	if importpath == "" {
		repourl := os.Getenv("SEMMLE_REPO_URL")
		if repourl == "" {
			return ""
		}
		importpath = getImportPathFromRepoURL(repourl)
	}
	log.Printf("Import path is %s\n", importpath)
	return
}

func getImportPathFromRepoURL(repourl string) string {
	// check for scp-like URL as in "git@github.com:Semmle/go.git"
	shorturl := regexp.MustCompile("^([^@]+@)?([^:]+):([^/].*?)(\\.git)?$")
	m := shorturl.FindStringSubmatch(repourl)
	if m != nil {
		return m[2] + "/" + m[3]
	}

	// otherwise parse as proper URL
	u, err := url.Parse(repourl)
	if err != nil {
		log.Fatalf("Malformed repository URL %s.\n", repourl)
	}
	host := u.Hostname()
	path := u.Path
	// strip off leading slashes and trailing `.git` if present
	path = regexp.MustCompile("^/+|\\.git$").ReplaceAllString(path, "")
	return host + "/" + path
}

// DependencyInstallerMode is an enum describing how dependencies should be installed
type DependencyInstallerMode int

const (
	// GoGetNoModules represents dependency installation using `go get` without modules
	GoGetNoModules DependencyInstallerMode = iota
	// GoGetWithModules represents dependency installation using `go get` with modules
	GoGetWithModules
	// Dep represent dependency installation using `dep ensure`
	Dep
	// Glide represents dependency installation using `glide install`
	Glide
)

func main() {
	if len(os.Args) > 1 {
		usage()
		os.Exit(2)
	}

	log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), getEnvGoVersion())

	srcdir := os.Getenv("LGTM_SRC")
	inLGTM := srcdir != ""
	if inLGTM {
		log.Printf("LGTM_SRC is %s\n", srcdir)
	} else {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatalln("Failed to get current working directory.")
		}
		log.Printf("LGTM_SRC is not set; defaulting to current working directory %s\n", cwd)
		srcdir = cwd
	}

	// we set `SEMMLE_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency
	os.Setenv("SEMMLE_PATH_TRANSFORMER", "")

	// determine how to install dependencies and whether a GOPATH needs to be set up before
	// extraction
	depMode := GoGetNoModules
	needGopath := true
	if fileExists("go.mod") {
		depMode = GoGetWithModules
		needGopath = false
		log.Println("Found go.mod, enabling go modules")
	} else if fileExists("Gopkg.toml") {
		depMode = Dep
		log.Println("Found Gopkg.toml, using dep instead of go get")
	} else if fileExists("glide.yaml") {
		depMode = Glide
		log.Println("Found glide.yaml, enabling go modules")
	}

	// if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above
	if needGopathOverride := os.Getenv("LGTM_INDEX_NEED_GOPATH"); needGopathOverride != "" {
		inLGTM = true
		if needGopathOverride == "true" {
			needGopath = true
		} else if needGopathOverride == "false" {
			needGopath = false
		} else {
			log.Fatalf("Unexpected value for Boolean environment variable LGTM_NEED_GOPATH: %v.\n", needGopathOverride)
		}
	}

	importpath := getImportPath()
	if needGopath && importpath == "" {
		log.Printf("Failed to determine import path, not setting up GOPATH")
		needGopath = false
	}

	if inLGTM && needGopath {
		// a temporary directory where everything is moved while the correct
		// directory structure is created.
		scratch, err := ioutil.TempDir(srcdir, "scratch")
		if err != nil {
			log.Fatalf("Failed to create temporary directory %s in directory %s: %s\n",
				scratch, srcdir, err.Error())
		}
		log.Printf("Temporary directory is %s.\n", scratch)

		// move all files in `srcdir` to `scratch`
		dir, err := os.Open(srcdir)
		if err != nil {
			log.Fatalf("Failed to open source directory %s for reading: %s\n", srcdir, err.Error())
		}
		files, err := dir.Readdirnames(-1)
		if err != nil {
			log.Fatalf("Failed to read source directory %s: %s\n", srcdir, err.Error())
		}
		for _, file := range files {
			if file != filepath.Base(scratch) {
				log.Printf("Moving %s/%s to %s/%s.\n", srcdir, file, scratch, file)
				err := os.Rename(filepath.Join(srcdir, file), filepath.Join(scratch, file))
				if err != nil {
					log.Fatalf("Failed to move file %s to the temporary directory: %s\n", file, err.Error())
				}
			}
		}

		// create a new folder which we will add to GOPATH below
		root := filepath.Join(srcdir, "root")

		// move source files to where Go expects them to be
		newdir := filepath.Join(root, "src", importpath)
		err = os.MkdirAll(filepath.Dir(newdir), 0755)
		if err != nil {
			log.Fatalf("Failed to create directory %s: %s\n", newdir, err.Error())
		}
		log.Printf("Moving %s to %s.\n", scratch, newdir)
		err = os.Rename(scratch, newdir)
		if err != nil {
			log.Fatalf("Failed to rename %s to %s: %s\n", scratch, newdir, err.Error())
		}
		err = os.Chdir(newdir)
		if err != nil {
			log.Fatalf("Failed to chdir into %s: %s\n", newdir, err.Error())
		}

		// set up SEMMLE_PATH_TRANSFORMER to ensure paths in the source archive and the snapshot
		// match the original source location, not the location we moved it to
		pt, err := ioutil.TempFile("", "path-transformer")
		if err != nil {
			log.Fatalf("Unable to create path transformer file: %s.", err.Error())
		}
		defer os.Remove(pt.Name())
		_, err = pt.WriteString("#" + srcdir + "\n" + newdir + "//\n")
		if err != nil {
			log.Fatalf("Unable to write path transformer file: %s.", err.Error())
		}
		err = pt.Close()
		if err != nil {
			log.Fatalf("Unable to close path transformer file: %s.", err.Error())
		}
		err = os.Setenv("SEMMLE_PATH_TRANSFORMER", pt.Name())
		if err != nil {
			log.Fatalf("Unable to set SEMMLE_PATH_TRANSFORMER environment variable: %s.\n", err.Error())
		}

		// set/extend GOPATH
		oldGopath := os.Getenv("GOPATH")
		var newGopath string
		if oldGopath != "" {
			newGopath = strings.Join(
				[]string{root, oldGopath},
				string(os.PathListSeparator),
			)
		} else {
			newGopath = root
		}
		err = os.Setenv("GOPATH", newGopath)
		if err != nil {
			log.Fatalf("Unable to set GOPATH to %s: %s\n", newGopath, err.Error())
		}
		log.Printf("GOPATH set to %s.\n", newGopath)
	}

	// install dependencies
	inst := os.Getenv("LGTM_INDEX_BUILD_COMMAND")
	var install *exec.Cmd
	if inst == "" {
		// automatically determine command to install dependencies

		if depMode == Dep {
			// set up the dep cache if SEMMLE_CACHE is set
			cacheDir := os.Getenv("SEMMLE_CACHE")
			if cacheDir != "" {
				depCacheDir := filepath.Join(cacheDir, "go", "dep")
				log.Printf("Attempting to create dep cache dir %s\n", depCacheDir)
				err := os.MkdirAll(depCacheDir, 0755)
				if err != nil {
					log.Printf("Failed to create dep cache directory: %s\n", err.Error())
				} else {
					log.Printf("Setting dep cache directory to %s\n", depCacheDir)
					err = os.Setenv("DEPCACHEDIR", depCacheDir)
					if err != nil {
						log.Println("Failed to set dep cache directory")
					} else {
						err = os.Setenv("DEPCACHEAGE", "720h") // 30 days
						if err != nil {
							log.Println("Failed to set dep cache age")
						}
					}
				}
			}

			if fileExists("Gopkg.lock") {
				// if Gopkg.lock exists, don't update it and only vendor dependencies
				install = exec.Command("dep", "ensure", "-v", "-vendor-only")
			} else {
				install = exec.Command("dep", "ensure", "-v")
			}
			log.Println("Installing dependencies using `dep ensure`.")
		} else if depMode == Glide {
			install = exec.Command("glide", "install")
			log.Println("Installing dependencies using `glide install`")
		} else {
			if depMode == GoGetWithModules {
				// enable go modules if used
				os.Setenv("GO111MODULE", "on")
			}

			// get dependencies
			install = exec.Command("go", "get", "-v", "./...")
			log.Println("Installing dependencies using `go get -v ./...`.")
		}
	} else {
		// write custom build commands into a script, then run it
		var (
			ext    = ""
			header = ""
			footer = ""
		)
		if runtime.GOOS == "windows" {
			ext = ".cmd"
			header = "@echo on\n@prompt +$S\n"
			footer = "\nIF %ERRORLEVEL% NEQ 0 EXIT"
		} else {
			ext = ".sh"
			header = "#! /bin/bash\nset -xe +u\n"
		}
		script, err := ioutil.TempFile("", "go-build-command-*"+ext)
		if err != nil {
			log.Fatalf("Unable to create temporary script holding custom build commands: %s\n", err.Error())
		}
		defer os.Remove(script.Name())
		_, err = script.WriteString(header + inst + footer)
		if err != nil {
			log.Fatalf("Unable to write to temporary script holding custom build commands: %s\n", err.Error())
		}
		err = script.Close()
		if err != nil {
			log.Fatalf("Unable to close temporary script holding custom build commands: %s\n", err.Error())
		}
		os.Chmod(script.Name(), 0700)
		install = exec.Command(script.Name())
		log.Println("Installing dependencies using custom build command.")
	}

	if install != nil {
		install.Stdout = os.Stdout
		install.Stderr = os.Stderr
		err := install.Run()
		if err != nil {
			log.Printf("Installation of dependencies failed, continuing anyway: %s\n", err.Error())
		}
	}

	// extract
	mypath, err := os.Executable()
	if err != nil {
		log.Fatalf("Could not determine path of autobuilder: %v.\n", err)
	}
	extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
	if runtime.GOOS == "windows" {
		extractor = extractor + ".exe"
	}

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Unable to determine current directory: %s\n", err.Error())
	}
	log.Printf("Running extractor command '%s ./...' from directory '%s'.\n", extractor, cwd)

	cmd := exec.Command(extractor, "./...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		log.Fatalf("Extraction failed: %s\n", err.Error())
	}
}
<commit_msg>autobuilder: Add a missing newline to the usage blurb<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprintf(os.Stderr,
		`%s is a wrapper script that installs dependencies and calls the extractor.

When LGTM_SRC is not set, the script installs dependencies as described below, and then invokes the
extractor in the working directory.

If LGTM_SRC is set, it checks for the presence of the files 'go.mod', 'Gopkg.toml', and
'glide.yaml' to determine how to install dependencies: if a 'Gopkg.toml' file is present, it uses
'dep ensure', if there is a 'glide.yaml' it uses 'glide install', and otherwise 'go get'.
Additionally, unless a 'go.mod' file is detected, it sets up a temporary GOPATH and moves all
source files into a folder corresponding to the package's import path before installing
dependencies.

This behavior can be further customized using environment variables: setting LGTM_INDEX_NEED_GOPATH
to 'false' disables the GOPATH set-up, LGTM_INDEX_BUILD_COMMAND can be set to a newline-separated
list of commands to run in order to install dependencies, and LGTM_INDEX_IMPORT_PATH can be used to
override the package import path, which is otherwise inferred from the SEMMLE_REPO_URL environment
variable.
`,
		os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s\n", os.Args[0])
}

func getEnvGoVersion() string {
	gover, err := exec.Command("go", "version").CombinedOutput()
	if err != nil {
		log.Fatalf("Unable to run the go command, is it installed?\nError: %s", err.Error())
	}
	return strings.Fields(string(gover))[2]
}

func fileExists(filename string) bool {
	_, err := os.Stat(filename)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("Unable to stat %s: %s\n", filename, err.Error())
	}
	return err == nil
}

func getImportPath() (importpath string) {
	importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
	if importpath == "" {
		repourl := os.Getenv("SEMMLE_REPO_URL")
		if repourl == "" {
			return ""
		}
		importpath = getImportPathFromRepoURL(repourl)
	}
	log.Printf("Import path is %s\n", importpath)
	return
}

func getImportPathFromRepoURL(repourl string) string {
	// check for scp-like URL as in "git@github.com:Semmle/go.git"
	shorturl := regexp.MustCompile("^([^@]+@)?([^:]+):([^/].*?)(\\.git)?$")
	m := shorturl.FindStringSubmatch(repourl)
	if m != nil {
		return m[2] + "/" + m[3]
	}

	// otherwise parse as proper URL
	u, err := url.Parse(repourl)
	if err != nil {
		log.Fatalf("Malformed repository URL %s.\n", repourl)
	}
	host := u.Hostname()
	path := u.Path
	// strip off leading slashes and trailing `.git` if present
	path = regexp.MustCompile("^/+|\\.git$").ReplaceAllString(path, "")
	return host + "/" + path
}

// DependencyInstallerMode is an enum describing how dependencies should be installed
type DependencyInstallerMode int

const (
	// GoGetNoModules represents dependency installation using `go get` without modules
	GoGetNoModules DependencyInstallerMode = iota
	// GoGetWithModules represents dependency installation using `go get` with modules
	GoGetWithModules
	// Dep represent dependency installation using `dep ensure`
	Dep
	// Glide represents dependency installation using `glide install`
	Glide
)

func main() {
	if len(os.Args) > 1 {
		usage()
		os.Exit(2)
	}

	log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), getEnvGoVersion())

	srcdir := os.Getenv("LGTM_SRC")
	inLGTM := srcdir != ""
	if inLGTM {
		log.Printf("LGTM_SRC is %s\n", srcdir)
	} else {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatalln("Failed to get current working directory.")
		}
		log.Printf("LGTM_SRC is not set; defaulting to current working directory %s\n", cwd)
		srcdir = cwd
	}

	// we set `SEMMLE_PATH_TRANSFORMER` ourselves in some cases, so blank it out first for consistency
	os.Setenv("SEMMLE_PATH_TRANSFORMER", "")

	// determine how to install dependencies and whether a GOPATH needs to be set up before
	// extraction
	depMode := GoGetNoModules
	needGopath := true
	if fileExists("go.mod") {
		depMode = GoGetWithModules
		needGopath = false
		log.Println("Found go.mod, enabling go modules")
	} else if fileExists("Gopkg.toml") {
		depMode = Dep
		log.Println("Found Gopkg.toml, using dep instead of go get")
	} else if fileExists("glide.yaml") {
		depMode = Glide
		log.Println("Found glide.yaml, enabling go modules")
	}

	// if `LGTM_INDEX_NEED_GOPATH` is set, it overrides the value for `needGopath` inferred above
	if needGopathOverride := os.Getenv("LGTM_INDEX_NEED_GOPATH"); needGopathOverride != "" {
		inLGTM = true
		if needGopathOverride == "true" {
			needGopath = true
		} else if needGopathOverride == "false" {
			needGopath = false
		} else {
			log.Fatalf("Unexpected value for Boolean environment variable LGTM_NEED_GOPATH: %v.\n", needGopathOverride)
		}
	}

	importpath := getImportPath()
	if needGopath && importpath == "" {
		log.Printf("Failed to determine import path, not setting up GOPATH")
		needGopath = false
	}

	if inLGTM && needGopath {
		// a temporary directory where everything is moved while the correct
		// directory structure is created.
		scratch, err := ioutil.TempDir(srcdir, "scratch")
		if err != nil {
			log.Fatalf("Failed to create temporary directory %s in directory %s: %s\n",
				scratch, srcdir, err.Error())
		}
		log.Printf("Temporary directory is %s.\n", scratch)

		// move all files in `srcdir` to `scratch`
		dir, err := os.Open(srcdir)
		if err != nil {
			log.Fatalf("Failed to open source directory %s for reading: %s\n", srcdir, err.Error())
		}
		files, err := dir.Readdirnames(-1)
		if err != nil {
			log.Fatalf("Failed to read source directory %s: %s\n", srcdir, err.Error())
		}
		for _, file := range files {
			if file != filepath.Base(scratch) {
				log.Printf("Moving %s/%s to %s/%s.\n", srcdir, file, scratch, file)
				err := os.Rename(filepath.Join(srcdir, file), filepath.Join(scratch, file))
				if err != nil {
					log.Fatalf("Failed to move file %s to the temporary directory: %s\n", file, err.Error())
				}
			}
		}

		// create a new folder which we will add to GOPATH below
		root := filepath.Join(srcdir, "root")

		// move source files to where Go expects them to be
		newdir := filepath.Join(root, "src", importpath)
		err = os.MkdirAll(filepath.Dir(newdir), 0755)
		if err != nil {
			log.Fatalf("Failed to create directory %s: %s\n", newdir, err.Error())
		}
		log.Printf("Moving %s to %s.\n", scratch, newdir)
		err = os.Rename(scratch, newdir)
		if err != nil {
			log.Fatalf("Failed to rename %s to %s: %s\n", scratch, newdir, err.Error())
		}
		err = os.Chdir(newdir)
		if err != nil {
			log.Fatalf("Failed to chdir into %s: %s\n", newdir, err.Error())
		}

		// set up SEMMLE_PATH_TRANSFORMER to ensure paths in the source archive and the snapshot
		// match the original source location, not the location we moved it to
		pt, err := ioutil.TempFile("", "path-transformer")
		if err != nil {
			log.Fatalf("Unable to create path transformer file: %s.", err.Error())
		}
		defer os.Remove(pt.Name())
		_, err = pt.WriteString("#" + srcdir + "\n" + newdir + "//\n")
		if err != nil {
			log.Fatalf("Unable to write path transformer file: %s.", err.Error())
		}
		err = pt.Close()
		if err != nil {
			log.Fatalf("Unable to close path transformer file: %s.", err.Error())
		}
		err = os.Setenv("SEMMLE_PATH_TRANSFORMER", pt.Name())
		if err != nil {
			log.Fatalf("Unable to set SEMMLE_PATH_TRANSFORMER environment variable: %s.\n", err.Error())
		}

		// set/extend GOPATH
		oldGopath := os.Getenv("GOPATH")
		var newGopath string
		if oldGopath != "" {
			newGopath = strings.Join(
				[]string{root, oldGopath},
				string(os.PathListSeparator),
			)
		} else {
			newGopath = root
		}
		err = os.Setenv("GOPATH", newGopath)
		if err != nil {
			log.Fatalf("Unable to set GOPATH to %s: %s\n", newGopath, err.Error())
		}
		log.Printf("GOPATH set to %s.\n", newGopath)
	}

	// install dependencies
	inst := os.Getenv("LGTM_INDEX_BUILD_COMMAND")
	var install *exec.Cmd
	if inst == "" {
		// automatically determine command to install dependencies

		if depMode == Dep {
			// set up the dep cache if SEMMLE_CACHE is set
			cacheDir := os.Getenv("SEMMLE_CACHE")
			if cacheDir != "" {
				depCacheDir := filepath.Join(cacheDir, "go", "dep")
				log.Printf("Attempting to create dep cache dir %s\n", depCacheDir)
				err := os.MkdirAll(depCacheDir, 0755)
				if err != nil {
					log.Printf("Failed to create dep cache directory: %s\n", err.Error())
				} else {
					log.Printf("Setting dep cache directory to %s\n", depCacheDir)
					err = os.Setenv("DEPCACHEDIR", depCacheDir)
					if err != nil {
						log.Println("Failed to set dep cache directory")
					} else {
						err = os.Setenv("DEPCACHEAGE", "720h") // 30 days
						if err != nil {
							log.Println("Failed to set dep cache age")
						}
					}
				}
			}

			if fileExists("Gopkg.lock") {
				// if Gopkg.lock exists, don't update it and only vendor dependencies
				install = exec.Command("dep", "ensure", "-v", "-vendor-only")
			} else {
				install = exec.Command("dep", "ensure", "-v")
			}
			log.Println("Installing dependencies using `dep ensure`.")
		} else if depMode == Glide {
			install = exec.Command("glide", "install")
			log.Println("Installing dependencies using `glide install`")
		} else {
			if depMode == GoGetWithModules {
				// enable go modules if used
				os.Setenv("GO111MODULE", "on")
			}

			// get dependencies
			install = exec.Command("go", "get", "-v", "./...")
			log.Println("Installing dependencies using `go get -v ./...`.")
		}
	} else {
		// write custom build commands into a script, then run it
		var (
			ext    = ""
			header = ""
			footer = ""
		)
		if runtime.GOOS == "windows" {
			ext = ".cmd"
			header = "@echo on\n@prompt +$S\n"
			footer = "\nIF %ERRORLEVEL% NEQ 0 EXIT"
		} else {
			ext = ".sh"
			header = "#! /bin/bash\nset -xe +u\n"
		}
		script, err := ioutil.TempFile("", "go-build-command-*"+ext)
		if err != nil {
			log.Fatalf("Unable to create temporary script holding custom build commands: %s\n", err.Error())
		}
		defer os.Remove(script.Name())
		_, err = script.WriteString(header + inst + footer)
		if err != nil {
			log.Fatalf("Unable to write to temporary script holding custom build commands: %s\n", err.Error())
		}
		err = script.Close()
		if err != nil {
			log.Fatalf("Unable to close temporary script holding custom build commands: %s\n", err.Error())
		}
		os.Chmod(script.Name(), 0700)
		install = exec.Command(script.Name())
		log.Println("Installing dependencies using custom build command.")
	}

	if install != nil {
		install.Stdout = os.Stdout
		install.Stderr = os.Stderr
		err := install.Run()
		if err != nil {
			log.Printf("Installation of dependencies failed, continuing anyway: %s\n", err.Error())
		}
	}

	// extract
	mypath, err := os.Executable()
	if err != nil {
		log.Fatalf("Could not determine path of autobuilder: %v.\n", err)
	}
	extractor := filepath.Join(filepath.Dir(mypath), "go-extractor")
	if runtime.GOOS == "windows" {
		extractor = extractor + ".exe"
	}

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Unable to determine current directory: %s\n", err.Error())
	}
	log.Printf("Running extractor command '%s ./...' from directory '%s'.\n", extractor, cwd)

	cmd := exec.Command(extractor, "./...")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	if err != nil {
		log.Fatalf("Extraction failed: %s\n", err.Error())
	}
}
<|endoftext|>


<commit_before>package siacore

import (
	"bytes"
	"errors"
	"math/big"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/hash"
	"github.com/NebulousLabs/Andromeda/signatures"
)

// Though these are variables, they should never be changed during runtime.
// They get altered during testing.
var (
	BlockFrequency  = Timestamp(45)          // In seconds.
	TargetWindow    = BlockHeight(40)        // Number of blocks to use when calculating the target.
	FutureThreshold = Timestamp(3 * 60 * 60) // Seconds into the future block timestamps are valid.
	RootTarget      = Target{0, 1}
	RootDepth       = Target{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}

	MaxAdjustmentUp   = big.NewRat(103, 100)
	MaxAdjustmentDown = big.NewRat(97, 100)

	InitialCoinbase = Currency(300000)
	MinimumCoinbase = Currency(30000)

	GenesisAddress   = CoinAddress{}         // TODO: NEED TO CREATE A HARDCODED ADDRESS.
	GenesisTimestamp = Timestamp(1417070299) // Approx. 1:47pm EST Nov. 13th, 2014

	DEBUG = false
)

type (
	Timestamp   int64
	BlockHeight uint64
	Currency    uint64

	BlockID       hash.Hash
	OutputID      hash.Hash // An output id points to a specific output.
	ContractID    hash.Hash
	TransactionID hash.Hash
	CoinAddress   hash.Hash // An address is the hash of the spend conditions that unlock the output.
	Target        hash.Hash
)

// Eventually, the Block and the block header will be two separate structs.
// This will be put into practice when we implement merged mining.
type Block struct {
	ParentBlockID BlockID
	Timestamp     Timestamp
	Nonce         uint64
	MinerAddress  CoinAddress
	MerkleRoot    hash.Hash
	Transactions  []Transaction
}

// A Transaction is an update to the state of the network, can move money
// around, make contracts, etc.
type Transaction struct {
	ArbitraryData []byte
	Inputs        []Input
	MinerFees     []Currency
	Outputs       []Output
	FileContracts []FileContract
	StorageProofs []StorageProof
	Signatures    []TransactionSignature
}

// An Input contains the ID of the output it's trying to spend, and the spend
// conditions that unlock the output.
type Input struct {
	OutputID        OutputID // the source of coins for the input
	SpendConditions SpendConditions
}

// An Output contains a volume of currency and a 'CoinAddress', which is just a
// hash of the spend conditions which unlock the output.
type Output struct {
	Value     Currency // how many coins are in the output
	SpendHash CoinAddress
}

// SpendConditions is a timelock and a set of public keys that are used to
// unlock ouptuts.
type SpendConditions struct {
	TimeLock      BlockHeight
	NumSignatures uint64
	PublicKeys    []signatures.PublicKey
}

// A StorageProof contains the fields needed for a host to prove that they are
// still storing a file.
type StorageProof struct {
	ContractID ContractID
	Segment    [hash.SegmentSize]byte
	HashSet    []hash.Hash
}

// A FileContract contains the information necessary to enforce that a host
// stores a file.
type FileContract struct {
	ContractFund       Currency
	FileMerkleRoot     hash.Hash
	FileSize           uint64 // probably in bytes, which means the last element in the merkle tree may not be exactly 64 bytes.
	Start, End         BlockHeight
	ChallengeWindow    BlockHeight // size of window, one window at a time
	Tolerance          uint64      // number of missed proofs before triggering unsuccessful termination
	ValidProofPayout   Currency
	ValidProofAddress  CoinAddress
	MissedProofPayout  Currency
	MissedProofAddress CoinAddress
}

// A TransactionSignature signs a single input to a transaction to help fulfill
// the unlock conditions of the transaction. It points to an input, a
// particular public key, has a timelock, and also indicates which parts of the
// transaction have been signed.
type TransactionSignature struct {
	InputID        OutputID // the OutputID of the Input that this signature is addressing. Using the index has also been considered.
	TimeLock       BlockHeight
	CoveredFields  CoveredFields
	PublicKeyIndex uint64
	Signature      signatures.Signature
}

type CoveredFields struct {
	WholeTransaction bool
	ArbitraryData    bool
	MinerFees        []uint64 // each element indicates an index which is signed.
	Inputs           []uint64
	Outputs          []uint64
	Contracts        []uint64
	StorageProofs    []uint64
	Signatures       []uint64
}

// CalculateCoinbase takes a height and from that derives the coinbase.
func CalculateCoinbase(height BlockHeight) Currency {
	if Currency(height) >= InitialCoinbase-MinimumCoinbase {
		return MinimumCoinbase
	} else {
		return InitialCoinbase - Currency(height)
	}
}

// Int returns a Target as a big.Int.
func (t Target) Int() *big.Int {
	return new(big.Int).SetBytes(t[:])
}

// Rat returns a Target as a big.Rat.
func (t Target) Rat() *big.Rat {
	return new(big.Rat).SetInt(t.Int())
}

// Inv returns the inverse of a Target as a big.Rat
func (t Target) Inverse() *big.Rat {
	r := t.Rat()
	return r.Inv(r)
}

// IntToTarget converts a big.Int to a Target.
func IntToTarget(i *big.Int) (t Target) {
	// i may overflow the maximum target.
	// In the event of overflow, return the maximum.
	if i.BitLen() > 256 {
		return RootDepth
	}
	b := i.Bytes()
	// need to preserve big-endianness
	offset := hash.HashSize - len(b)
	copy(t[offset:], b)
	return
}

// RatToTarget converts a big.Rat to a Target.
func RatToTarget(r *big.Rat) Target {
	// convert to big.Int to truncate decimal
	i := new(big.Int).Div(r.Num(), r.Denom())
	return IntToTarget(i)
}

// Block.ID() returns a hash of the block, which is used as the block
// identifier. Transactions are not included in the hash.
func (b Block) ID() BlockID {
	return BlockID(hash.HashBytes(encoding.MarshalAll(
		b.ParentBlockID,
		b.Timestamp,
		b.Nonce,
		b.MinerAddress,
		b.MerkleRoot,
	)))
}

// CheckTarget() returns true if the block id is lower than the target.
func (b Block) CheckTarget(target Target) bool {
	blockHash := b.ID()
	return bytes.Compare(target[:], blockHash[:]) >= 0
}

// ExpectedTransactionMerkleRoot() returns the expected transaction
// merkle root of the block.
func (b Block) TransactionMerkleRoot() hash.Hash {
	var transactionHashes []hash.Hash
	for _, transaction := range b.Transactions {
		transactionHashes = append(transactionHashes, hash.HashBytes(encoding.Marshal(transaction)))
	}
	return hash.MerkleRoot(transactionHashes)
}

// SubisdyID() returns the id of the output created by the block subsidy.
func (b Block) SubsidyID() OutputID {
	bid := b.ID()
	return OutputID(hash.HashBytes(append(bid[:], "blockreward"...)))
}

// SigHash returns the hash of a transaction for a specific index.
// The index determines which TransactionSignature is included in the hash.
func (t *Transaction) SigHash(i int) hash.Hash {
	var signedData []byte
	if t.Signatures[i].CoveredFields.WholeTransaction {
		signedData = append(signedData, encoding.MarshalAll(
			t.ArbitraryData,
			t.Inputs,
			t.MinerFees,
			t.Outputs,
			t.FileContracts,
			t.StorageProofs,
			t.Signatures[i].InputID,
			t.Signatures[i].PublicKeyIndex,
			t.Signatures[i].TimeLock,
		)...)
	} else {
		if t.Signatures[i].CoveredFields.ArbitraryData {
			signedData = append(signedData, encoding.Marshal(t.ArbitraryData)...)
		}
		for _, minerFee := range t.Signatures[i].CoveredFields.MinerFees {
			signedData = append(signedData, encoding.Marshal(t.MinerFees[minerFee])...)
		}
		for _, input := range t.Signatures[i].CoveredFields.Inputs {
			signedData = append(signedData, encoding.Marshal(t.Inputs[input])...)
		}
		for _, output := range t.Signatures[i].CoveredFields.Outputs {
			signedData = append(signedData, encoding.Marshal(t.Outputs[output])...)
		}
		for _, contract := range t.Signatures[i].CoveredFields.Contracts {
			signedData = append(signedData, encoding.Marshal(t.FileContracts[contract])...)
		}
		for _, storageProof := range t.Signatures[i].CoveredFields.StorageProofs {
			signedData = append(signedData, encoding.Marshal(t.StorageProofs[storageProof])...)
		}
	}

	for _, sig := range t.Signatures[i].CoveredFields.Signatures {
		signedData = append(signedData, encoding.Marshal(t.Signatures[sig])...)
	}

	return hash.HashBytes(signedData)
}

// Transaction.OuptutID() takes the index of the output and returns the
// output's ID.
func (t Transaction) OutputID(index int) OutputID {
	return OutputID(hash.HashAll(
		encoding.Marshal(t),
		[]byte("coinsend"),
		encoding.Marshal(index),
	))
}

// SpendConditions.CoinAddress() calculates the root hash of a merkle tree of the
// SpendConditions object, using the timelock, number of signatures required,
// and each public key as leaves.
func (sc *SpendConditions) CoinAddress() CoinAddress {
	tlHash := hash.HashObject(sc.TimeLock)
	nsHash := hash.HashObject(sc.NumSignatures)
	pkHashes := make([]hash.Hash, len(sc.PublicKeys))
	for i := range sc.PublicKeys {
		pkHashes[i] = hash.HashObject(sc.PublicKeys[i])
	}
	leaves := append([]hash.Hash{tlHash, nsHash}, pkHashes...)
	return CoinAddress(hash.MerkleRoot(leaves))
}

// Transaction.fileContractID returns the id of a file contract given the index of the contract.
func (t Transaction) FileContractID(index int) ContractID {
	return ContractID(hash.HashAll(
		encoding.Marshal(t),
		[]byte("contract"),
		encoding.Marshal(index),
	))
}

// WindowIndex returns the index of the challenge window that is
// open during block height 'height'.
func (fc *FileContract) WindowIndex(height BlockHeight) (windowIndex BlockHeight, err error) {
	if height < fc.Start {
		err = errors.New("height below start point")
		return
	} else if height >= fc.End {
		err = errors.New("height above end point")
		return
	}

	windowIndex = (height - fc.Start) / fc.ChallengeWindow
	return
}

// StorageProofOutput() returns the OutputID of the output created
// during the window index that was active at height 'height'.
func (fc *FileContract) StorageProofOutputID(fcID ContractID, height BlockHeight, proofValid bool) (outputID OutputID, err error) {
	proofString := proofString(proofValid)
	windowIndex, err := fc.WindowIndex(height)
	if err != nil {
		return
	}

	outputID = OutputID(hash.HashAll(
		fcID[:],
		proofString,
		encoding.Marshal(windowIndex),
	))
	return
}

// ContractTerminationOutputID() returns the ID of a contract termination
// output, given the id of the contract and the status of the termination.
func ContractTerminationOutputID(fcID ContractID, successfulTermination bool) OutputID {
	return OutputID(hash.HashAll(
		fcID[:],
		terminationString(successfulTermination),
	))
}

// proofString() returns the string to be used when generating the output id of
// a valid proof if bool is set to true, and it returns the string to be used
// in a missed proof if the bool is set to false.
func proofString(proofValid bool) []byte {
	if proofValid {
		return []byte("validproof")
	} else {
		return []byte("missedproof")
	}
}

// terminationString() returns the string to be used when generating the output
// id of a successful terminated contract if the bool is set to true, and of an
// unsuccessful termination if the bool is set to false.
func terminationString(success bool) []byte {
	if success {
		return []byte("successfultermination")
	} else {
		return []byte("unsuccessfultermination")
	}
}
<commit_msg>new testing constants<commit_after>package siacore

import (
	"bytes"
	"errors"
	"math/big"

	"github.com/NebulousLabs/Andromeda/encoding"
	"github.com/NebulousLabs/Andromeda/hash"
	"github.com/NebulousLabs/Andromeda/signatures"
)

// Though these are variables, they should never be changed during runtime.
// They get altered during testing.
var (
	BlockFrequency  = Timestamp(10)          // In seconds.
	TargetWindow    = BlockHeight(80)        // Number of blocks to use when calculating the target.
	FutureThreshold = Timestamp(3 * 60 * 60) // Seconds into the future block timestamps are valid.
	RootTarget      = Target{0, 1}
	RootDepth       = Target{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}

	MaxAdjustmentUp   = big.NewRat(103, 100)
	MaxAdjustmentDown = big.NewRat(97, 100)

	InitialCoinbase = Currency(300000)
	MinimumCoinbase = Currency(30000)

	GenesisAddress   = CoinAddress{}         // TODO: NEED TO CREATE A HARDCODED ADDRESS.
	GenesisTimestamp = Timestamp(1417070299) // Approx. 1:47pm EST Nov. 13th, 2014

	DEBUG = false
)

type (
	Timestamp   int64
	BlockHeight uint64
	Currency    uint64

	BlockID       hash.Hash
	OutputID      hash.Hash // An output id points to a specific output.
	ContractID    hash.Hash
	TransactionID hash.Hash
	CoinAddress   hash.Hash // An address is the hash of the spend conditions that unlock the output.
	Target        hash.Hash
)

// Eventually, the Block and the block header will be two separate structs.
// This will be put into practice when we implement merged mining.
type Block struct {
	ParentBlockID BlockID
	Timestamp     Timestamp
	Nonce         uint64
	MinerAddress  CoinAddress
	MerkleRoot    hash.Hash
	Transactions  []Transaction
}

// A Transaction is an update to the state of the network, can move money
// around, make contracts, etc.
type Transaction struct {
	ArbitraryData []byte
	Inputs        []Input
	MinerFees     []Currency
	Outputs       []Output
	FileContracts []FileContract
	StorageProofs []StorageProof
	Signatures    []TransactionSignature
}

// An Input contains the ID of the output it's trying to spend, and the spend
// conditions that unlock the output.
type Input struct {
	OutputID        OutputID // the source of coins for the input
	SpendConditions SpendConditions
}

// An Output contains a volume of currency and a 'CoinAddress', which is just a
// hash of the spend conditions which unlock the output.
type Output struct {
	Value     Currency // how many coins are in the output
	SpendHash CoinAddress
}

// SpendConditions is a timelock and a set of public keys that are used to
// unlock ouptuts.
type SpendConditions struct {
	TimeLock      BlockHeight
	NumSignatures uint64
	PublicKeys    []signatures.PublicKey
}

// A StorageProof contains the fields needed for a host to prove that they are
// still storing a file.
type StorageProof struct {
	ContractID ContractID
	Segment    [hash.SegmentSize]byte
	HashSet    []hash.Hash
}

// A FileContract contains the information necessary to enforce that a host
// stores a file.
type FileContract struct {
	ContractFund       Currency
	FileMerkleRoot     hash.Hash
	FileSize           uint64 // probably in bytes, which means the last element in the merkle tree may not be exactly 64 bytes.
	Start, End         BlockHeight
	ChallengeWindow    BlockHeight // size of window, one window at a time
	Tolerance          uint64      // number of missed proofs before triggering unsuccessful termination
	ValidProofPayout   Currency
	ValidProofAddress  CoinAddress
	MissedProofPayout  Currency
	MissedProofAddress CoinAddress
}

// A TransactionSignature signs a single input to a transaction to help fulfill
// the unlock conditions of the transaction. It points to an input, a
// particular public key, has a timelock, and also indicates which parts of the
// transaction have been signed.
type TransactionSignature struct {
	InputID        OutputID // the OutputID of the Input that this signature is addressing. Using the index has also been considered.
	TimeLock       BlockHeight
	CoveredFields  CoveredFields
	PublicKeyIndex uint64
	Signature      signatures.Signature
}

type CoveredFields struct {
	WholeTransaction bool
	ArbitraryData    bool
	MinerFees        []uint64 // each element indicates an index which is signed.
	Inputs           []uint64
	Outputs          []uint64
	Contracts        []uint64
	StorageProofs    []uint64
	Signatures       []uint64
}

// CalculateCoinbase takes a height and from that derives the coinbase.
func CalculateCoinbase(height BlockHeight) Currency {
	if Currency(height) >= InitialCoinbase-MinimumCoinbase {
		return MinimumCoinbase
	} else {
		return InitialCoinbase - Currency(height)
	}
}

// Int returns a Target as a big.Int.
func (t Target) Int() *big.Int {
	return new(big.Int).SetBytes(t[:])
}

// Rat returns a Target as a big.Rat.
func (t Target) Rat() *big.Rat {
	return new(big.Rat).SetInt(t.Int())
}

// Inv returns the inverse of a Target as a big.Rat
func (t Target) Inverse() *big.Rat {
	r := t.Rat()
	return r.Inv(r)
}

// IntToTarget converts a big.Int to a Target.
func IntToTarget(i *big.Int) (t Target) {
	// i may overflow the maximum target.
	// In the event of overflow, return the maximum.
	if i.BitLen() > 256 {
		return RootDepth
	}
	b := i.Bytes()
	// need to preserve big-endianness
	offset := hash.HashSize - len(b)
	copy(t[offset:], b)
	return
}

// RatToTarget converts a big.Rat to a Target.
func RatToTarget(r *big.Rat) Target {
	// convert to big.Int to truncate decimal
	i := new(big.Int).Div(r.Num(), r.Denom())
	return IntToTarget(i)
}

// Block.ID() returns a hash of the block, which is used as the block
// identifier. Transactions are not included in the hash.
func (b Block) ID() BlockID {
	return BlockID(hash.HashBytes(encoding.MarshalAll(
		b.ParentBlockID,
		b.Timestamp,
		b.Nonce,
		b.MinerAddress,
		b.MerkleRoot,
	)))
}

// CheckTarget() returns true if the block id is lower than the target.
func (b Block) CheckTarget(target Target) bool {
	blockHash := b.ID()
	return bytes.Compare(target[:], blockHash[:]) >= 0
}

// ExpectedTransactionMerkleRoot() returns the expected transaction
// merkle root of the block.
func (b Block) TransactionMerkleRoot() hash.Hash {
	var transactionHashes []hash.Hash
	for _, transaction := range b.Transactions {
		transactionHashes = append(transactionHashes, hash.HashBytes(encoding.Marshal(transaction)))
	}
	return hash.MerkleRoot(transactionHashes)
}

// SubisdyID() returns the id of the output created by the block subsidy.
func (b Block) SubsidyID() OutputID {
	bid := b.ID()
	return OutputID(hash.HashBytes(append(bid[:], "blockreward"...)))
}

// SigHash returns the hash of a transaction for a specific index.
// The index determines which TransactionSignature is included in the hash.
func (t *Transaction) SigHash(i int) hash.Hash {
	var signedData []byte
	if t.Signatures[i].CoveredFields.WholeTransaction {
		signedData = append(signedData, encoding.MarshalAll(
			t.ArbitraryData,
			t.Inputs,
			t.MinerFees,
			t.Outputs,
			t.FileContracts,
			t.StorageProofs,
			t.Signatures[i].InputID,
			t.Signatures[i].PublicKeyIndex,
			t.Signatures[i].TimeLock,
		)...)
	} else {
		if t.Signatures[i].CoveredFields.ArbitraryData {
			signedData = append(signedData, encoding.Marshal(t.ArbitraryData)...)
		}
		for _, minerFee := range t.Signatures[i].CoveredFields.MinerFees {
			signedData = append(signedData, encoding.Marshal(t.MinerFees[minerFee])...)
		}
		for _, input := range t.Signatures[i].CoveredFields.Inputs {
			signedData = append(signedData, encoding.Marshal(t.Inputs[input])...)
		}
		for _, output := range t.Signatures[i].CoveredFields.Outputs {
			signedData = append(signedData, encoding.Marshal(t.Outputs[output])...)
		}
		for _, contract := range t.Signatures[i].CoveredFields.Contracts {
			signedData = append(signedData, encoding.Marshal(t.FileContracts[contract])...)
		}
		for _, storageProof := range t.Signatures[i].CoveredFields.StorageProofs {
			signedData = append(signedData, encoding.Marshal(t.StorageProofs[storageProof])...)
		}
	}

	for _, sig := range t.Signatures[i].CoveredFields.Signatures {
		signedData = append(signedData, encoding.Marshal(t.Signatures[sig])...)
	}

	return hash.HashBytes(signedData)
}

// Transaction.OuptutID() takes the index of the output and returns the
// output's ID.
func (t Transaction) OutputID(index int) OutputID {
	return OutputID(hash.HashAll(
		encoding.Marshal(t),
		[]byte("coinsend"),
		encoding.Marshal(index),
	))
}

// SpendConditions.CoinAddress() calculates the root hash of a merkle tree of the
// SpendConditions object, using the timelock, number of signatures required,
// and each public key as leaves.
func (sc *SpendConditions) CoinAddress() CoinAddress {
	tlHash := hash.HashObject(sc.TimeLock)
	nsHash := hash.HashObject(sc.NumSignatures)
	pkHashes := make([]hash.Hash, len(sc.PublicKeys))
	for i := range sc.PublicKeys {
		pkHashes[i] = hash.HashObject(sc.PublicKeys[i])
	}
	leaves := append([]hash.Hash{tlHash, nsHash}, pkHashes...)
	return CoinAddress(hash.MerkleRoot(leaves))
}

// Transaction.fileContractID returns the id of a file contract given the index of the contract.
func (t Transaction) FileContractID(index int) ContractID {
	return ContractID(hash.HashAll(
		encoding.Marshal(t),
		[]byte("contract"),
		encoding.Marshal(index),
	))
}

// WindowIndex returns the index of the challenge window that is
// open during block height 'height'.
func (fc *FileContract) WindowIndex(height BlockHeight) (windowIndex BlockHeight, err error) {
	if height < fc.Start {
		err = errors.New("height below start point")
		return
	} else if height >= fc.End {
		err = errors.New("height above end point")
		return
	}

	windowIndex = (height - fc.Start) / fc.ChallengeWindow
	return
}

// StorageProofOutput() returns the OutputID of the output created
// during the window index that was active at height 'height'.
func (fc *FileContract) StorageProofOutputID(fcID ContractID, height BlockHeight, proofValid bool) (outputID OutputID, err error) {
	proofString := proofString(proofValid)
	windowIndex, err := fc.WindowIndex(height)
	if err != nil {
		return
	}

	outputID = OutputID(hash.HashAll(
		fcID[:],
		proofString,
		encoding.Marshal(windowIndex),
	))
	return
}

// ContractTerminationOutputID() returns the ID of a contract termination
// output, given the id of the contract and the status of the termination.
func ContractTerminationOutputID(fcID ContractID, successfulTermination bool) OutputID {
	return OutputID(hash.HashAll(
		fcID[:],
		terminationString(successfulTermination),
	))
}

// proofString() returns the string to be used when generating the output id of
// a valid proof if bool is set to true, and it returns the string to be used
// in a missed proof if the bool is set to false.
func proofString(proofValid bool) []byte {
	if proofValid {
		return []byte("validproof")
	} else {
		return []byte("missedproof")
	}
}

// terminationString() returns the string to be used when generating the output
// id of a successful terminated contract if the bool is set to true, and of an
// unsuccessful termination if the bool is set to false.
func terminationString(success bool) []byte {
	if success {
		return []byte("successfultermination")
	} else {
		return []byte("unsuccessfultermination")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"fmt"
	"os"
	"path"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/resources"
	"sigs.k8s.io/kubetest2/pkg/exec"
	"sigs.k8s.io/yaml"
)

func (d *deployer) DumpClusterLogs() error {
	yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "toolbox-dump.yaml"))
	if err != nil {
		panic(err)
	}
	defer yamlFile.Close()

	args := []string{
		d.KopsBinaryPath, "toolbox", "dump",
		"--name", d.ClusterName,
		"--dir", d.ArtifactsDir,
		"--private-key", d.SSHPrivateKeyPath,
		"--ssh-user", d.SSHUser,
	}
	klog.Info(strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)
	cmd.SetStdout(yamlFile)
	if err := cmd.Run(); err != nil {
		return err
	}

	if err := d.dumpClusterManifest(); err != nil {
		return err
	}

	if err := d.dumpClusterInfo(); err != nil {
		return err
	}

	return nil
}

func (d *deployer) dumpClusterManifest() error {
	resourceTypes := []string{"cluster", "instancegroups"}
	for _, rt := range resourceTypes {
		yamlFile, err := os.Create(path.Join(d.ArtifactsDir, fmt.Sprintf("%v.yaml", rt)))
		if err != nil {
			panic(err)
		}
		defer yamlFile.Close()

		args := []string{
			d.KopsBinaryPath, "get", rt,
			"--name", d.ClusterName,
			"-o", "yaml",
		}
		klog.Info(strings.Join(args, " "))

		cmd := exec.Command(args[0], args[1:]...)
		cmd.SetStdout(yamlFile)
		cmd.SetEnv(d.env()...)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}

func (d *deployer) dumpClusterInfo() error {
	args := []string{
		"kubectl", "cluster-info", "dump",
		"--all-namespaces",
		"-o", "yaml",
		"--output-directory", path.Join(d.ArtifactsDir, "cluster-info"),
	}
	klog.Info(strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)
	if err := cmd.Run(); err != nil {
		if err = d.dumpClusterInfoSSH(); err != nil {
			return err
		}
	}

	resourceTypes := []string{
		"csinodes", "csidrivers", "storageclasses", "persistentvolumes",
		"mutatingwebhookconfigurations", "validatingwebhookconfigurations",
	}
	if err := os.MkdirAll(path.Join(d.ArtifactsDir, "cluster-info"), 0o755); err != nil {
		return err
	}
	for _, resType := range resourceTypes {
		yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "cluster-info", fmt.Sprintf("%v.yaml", resType)))
		if err != nil {
			return err
		}
		defer yamlFile.Close()

		args = []string{
			"kubectl", "--request-timeout", "5s", "get", resType,
			"--all-namespaces",
			"--show-managed-fields",
			"-o", "yaml",
		}
		klog.Info(strings.Join(args, " "))

		cmd := exec.Command(args[0], args[1:]...)
		cmd.SetEnv(d.env()...)
		cmd.SetStdout(yamlFile)
		if err := cmd.Run(); err != nil {
			klog.Warningf("Failed to get %v: %v", resType, err)
		}
	}

	nsCmd := exec.Command(
		"kubectl", "--request-timeout", "5s", "get", "namespaces", "--no-headers", "-o", "custom-columns=name:.metadata.name",
	)
	namespaces, err := exec.OutputLines(nsCmd)
	if err != nil {
		return fmt.Errorf("failed to get namespaces: %s", err)
	}

	namespacedResourceTypes := []string{
		"configmaps",
		"endpoints",
		"endpointslices",
		"leases",
		"persistentvolumeclaims",
		"poddisruptionbudgets",
	}
	for _, namespace := range namespaces {
		namespace = strings.TrimSpace(namespace)
		if err := os.MkdirAll(path.Join(d.ArtifactsDir, "cluster-info", namespace), 0o755); err != nil {
			return err
		}
		for _, resType := range namespacedResourceTypes {
			yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "cluster-info", namespace, fmt.Sprintf("%v.yaml", resType)))
			if err != nil {
				return err
			}
			defer yamlFile.Close()

			args = []string{
				"kubectl", "get", resType,
				"-n", namespace,
				"--show-managed-fields",
				"-o", "yaml",
			}
			klog.Info(strings.Join(args, " "))

			cmd := exec.Command(args[0], args[1:]...)
			cmd.SetEnv(d.env()...)
			cmd.SetStdout(yamlFile)
			if err := cmd.Run(); err != nil {
				if err = d.dumpClusterInfoSSH(); err != nil {
					klog.Warningf("Failed to get %v: %v", resType, err)
				}
			}
		}
	}
	return nil
}

// dumpClusterInfoSSH runs `kubectl cluster-info dump` on a control plane host via SSH
// and copies the output to the local artifacts directory.
// This can be useful when the k8s API is inaccessible from kubetest2-kops directly
func (d *deployer) dumpClusterInfoSSH() error {
	toolboxDumpArgs := []string{
		d.KopsBinaryPath, "toolbox", "dump",
		"--name", d.ClusterName,
		"--private-key", d.SSHPrivateKeyPath,
		"--ssh-user", d.SSHUser,
		"-o", "yaml",
	}
	klog.Info(strings.Join(toolboxDumpArgs, " "))

	cmd := exec.Command(toolboxDumpArgs[0], toolboxDumpArgs[1:]...)
	dumpOutput, err := exec.Output(cmd)
	if err != nil {
		return err
	}

	var dump resources.Dump
	err = yaml.Unmarshal(dumpOutput, &dump)
	if err != nil {
		return err
	}
	controlPlaneIP, controlPlaneUser, found := findControlPlaneIPUser(dump)
	if !found {
		return nil
	}

	sshURL := fmt.Sprintf("%v@%v", controlPlaneUser, controlPlaneIP)
	sshArgs := []string{
		"ssh", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		sshURL, "--",
		"kubectl", "cluster-info", "dump",
		"--all-namespaces",
		"-o", "yaml",
		"--output-directory", "/tmp/cluster-info",
	}
	klog.Info(strings.Join(sshArgs, " "))

	cmd = exec.Command(sshArgs[0], sshArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}
	scpArgs := []string{
		"scp", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null", "-r",
		fmt.Sprintf("%v:/tmp/cluster-info", sshURL),
		path.Join(d.ArtifactsDir, "cluster-info"),
	}
	klog.Info(strings.Join(scpArgs, " "))

	cmd = exec.Command(scpArgs[0], scpArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}

	rmArgs := []string{
		"ssh", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		sshURL, "--",
		"rm", "-rf", "/tmp/cluster-info",
	}
	klog.Info(strings.Join(rmArgs, " "))

	cmd = exec.Command(rmArgs[0], rmArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func findControlPlaneIPUser(dump resources.Dump) (string, string, bool) {
	for _, instance := range dump.Instances {
		if len(instance.PublicAddresses) == 0 {
			continue
		}
		for _, role := range instance.Roles {
			if role == "master" {
				return instance.PublicAddresses[0], instance.SSHUser, true
			}
		}
	}
	klog.Warning("ControlPlane instance not found from kops toolbox dump")
	return "", "", false
}

func runWithOutput(cmd exec.Cmd) error {
	exec.InheritOutput(cmd)
	return cmd.Run()
}
<commit_msg>Fix dumping of control-plane nodes<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployer

import (
	"fmt"
	"os"
	"path"
	"strings"

	"k8s.io/klog/v2"
	"k8s.io/kops/pkg/resources"
	"sigs.k8s.io/kubetest2/pkg/exec"
	"sigs.k8s.io/yaml"
)

func (d *deployer) DumpClusterLogs() error {
	yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "toolbox-dump.yaml"))
	if err != nil {
		panic(err)
	}
	defer yamlFile.Close()

	args := []string{
		d.KopsBinaryPath, "toolbox", "dump",
		"--name", d.ClusterName,
		"--dir", d.ArtifactsDir,
		"--private-key", d.SSHPrivateKeyPath,
		"--ssh-user", d.SSHUser,
	}
	klog.Info(strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)
	cmd.SetStdout(yamlFile)
	if err := cmd.Run(); err != nil {
		return err
	}

	if err := d.dumpClusterManifest(); err != nil {
		return err
	}

	if err := d.dumpClusterInfo(); err != nil {
		return err
	}

	return nil
}

func (d *deployer) dumpClusterManifest() error {
	resourceTypes := []string{"cluster", "instancegroups"}
	for _, rt := range resourceTypes {
		yamlFile, err := os.Create(path.Join(d.ArtifactsDir, fmt.Sprintf("%v.yaml", rt)))
		if err != nil {
			panic(err)
		}
		defer yamlFile.Close()

		args := []string{
			d.KopsBinaryPath, "get", rt,
			"--name", d.ClusterName,
			"-o", "yaml",
		}
		klog.Info(strings.Join(args, " "))

		cmd := exec.Command(args[0], args[1:]...)
		cmd.SetStdout(yamlFile)
		cmd.SetEnv(d.env()...)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}

func (d *deployer) dumpClusterInfo() error {
	args := []string{
		"kubectl", "cluster-info", "dump",
		"--all-namespaces",
		"-o", "yaml",
		"--output-directory", path.Join(d.ArtifactsDir, "cluster-info"),
	}
	klog.Info(strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	cmd.SetEnv(d.env()...)
	if err := cmd.Run(); err != nil {
		if err = d.dumpClusterInfoSSH(); err != nil {
			return err
		}
	}

	resourceTypes := []string{
		"csinodes", "csidrivers", "storageclasses", "persistentvolumes",
		"mutatingwebhookconfigurations", "validatingwebhookconfigurations",
	}
	if err := os.MkdirAll(path.Join(d.ArtifactsDir, "cluster-info"), 0o755); err != nil {
		return err
	}
	for _, resType := range resourceTypes {
		yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "cluster-info", fmt.Sprintf("%v.yaml", resType)))
		if err != nil {
			return err
		}
		defer yamlFile.Close()

		args = []string{
			"kubectl", "--request-timeout", "5s", "get", resType,
			"--all-namespaces",
			"--show-managed-fields",
			"-o", "yaml",
		}
		klog.Info(strings.Join(args, " "))

		cmd := exec.Command(args[0], args[1:]...)
		cmd.SetEnv(d.env()...)
		cmd.SetStdout(yamlFile)
		if err := cmd.Run(); err != nil {
			klog.Warningf("Failed to get %v: %v", resType, err)
		}
	}

	nsCmd := exec.Command(
		"kubectl", "--request-timeout", "5s", "get", "namespaces", "--no-headers", "-o", "custom-columns=name:.metadata.name",
	)
	namespaces, err := exec.OutputLines(nsCmd)
	if err != nil {
		return fmt.Errorf("failed to get namespaces: %s", err)
	}

	namespacedResourceTypes := []string{
		"configmaps",
		"endpoints",
		"endpointslices",
		"leases",
		"persistentvolumeclaims",
		"poddisruptionbudgets",
	}
	for _, namespace := range namespaces {
		namespace = strings.TrimSpace(namespace)
		if err := os.MkdirAll(path.Join(d.ArtifactsDir, "cluster-info", namespace), 0o755); err != nil {
			return err
		}
		for _, resType := range namespacedResourceTypes {
			yamlFile, err := os.Create(path.Join(d.ArtifactsDir, "cluster-info", namespace, fmt.Sprintf("%v.yaml", resType)))
			if err != nil {
				return err
			}
			defer yamlFile.Close()

			args = []string{
				"kubectl", "get", resType,
				"-n", namespace,
				"--show-managed-fields",
				"-o", "yaml",
			}
			klog.Info(strings.Join(args, " "))

			cmd := exec.Command(args[0], args[1:]...)
			cmd.SetEnv(d.env()...)
			cmd.SetStdout(yamlFile)
			if err := cmd.Run(); err != nil {
				if err = d.dumpClusterInfoSSH(); err != nil {
					klog.Warningf("Failed to get %v: %v", resType, err)
				}
			}
		}
	}
	return nil
}

// dumpClusterInfoSSH runs `kubectl cluster-info dump` on a control plane host via SSH
// and copies the output to the local artifacts directory.
// This can be useful when the k8s API is inaccessible from kubetest2-kops directly
func (d *deployer) dumpClusterInfoSSH() error {
	toolboxDumpArgs := []string{
		d.KopsBinaryPath, "toolbox", "dump",
		"--name", d.ClusterName,
		"--private-key", d.SSHPrivateKeyPath,
		"--ssh-user", d.SSHUser,
		"-o", "yaml",
	}
	klog.Info(strings.Join(toolboxDumpArgs, " "))

	cmd := exec.Command(toolboxDumpArgs[0], toolboxDumpArgs[1:]...)
	dumpOutput, err := exec.Output(cmd)
	if err != nil {
		return err
	}

	var dump resources.Dump
	err = yaml.Unmarshal(dumpOutput, &dump)
	if err != nil {
		return err
	}
	controlPlaneIP, controlPlaneUser, found := findControlPlaneIPUser(dump)
	if !found {
		return nil
	}

	sshURL := fmt.Sprintf("%v@%v", controlPlaneUser, controlPlaneIP)
	sshArgs := []string{
		"ssh", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		sshURL, "--",
		"kubectl", "cluster-info", "dump",
		"--all-namespaces",
		"-o", "yaml",
		"--output-directory", "/tmp/cluster-info",
	}
	klog.Info(strings.Join(sshArgs, " "))

	cmd = exec.Command(sshArgs[0], sshArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}
	scpArgs := []string{
		"scp", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null", "-r",
		fmt.Sprintf("%v:/tmp/cluster-info", sshURL),
		path.Join(d.ArtifactsDir, "cluster-info"),
	}
	klog.Info(strings.Join(scpArgs, " "))

	cmd = exec.Command(scpArgs[0], scpArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}

	rmArgs := []string{
		"ssh", "-i", d.SSHPrivateKeyPath,
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		sshURL, "--",
		"rm", "-rf", "/tmp/cluster-info",
	}
	klog.Info(strings.Join(rmArgs, " "))

	cmd = exec.Command(rmArgs[0], rmArgs[1:]...)
	exec.InheritOutput(cmd)

	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func findControlPlaneIPUser(dump resources.Dump) (string, string, bool) {
	for _, instance := range dump.Instances {
		if len(instance.PublicAddresses) == 0 {
			continue
		}
		for _, role := range instance.Roles {
			if role == "control-plane" {
				return instance.PublicAddresses[0], instance.SSHUser, true
			}
		}
	}
	klog.Warning("ControlPlane instance not found from kops toolbox dump")
	return "", "", false
}

func runWithOutput(cmd exec.Cmd) error {
	exec.InheritOutput(cmd)
	return cmd.Run()
}
<|endoftext|>