<commit_before>package oss

import (
	"encoding/xml"
	"time"
)

// Bucket defined bucket
type Bucket struct {
	XMLName      xml.Name `xml:"Bucket"`
	Location     string
	Name         string
	CreationDate time.Time
}

// Owner defined owner
type Owner struct {
	XMLName     xml.Name `xml:"Owner"`
	ID          string
	DisplayName string
}

// ListAllMyBucketsResult defined list all my buckets result
type ListAllMyBucketsResult struct {
	XMLName     xml.Name `xml:"ListAllMyBucketsResult"`
	Prefix      string
	Marker      string
	MaxKeys     string
	IsTruncated bool
	NextMarker  string
	Owner       Owner
	Buckets     []Bucket `xml:"Buckets>Bucket"`
}

// Content defined get bucket content
type Content struct {
	XMLName      xml.Name `xml:"Contents"`
	Key          string
	LastModified time.Time
	ETag         string
	Type         string
	Size         int
	StorageClass string
	Owner        Owner
}

// ListBucketResult defined list bucket result
type ListBucketResult struct {
	XMLName        xml.Name `xml:"ListBucketResult"`
	Contents       []Content
	CommonPrefixes string
	Delimiter      string
	IsTruncated    bool
	Marker         string
	MaxKeys        string
	Name           string
	Owner          Owner
	Prefix         string
	EncodingType   string `xml:"encoding-type"`
}

// AccessControlPolicy defined access control policy
type AccessControlPolicy struct {
	XMLName           xml.Name `xml:"AccessControlPolicy"`
	Owner             Owner
	AccessControlList []string `xml:"AccessControlList>Grant"`
}

// LocationConstraint defined location constraint
type LocationConstraint string

// BucketLoggingStatus defined bucket logging status
type BucketLoggingStatus struct {
	XMLName xml.Name `xml:"BucketLoggingStatus"`
	Bucket  string   `xml:"LoggingEnabled>TargetBucket"`
	Prefix  string   `xml:"LoggingEnabled>TargetPrefix"`
}

// WebsiteConfiguration defined website configuration
type WebsiteConfiguration struct {
	XMLName     xml.Name `xml:"WebsiteConfiguration"`
	IndexSuffix string   `xml:"IndexDocument>Suffix"`
	ErrorKey    string   `xml:"ErrorDocument>Key"`
}

// Error defined error result
type Error struct {
	XMLName    xml.Name `xml:"Error"`
	Code       string
	Message    string
	BucketName string
	RequestID  string `xml:"RequestId"`
	HostID     string `xml:"HostId"`
}

// RefererConfiguration defined referer configuration
type RefererConfiguration struct {
	XMLName           xml.Name `xml:"RefererConfiguration"`
	AllowEmptyReferer bool
	RefererList       []string `xml:"RefererList>Referer"`
}

// LifecycleRule defined lifecycle configuration rule
type LifecycleRule struct {
	XMLName        xml.Name `xml:"Rule"`
	ID             string
	Prefix         string
	Status         string
	ExpirationDays int       `xml:"Expiration>Days"`
	ExpirationDate time.Time `xml:"Expiration>Date"`
}

// LifecycleConfiguration defined lifecycle configuration
type LifecycleConfiguration struct {
	XMLName xml.Name `xml:"LifecycleConfiguration"`
	Rule    LifecycleRule
}

// CreateBucketConfiguration defined create bucket configuration
type CreateBucketConfiguration struct {
	XMLName            xml.Name `xml:"CreateBucketConfiguration"`
	LocationConstraint string
}

// CopyObjectResult defined copy object result
type CopyObjectResult struct {
	XMLName      xml.Name `xml:"CopyObjectResult"`
	LastModified time.Time
	ETag         string
}

// DeleteXML defined delete multiple objects xml
type DeleteXML struct {
	XMLName xml.Name `xml:"Delete"`
	Quiet   bool
	Objects []string `xml:"Object>Key"`
}

// DeleteResult defined delete result
type DeleteResult struct {
	XMLName xml.Name `xml:"DeleteResult"`
	Objects []string `xml:"Deleted>Key"`
}

// InitiateMultipartUploadResult defined initiate multipart upload result
type InitiateMultipartUploadResult struct {
	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
	Bucket   string
	Key      string
	UploadID string `xml:"UploadId"`
}

// CopyPartResult defined copy part result
type CopyPartResult struct {
	XMLName      xml.Name `xml:"CopyPartResult"`
	LastModified string
	ETag         string
}

// Part defined part xml
type Part struct {
	XMLName      xml.Name `xml:"Part"`
	PartNumber   int
	ETag         string
	LastModified time.Time
	Size         int
}

// CompleteMultipartUpload defined complete multipart upload xml
type CompleteMultipartUpload struct {
	XMLName xml.Name `xml:"CompleteMultipartUpload"`
	Parts   []Part
}

// CompleteMultipartUploadResult defined complete multipart upload result xml
type CompleteMultipartUploadResult struct {
	XMLName  xml.Name `xml:"CompleteMultipartUploadResult"`
	Location string
	Bucket   string
	Key      string
	ETag     string
}

// Upload defined upload xml
type Upload struct {
	XMLName   xml.Name `xml:"Upload"`
	Key       string
	UploadID  string `xml:"UploadId"`
	Initiated time.Time
}

// ListMultipartUploadsResult defined list multipart uploads result
type ListMultipartUploadsResult struct {
	XMLName            xml.Name `xml:"ListMultipartUploadsResult"`
	Bucket             string
	KeyMarker          string
	UploadIDMarker     string `xml:"UploadIdMarker"`
	NextKeyMarker      string
	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
	Delimiter          string
	Prefix             string
	MaxUploads         string
	IsTruncated        bool
	Uploads            []Upload
}

// ListPartsResult defined list parts result
type ListPartsResult struct {
	XMLName              xml.Name `xml:"ListPartsResult"`
	Bucket               string
	Key                  string
	UploadID             string `xml:"UploadId"`
	NextPartNumberMarker int
	MaxParts             int
	IsTruncated          bool
	Parts                []Part
}

// CORSRule defined cors rule
type CORSRule struct {
	XMLName       xml.Name `xml:"CORSRule"`
	AllowedOrigin []string
	AllowedMethod []string
	AllowedHeader []string
	ExposeHeader  []string
	MaxAgeSeconds int
}

// CORSConfiguration defined cors configuration
type CORSConfiguration struct {
	XMLName xml.Name `xml:"CORSConfiguration"`
	Rules   []CORSRule
}
<commit_msg>Fixed. invalid xml format<commit_after>package oss

import (
	"encoding/xml"
	"time"
)

// Bucket defined bucket
type Bucket struct {
	XMLName      xml.Name `xml:"Bucket"`
	Location     string
	Name         string
	CreationDate time.Time
}

// Owner defined owner
type Owner struct {
	XMLName     xml.Name `xml:"Owner"`
	ID          string
	DisplayName string
}

// ListAllMyBucketsResult defined list all my buckets result
type ListAllMyBucketsResult struct {
	XMLName     xml.Name `xml:"ListAllMyBucketsResult"`
	Prefix      string
	Marker      string
	MaxKeys     string
	IsTruncated bool
	NextMarker  string
	Owner       Owner
	Buckets     []Bucket `xml:"Buckets>Bucket"`
}

// Content defined get bucket content
type Content struct {
	XMLName      xml.Name `xml:"Contents"`
	Key          string
	LastModified time.Time
	ETag         string
	Type         string
	Size         int
	StorageClass string
	Owner        Owner
}

// ListBucketResult defined list bucket result
type ListBucketResult struct {
	XMLName        xml.Name `xml:"ListBucketResult"`
	Contents       []Content
	CommonPrefixes string
	Delimiter      string
	IsTruncated    bool
	Marker         string
	MaxKeys        string
	Name           string
	Owner          Owner
	Prefix         string
	EncodingType   string `xml:"encoding-type"`
}

// AccessControlPolicy defined access control policy
type AccessControlPolicy struct {
	XMLName           xml.Name `xml:"AccessControlPolicy"`
	Owner             Owner
	AccessControlList []string `xml:"AccessControlList>Grant"`
}

// LocationConstraint defined location constraint
type LocationConstraint string

// BucketLoggingStatus defined bucket logging status
type BucketLoggingStatus struct {
	XMLName xml.Name `xml:"BucketLoggingStatus"`
	Bucket  string   `xml:"LoggingEnabled>TargetBucket"`
	Prefix  string   `xml:"LoggingEnabled>TargetPrefix"`
}

// WebsiteConfiguration defined website configuration
type WebsiteConfiguration struct {
	XMLName     xml.Name `xml:"WebsiteConfiguration"`
	IndexSuffix string   `xml:"IndexDocument>Suffix"`
	ErrorKey    string   `xml:"ErrorDocument>Key"`
}

// Error defined error result
type Error struct {
	XMLName    xml.Name `xml:"Error"`
	Code       string
	Message    string
	BucketName string
	RequestID  string `xml:"RequestId"`
	HostID     string `xml:"HostId"`
}

// RefererConfiguration defined referer configuration
type RefererConfiguration struct {
	XMLName           xml.Name `xml:"RefererConfiguration"`
	AllowEmptyReferer bool
	RefererList       []string `xml:"RefererList>Referer"`
}

// LifecycleRule defined lifecycle configuration rule
type LifecycleRule struct {
	XMLName        xml.Name `xml:"Rule"`
	ID             string
	Prefix         string
	Status         string
	ExpirationDays int `xml:"Expiration>Days"`
}

// LifecycleConfiguration defined lifecycle configuration
type LifecycleConfiguration struct {
	XMLName xml.Name `xml:"LifecycleConfiguration"`
	Rule    LifecycleRule
}

// CreateBucketConfiguration defined create bucket configuration
type CreateBucketConfiguration struct {
	XMLName            xml.Name `xml:"CreateBucketConfiguration"`
	LocationConstraint string
}

// CopyObjectResult defined copy object result
type CopyObjectResult struct {
	XMLName      xml.Name `xml:"CopyObjectResult"`
	LastModified time.Time
	ETag         string
}

// ObjectKey defined delete multiple objects object key
type ObjectKey struct {
	XMLName xml.Name `xml:"Object"`
	Key     string
}

// DeleteXML defined delete multiple objects xml
type DeleteXML struct {
	XMLName xml.Name `xml:"Delete"`
	Quiet   bool
	Objects []ObjectKey `xml:"--"`
}

// DeleteResult defined delete result
type DeleteResult struct {
	XMLName xml.Name `xml:"DeleteResult"`
	Objects []string `xml:"Deleted>Key"`
}

// InitiateMultipartUploadResult defined initiate multipart upload result
type InitiateMultipartUploadResult struct {
	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
	Bucket   string
	Key      string
	UploadID string `xml:"UploadId"`
}

// CopyPartResult defined copy part result
type CopyPartResult struct {
	XMLName      xml.Name `xml:"CopyPartResult"`
	LastModified string
	ETag         string
}

// Part defined part xml
type Part struct {
	XMLName      xml.Name `xml:"Part"`
	PartNumber   int
	ETag         string
	LastModified time.Time
	Size         int
}

// CompleteMultipartUpload defined complete multipart upload xml
type CompleteMultipartUpload struct {
	XMLName xml.Name `xml:"CompleteMultipartUpload"`
	Parts   []Part
}

// CompleteMultipartUploadResult defined complete multipart upload result xml
type CompleteMultipartUploadResult struct {
	XMLName  xml.Name `xml:"CompleteMultipartUploadResult"`
	Location string
	Bucket   string
	Key      string
	ETag     string
}

// Upload defined upload xml
type Upload struct {
	XMLName   xml.Name `xml:"Upload"`
	Key       string
	UploadID  string `xml:"UploadId"`
	Initiated time.Time
}

// ListMultipartUploadsResult defined list multipart uploads result
type ListMultipartUploadsResult struct {
	XMLName            xml.Name `xml:"ListMultipartUploadsResult"`
	Bucket             string
	KeyMarker          string
	UploadIDMarker     string `xml:"UploadIdMarker"`
	NextKeyMarker      string
	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
	Delimiter          string
	Prefix             string
	MaxUploads         string
	IsTruncated        bool
	Uploads            []Upload
}

// ListPartsResult defined list parts result
type ListPartsResult struct {
	XMLName              xml.Name `xml:"ListPartsResult"`
	Bucket               string
	Key                  string
	UploadID             string `xml:"UploadId"`
	NextPartNumberMarker int
	MaxParts             int
	IsTruncated          bool
	Parts                []Part
}

// CORSRule defined cors rule
type CORSRule struct {
	XMLName       xml.Name `xml:"CORSRule"`
	AllowedOrigin []string
	AllowedMethod []string
	AllowedHeader []string
	ExposeHeader  []string
	MaxAgeSeconds int
}

// CORSConfiguration defined cors configuration
type CORSConfiguration struct {
	XMLName xml.Name `xml:"CORSConfiguration"`
	Rules   []CORSRule
}
<|endoftext|>


<commit_before>// +build windows
package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/microsoft/hcsshim"
)

func folderexists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return true
}

func main() {
	var folder string
	flag.StringVar(&folder, "folder", "", "Folder to zap.")
	flag.Parse()
	if folder == "" {
		fmt.Println("Error: folder must be supplied")
		return
	}
	if folderexists(folder) {
		location, foldername := filepath.Split(folder)
		info := hcsshim.DriverInfo{
			HomeDir: location,
			Flavour: 0,
		}
		if err := hcsshim.DestroyLayer(info, foldername); err != nil {
			fmt.Println("ERROR: ", err)
		} else {
			fmt.Println("INFO: Zapped successfully")
		}
	} else {
		fmt.Println("ERROR: Folder does not exist")
	}
}
<commit_msg>fix import collision error<commit_after>// +build windows
package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/Microsoft/hcsshim"
)

func folderexists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return true
}

func main() {
	var folder string
	flag.StringVar(&folder, "folder", "", "Folder to zap.")
	flag.Parse()
	if folder == "" {
		fmt.Println("Error: folder must be supplied")
		return
	}
	if folderexists(folder) {
		location, foldername := filepath.Split(folder)
		info := hcsshim.DriverInfo{
			HomeDir: location,
			Flavour: 0,
		}
		if err := hcsshim.DestroyLayer(info, foldername); err != nil {
			fmt.Println("ERROR: ", err)
		} else {
			fmt.Println("INFO: Zapped successfully")
		}
	} else {
		fmt.Println("ERROR: Folder does not exist")
	}
}
<|endoftext|>


<commit_before>package zim

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"syscall"
)

const (
	ZIM = 72173914
)

type ZimReader struct {
	f             *os.File
	mmap          [][]byte
	ArticleCount  uint32
	clusterCount  uint32
	urlPtrPos     uint64
	titlePtrPos   uint64
	clusterPtrPos uint64
	mimeListPos   uint64
	mainPage      uint32
	layoutPage    uint32
	mimeTypeList  []string
}

// create a new zim reader
func NewReader(path string) (*ZimReader, error) {
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	z := ZimReader{f: f, mainPage: 0xffffffff, layoutPage: 0xffffffff}

	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	if uint64(fi.Size()) < (1<<31) || runtime.GOARCH == "amd64" {
		fmt.Println("Using 64 bits addressing")

		mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
		if err != nil {
			panic(err)
		}
		z.mmap = make([][]byte, 1)
		z.mmap[0] = mmap
	} else {
		// 32-bit architecture such as Intel's IA-32 can only directly address 4 GiB or smaller portions
		// of files. An even smaller amount of addressible space is available to individual programs
		// typically in the range of 2 to 3 GiB, depending on the operating system kernel.
		fmt.Println("Using 32 bits addressing")

		splits := int64(fi.Size() / (1 << 31))
		var i int64
		for i = 0; i <= splits; i++ {
			mmap, err := syscall.Mmap(int(f.Fd()), i*(1<<31), 1<<31, syscall.PROT_READ, syscall.MAP_SHARED)
			if err != nil {
				panic(err)
			}
			z.mmap = append(z.mmap, mmap)
		}

	}

	err = z.readFileHeaders()
	return &z, err
}

func (z *ZimReader) getBytesRangeAt(start, end uint64) []byte {
	if len(z.mmap) == 1 {
		return z.mmap[0][start:end]
	}
	fns := start / (1 << 31)
	fne := end / (1 << 31)
	if fns == fne {
		return z.mmap[fns][start%(1<<31) : end%(1<<31)]
	} else {
		//TODO: end is on another segment
		fmt.Println("read end on over file not implemented yet")
	}

	return nil
}

func (z *ZimReader) getByteAt(offset uint64) byte {
	if len(z.mmap) == 1 {
		return z.mmap[0][offset]
	}

	fn := offset / (1 << 31)
	return z.mmap[fn][offset%(1<<31)]
}

// populate the ZimReader structs with headers
// It is decided to panic on corrupted zim file
func (z *ZimReader) readFileHeaders() error {
	// checking for file type
	v, err := readInt32(z.getBytesRangeAt(0, 0+4))
	if err != nil {
		panic(err)
	}
	if v != ZIM {
		return errors.New("Not a ZIM file")
	}

	// checking for version
	v, err = readInt32(z.getBytesRangeAt(4, 4+4))
	if err != nil {
		panic(err)
	}
	if v != 5 {
		return errors.New("Unsupported version 5 only")
	}

	// checking for articles count
	v, err = readInt32(z.getBytesRangeAt(24, 24+4))
	if err != nil {
		panic(err)
	}
	z.ArticleCount = v

	// checking for cluster count
	v, err = readInt32(z.getBytesRangeAt(28, 28+4))
	if err != nil {
		panic(err)
	}
	z.clusterCount = v

	// checking for urlPtrPos
	vb, err := readInt64(z.getBytesRangeAt(32, 32+8))
	if err != nil {
		panic(err)
	}
	z.urlPtrPos = vb

	// checking for titlePtrPos
	vb, err = readInt64(z.getBytesRangeAt(40, 40+8))
	if err != nil {
		panic(err)
	}
	z.titlePtrPos = vb

	// checking for clusterPtrPos
	vb, err = readInt64(z.getBytesRangeAt(48, 48+8))
	if err != nil {
		panic(err)
	}
	z.clusterPtrPos = vb

	// checking for mimeListPos
	vb, err = readInt64(z.getBytesRangeAt(56, 56+8))
	if err != nil {
		panic(err)
	}
	z.mimeListPos = vb

	// checking for mainPage
	v, err = readInt32(z.getBytesRangeAt(64, 64+4))
	if err != nil {
		panic(err)
	}
	z.mainPage = v

	// checking for layoutPage
	v, err = readInt32(z.getBytesRangeAt(68, 68+4))
	if err != nil {
		panic(err)
	}
	z.layoutPage = v

	z.MimeTypes()
	return err
}

// Return an ordered list of mime types present in the ZIM file
func (z *ZimReader) MimeTypes() []string {
	if len(z.mimeTypeList) != 0 {
		return z.mimeTypeList
	}

	var s []string
	// assume mime list fit in 2k
	b := bytes.NewBuffer(z.getBytesRangeAt(z.mimeListPos, z.mimeListPos+2048))

	for {
		line, err := b.ReadBytes('\x00')
		if err != nil && err != io.EOF {
			panic(err)
		}
		// a line of 1 is a line containing only \x00 and it's the marker for the
		// end of mime types list
		if len(line) == 1 {
			break
		}
		s = append(s, strings.TrimRight(string(line), "\x00"))
	}
	z.mimeTypeList = s
	return s
}

// list all urls contained in a zim file
// note that this is a slow implementation, a real iterator is faster
// but you are not suppose to use this method on big zim files, use indexes
func (z *ZimReader) ListArticles() <-chan *Article {
	ch := make(chan *Article, 10)

	go func() {
		var idx uint32
		// starting at 1 to avoid "con" entry
		var start uint32 = 1

		for idx = start; idx < z.ArticleCount; idx++ {
			offset := z.GetUrlOffsetAtIdx(idx)
			art := z.getArticleAt(offset)
			if art == nil {
				//TODO: deal with redirect continue
			}
			ch <- art
		}
		close(ch)
	}()
	return ch
}

// get the offset in the zim pointing to URL at index pos
func (z *ZimReader) GetUrlOffsetAtIdx(idx uint32) uint64 {
	offset := z.urlPtrPos + uint64(idx)*8
	v, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	return v
}

// return the start and end offsets for cluster idx
func (z *ZimReader) getClusterOffsetsAtIdx(idx uint32) (start, end uint64) {
	offset := z.clusterPtrPos + (uint64(idx) * 8)
	start, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	offset = z.clusterPtrPos + (uint64(idx+1) * 8)
	end, err = readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	end -= 1
	return
}

// return the article at the exact url not using any index this is really slow on big ZIM
func (z *ZimReader) GetPageNoIndex(url string) *Article {
	var idx uint32
	// starting at 1 to avoid "con" entry
	var start uint32 = 1

	art := new(Article)
	for idx = start; idx < z.ArticleCount; idx++ {
		offset := z.GetUrlOffsetAtIdx(idx)
		art = z.FillArticleAt(art, offset)
		if art.FullURL() == url {
			return art
		}
	}
	return nil
}

// return the article main page if it exists
func (z *ZimReader) GetMainPage() *Article {
	if z.mainPage == 0xffffffff {
		return nil
	}
	return z.getArticleAt(z.GetUrlOffsetAtIdx(z.mainPage))
}

func (z *ZimReader) Close() {
	for _, m := range z.mmap {
		syscall.Munmap(m)
	}
	z.f.Close()
}

func (z *ZimReader) String() string {
	fi, err := z.f.Stat()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("Size: %d, ArticleCount: %d urlPtrPos: 0x%x titlePtrPos: 0x%x mimeListPos: 0x%x clusterPtrPos: 0x%x\nMimeTypes: %v",
		fi.Size(), z.ArticleCount, z.urlPtrPos, z.titlePtrPos, z.mimeListPos, z.clusterPtrPos, z.MimeTypes())
}
<commit_msg>use a constant chunksize (1<<31) -1<commit_after>package zim

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"syscall"
)

const (
	ZIM       = 72173914
	chunkSize = (1 << 31) - 1
)

type ZimReader struct {
	f             *os.File
	mmap          [][]byte
	ArticleCount  uint32
	clusterCount  uint32
	urlPtrPos     uint64
	titlePtrPos   uint64
	clusterPtrPos uint64
	mimeListPos   uint64
	mainPage      uint32
	layoutPage    uint32
	mimeTypeList  []string
}

// create a new zim reader
func NewReader(path string) (*ZimReader, error) {
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	z := ZimReader{f: f, mainPage: 0xffffffff, layoutPage: 0xffffffff}

	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	if uint64(fi.Size()) < (1<<31) || runtime.GOARCH == "amd64" {
		fmt.Println("Using 64 bits addressing")

		mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
		if err != nil {
			panic(err)
		}
		z.mmap = make([][]byte, 1)
		z.mmap[0] = mmap
	} else {
		// 32-bit architecture such as Intel's IA-32 can only directly address 4 GiB or smaller portions
		// of files. An even smaller amount of addressible space is available to individual programs
		// typically in the range of 2 to 3 GiB, depending on the operating system kernel.
		fmt.Println("Using 32 bits addressing")

		splits := int64(fi.Size() / chunkSize)
		var i int64
		for i = 0; i <= splits; i++ {
			mmap, err := syscall.Mmap(int(f.Fd()), i*chunkSize, chunkSize, syscall.PROT_READ, syscall.MAP_SHARED)
			if err != nil {
				panic(err)
			}
			z.mmap = append(z.mmap, mmap)
		}

	}

	err = z.readFileHeaders()
	return &z, err
}

func (z *ZimReader) getBytesRangeAt(start, end uint64) []byte {
	if len(z.mmap) == 1 {
		return z.mmap[0][start:end]
	}
	fns := start / chunkSize
	fne := end / chunkSize
	if fns == fne {
		return z.mmap[fns][start%chunkSize : end%chunkSize]
	} else {
		//TODO: end is on another segment
		fmt.Println("read end on over file not implemented yet")
	}

	return nil
}

func (z *ZimReader) getByteAt(offset uint64) byte {
	if len(z.mmap) == 1 {
		return z.mmap[0][offset]
	}

	fn := offset / chunkSize
	return z.mmap[fn][offset%chunkSize]
}

// populate the ZimReader structs with headers
// It is decided to panic on corrupted zim file
func (z *ZimReader) readFileHeaders() error {
	// checking for file type
	v, err := readInt32(z.getBytesRangeAt(0, 0+4))
	if err != nil {
		panic(err)
	}
	if v != ZIM {
		return errors.New("Not a ZIM file")
	}

	// checking for version
	v, err = readInt32(z.getBytesRangeAt(4, 4+4))
	if err != nil {
		panic(err)
	}
	if v != 5 {
		return errors.New("Unsupported version 5 only")
	}

	// checking for articles count
	v, err = readInt32(z.getBytesRangeAt(24, 24+4))
	if err != nil {
		panic(err)
	}
	z.ArticleCount = v

	// checking for cluster count
	v, err = readInt32(z.getBytesRangeAt(28, 28+4))
	if err != nil {
		panic(err)
	}
	z.clusterCount = v

	// checking for urlPtrPos
	vb, err := readInt64(z.getBytesRangeAt(32, 32+8))
	if err != nil {
		panic(err)
	}
	z.urlPtrPos = vb

	// checking for titlePtrPos
	vb, err = readInt64(z.getBytesRangeAt(40, 40+8))
	if err != nil {
		panic(err)
	}
	z.titlePtrPos = vb

	// checking for clusterPtrPos
	vb, err = readInt64(z.getBytesRangeAt(48, 48+8))
	if err != nil {
		panic(err)
	}
	z.clusterPtrPos = vb

	// checking for mimeListPos
	vb, err = readInt64(z.getBytesRangeAt(56, 56+8))
	if err != nil {
		panic(err)
	}
	z.mimeListPos = vb

	// checking for mainPage
	v, err = readInt32(z.getBytesRangeAt(64, 64+4))
	if err != nil {
		panic(err)
	}
	z.mainPage = v

	// checking for layoutPage
	v, err = readInt32(z.getBytesRangeAt(68, 68+4))
	if err != nil {
		panic(err)
	}
	z.layoutPage = v

	z.MimeTypes()
	return err
}

// Return an ordered list of mime types present in the ZIM file
func (z *ZimReader) MimeTypes() []string {
	if len(z.mimeTypeList) != 0 {
		return z.mimeTypeList
	}

	var s []string
	// assume mime list fit in 2k
	b := bytes.NewBuffer(z.getBytesRangeAt(z.mimeListPos, z.mimeListPos+2048))

	for {
		line, err := b.ReadBytes('\x00')
		if err != nil && err != io.EOF {
			panic(err)
		}
		// a line of 1 is a line containing only \x00 and it's the marker for the
		// end of mime types list
		if len(line) == 1 {
			break
		}
		s = append(s, strings.TrimRight(string(line), "\x00"))
	}
	z.mimeTypeList = s
	return s
}

// list all urls contained in a zim file
// note that this is a slow implementation, a real iterator is faster
// but you are not suppose to use this method on big zim files, use indexes
func (z *ZimReader) ListArticles() <-chan *Article {
	ch := make(chan *Article, 10)

	go func() {
		var idx uint32
		// starting at 1 to avoid "con" entry
		var start uint32 = 1

		for idx = start; idx < z.ArticleCount; idx++ {
			offset := z.GetUrlOffsetAtIdx(idx)
			art := z.getArticleAt(offset)
			if art == nil {
				//TODO: deal with redirect continue
			}
			ch <- art
		}
		close(ch)
	}()
	return ch
}

// get the offset in the zim pointing to URL at index pos
func (z *ZimReader) GetUrlOffsetAtIdx(idx uint32) uint64 {
	offset := z.urlPtrPos + uint64(idx)*8
	v, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	return v
}

// return the start and end offsets for cluster idx
func (z *ZimReader) getClusterOffsetsAtIdx(idx uint32) (start, end uint64) {
	offset := z.clusterPtrPos + (uint64(idx) * 8)
	start, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	offset = z.clusterPtrPos + (uint64(idx+1) * 8)
	end, err = readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	end -= 1
	return
}

// return the article at the exact url not using any index this is really slow on big ZIM
func (z *ZimReader) GetPageNoIndex(url string) *Article {
	var idx uint32
	// starting at 1 to avoid "con" entry
	var start uint32 = 1

	art := new(Article)
	for idx = start; idx < z.ArticleCount; idx++ {
		offset := z.GetUrlOffsetAtIdx(idx)
		art = z.FillArticleAt(art, offset)
		if art.FullURL() == url {
			return art
		}
	}
	return nil
}

// return the article main page if it exists
func (z *ZimReader) GetMainPage() *Article {
	if z.mainPage == 0xffffffff {
		return nil
	}
	return z.getArticleAt(z.GetUrlOffsetAtIdx(z.mainPage))
}

func (z *ZimReader) Close() {
	for _, m := range z.mmap {
		syscall.Munmap(m)
	}
	z.f.Close()
}

func (z *ZimReader) String() string {
	fi, err := z.f.Stat()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("Size: %d, ArticleCount: %d urlPtrPos: 0x%x titlePtrPos: 0x%x mimeListPos: 0x%x clusterPtrPos: 0x%x\nMimeTypes: %v",
		fi.Size(), z.ArticleCount, z.urlPtrPos, z.titlePtrPos, z.mimeListPos, z.clusterPtrPos, z.MimeTypes())
}
<|endoftext|>


<commit_before>package tty

import (
	"log"
	"os"
	"syscall"
	"unsafe"
)

type termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type Tty struct {
	original *termios
}

func tcSetAttr(fd uintptr, termios *termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func tcGetAttr(fd uintptr) *termios {
	var termios = &termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func (t *Tty) EnableRawMode() {
	t.original = tcGetAttr(os.Stdin.Fd())
	var raw termios
	raw = *t.original
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := tcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func (t *Tty) DisableRawMode() {
	if e := tcSetAttr(os.Stdin.Fd(), t.original); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}
<commit_msg>Add comments to exported struct and functions, as golint says to.<commit_after>package tty

import (
	"log"
	"os"
	"syscall"
	"unsafe"
)

type termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

// Ttty covers the unexported real tty info in
// a termios struct.
type Tty struct {
	original *termios
}

func tcSetAttr(fd uintptr, termios *termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func tcGetAttr(fd uintptr) *termios {
	var termios = &termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

// EnableRawMode changes tty settings so that the program can control
// cursor position etc, and so that the program only blocks for a short
// while when reading bytes from stdin.
func (t *Tty) EnableRawMode() {
	t.original = tcGetAttr(os.Stdin.Fd())
	var raw termios
	raw = *t.original
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := tcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

// DisableRawMode resets tty termios attributes to what they
// were originally.
func (t *Tty) DisableRawMode() {
	if e := tcSetAttr(os.Stdin.Fd(), t.original); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}
<|endoftext|>


<commit_before>package ssdb

import (
	"encoding/binary"
	"fmt"
	//	"strconv"
)

var (
	SEQ_SIZE      = 8
	DATATYPE_SIZE = 1
	CMD_SIZE      = 1
)

type Binlog struct {
	seq      uint64
	datatype uint8
	cmdtype  uint8
	cmd      []string
	body     [][]byte
}

func LoadBinlog(bytes [][]byte) (binlog *Binlog, err error) {
	binlog = &Binlog{}
	OFFSET := 0
	binlog.seq = binary.LittleEndian.Uint64(bytes[0][OFFSET:SEQ_SIZE])
	OFFSET += SEQ_SIZE
	binlog.datatype = (uint8)(bytes[0][OFFSET])
	OFFSET += DATATYPE_SIZE
	binlog.cmdtype = (uint8)(bytes[0][OFFSET])
	OFFSET += CMD_SIZE
	body := bytes[0][OFFSET:]
	binlog.body = bytes

	if len(body) > 1 {
		offset := 0
		cmdtag := body[offset]
		offset += 1 //first byte is cmd tag,second byte is split byte
		switch binlog.cmdtype {
		case BINLOGCOMMAND_KSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_KV {
				break
			}
			binlog.cmd = []string{
				"set",
				string(body[offset:]),
				string(bytes[1]),
			}
		case BINLOGCOMMAND_KDEL:
			if cmdtag != DATATYPE_KV {
				break
			}
			binlog.cmd = []string{
				"del",
				string(body[offset]),
			}
		case BINLOGCOMMAND_HSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_HASH {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			val := string(bytes[1])
			binlog.cmd = []string{
				"hset",
				name,
				key,
				val,
			}
		case BINLOGCOMMAND_HDEL:
			if cmdtag != DATATYPE_HASH {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			binlog.cmd = []string{
				"hdel",
				name,
				key,
			}
		case BINLOGCOMMAND_ZSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_ZSET {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			sign := ""
			if body[offset] == '-' {
				sign = "-"
			}
			offset += 1
			key := string(body[offset:])
			val := sign + string(bytes[1]) //add number's sign
			if name != SSDB_EXPIRATION_LIST_KEY {
				binlog.cmd = []string{
					"zadd",
					name,
					val,
					key,
				}
			} else {
				// set ttl key
				binlog.cmd = []string{
					"expireat",
					key,
					val[0:10],
				}
			}
		case BINLOGCOMMAND_ZDEL:
			if cmdtag != DATATYPE_ZSET {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			if name != SSDB_EXPIRATION_LIST_KEY {
				binlog.cmd = []string{
					"zrem",
					name,
					key,
				}
			}
		case BINLOGCOMMAND_QSET, BINLOGCOMMAND_QPUSH_BACK, BINLOGCOMMAND_QPUSH_FRONT:
			if len(bytes) != 2 {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			seq := binary.BigEndian.Uint64(body[offset:])
			if seq < SSDB_QITEM_MIN_SEQ || seq > SSDB_QITEM_MAX_SEQ {
				break
			}
			val := string(bytes[1]) //list val
			switch binlog.cmdtype{
			case BINLOGCOMMAND_QPUSH_BACK:
				binlog.cmd = []string{
					"rpush",
					name,
					val,
				}
			case BINLOGCOMMAND_QPUSH_FRONT:
				binlog.cmd = []string{
					"lpush",
					name,
					val,
				}
			case BINLOGCOMMAND_QSET:
				fmt.Println("unsuported qset binlog:", name, seq, val)
			default:
				fmt.Println("unknown qet/push binlog:", binlog)
			}
		case BINLOGCOMMAND_QPOP_BACK,BINLOGCOMMAND_QPOP_FRONT:
			name := string(body[:])
			switch binlog.cmdtype{
			case BINLOGCOMMAND_QPOP_BACK:
				binlog.cmd = []string{
					"rpop",
					name,
				}
			case BINLOGCOMMAND_QPOP_FRONT:
				binlog.cmd = []string{
					"lpop",
					name,
				}
			default:
				fmt.Println("unknown pop binlog:", binlog)
			}
		default:
			fmt.Println("unknown binlog:", binlog)
		}

	}
	return
}
<commit_msg>fix: miss ':'<commit_after>package ssdb

import (
	"encoding/binary"
	"fmt"
	//	"strconv"
)

var (
	SEQ_SIZE      = 8
	DATATYPE_SIZE = 1
	CMD_SIZE      = 1
)

type Binlog struct {
	seq      uint64
	datatype uint8
	cmdtype  uint8
	cmd      []string
	body     [][]byte
}

func LoadBinlog(bytes [][]byte) (binlog *Binlog, err error) {
	binlog = &Binlog{}
	OFFSET := 0
	binlog.seq = binary.LittleEndian.Uint64(bytes[0][OFFSET:SEQ_SIZE])
	OFFSET += SEQ_SIZE
	binlog.datatype = (uint8)(bytes[0][OFFSET])
	OFFSET += DATATYPE_SIZE
	binlog.cmdtype = (uint8)(bytes[0][OFFSET])
	OFFSET += CMD_SIZE
	body := bytes[0][OFFSET:]
	binlog.body = bytes

	if len(body) > 1 {
		offset := 0
		cmdtag := body[offset]
		offset += 1 //first byte is cmd tag,second byte is split byte
		switch binlog.cmdtype {
		case BINLOGCOMMAND_KSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_KV {
				break
			}
			binlog.cmd = []string{
				"set",
				string(body[offset:]),
				string(bytes[1]),
			}
		case BINLOGCOMMAND_KDEL:
			if cmdtag != DATATYPE_KV {
				break
			}
			binlog.cmd = []string{
				"del",
				string(body[offset:]),
			}
		case BINLOGCOMMAND_HSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_HASH {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			val := string(bytes[1])
			binlog.cmd = []string{
				"hset",
				name,
				key,
				val,
			}
		case BINLOGCOMMAND_HDEL:
			if cmdtag != DATATYPE_HASH {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			binlog.cmd = []string{
				"hdel",
				name,
				key,
			}
		case BINLOGCOMMAND_ZSET:
			if len(bytes) != 2 {
				break
			}
			if cmdtag != DATATYPE_ZSET {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			sign := ""
			if body[offset] == '-' {
				sign = "-"
			}
			offset += 1
			key := string(body[offset:])
			val := sign + string(bytes[1]) //add number's sign
			if name != SSDB_EXPIRATION_LIST_KEY {
				binlog.cmd = []string{
					"zadd",
					name,
					val,
					key,
				}
			} else {
				// set ttl key
				binlog.cmd = []string{
					"expireat",
					key,
					val[0:10],
				}
			}
		case BINLOGCOMMAND_ZDEL:
			if cmdtag != DATATYPE_ZSET {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			offset += 1
			key := string(body[offset:])
			if name != SSDB_EXPIRATION_LIST_KEY {
				binlog.cmd = []string{
					"zrem",
					name,
					key,
				}
			}
		case BINLOGCOMMAND_QSET, BINLOGCOMMAND_QPUSH_BACK, BINLOGCOMMAND_QPUSH_FRONT:
			if len(bytes) != 2 {
				break
			}
			size := int(body[offset])
			offset += 1
			name := string(body[offset : offset+size])
			offset = offset + size
			seq := binary.BigEndian.Uint64(body[offset:])
			if seq < SSDB_QITEM_MIN_SEQ || seq > SSDB_QITEM_MAX_SEQ {
				break
			}
			val := string(bytes[1]) //list val
			switch binlog.cmdtype{
			case BINLOGCOMMAND_QPUSH_BACK:
				binlog.cmd = []string{
					"rpush",
					name,
					val,
				}
			case BINLOGCOMMAND_QPUSH_FRONT:
				binlog.cmd = []string{
					"lpush",
					name,
					val,
				}
			case BINLOGCOMMAND_QSET:
				fmt.Println("unsuported qset binlog:", name, seq, val)
			default:
				fmt.Println("unknown qet/push binlog:", binlog)
			}
		case BINLOGCOMMAND_QPOP_BACK,BINLOGCOMMAND_QPOP_FRONT:
			name := string(body[:])
			switch binlog.cmdtype{
			case BINLOGCOMMAND_QPOP_BACK:
				binlog.cmd = []string{
					"rpop",
					name,
				}
			case BINLOGCOMMAND_QPOP_FRONT:
				binlog.cmd = []string{
					"lpop",
					name,
				}
			default:
				fmt.Println("unknown pop binlog:", binlog)
			}
		default:
			fmt.Println("unknown binlog:", binlog)
		}

	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"
)

func item() string {
	return [3]string{"apple", "banana", "orange"}[rand.Intn(3)]
}

func amount() string {
	return fmt.Sprintf("%d", rand.Intn(9)+1)
}

func printLine() {
	fmt.Printf("%s\t%s\n", item(), amount())
}

func main() {
	rand.Seed(time.Now().UnixNano())
	for i := 0; i < 100; i++ {
		printLine()
	}
}
<commit_msg>Make generate smarter. It crashes.<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
)

func item() string {
	return [3]string{"apple", "banana", "orange"}[rand.Intn(3)]
}

func amount() string {
	return fmt.Sprintf("%d", rand.Intn(9)+1)
}

func line() string {
	return fmt.Sprintf("%s\t%s\n", item(), amount())
}

type reader struct {
	lines int
}

func (r *reader) Read(p []byte) (int, error) {
	size := 0
	for ; r.lines > 0; r.lines-- {
		line := line()
		if len(line) > len(p[size:]) {
			break
		}
		copy(p[size:], line)
		size += len(line)
	}
	return size, nil
}

func main() {
	var commits int
	var lines int
	cmd := &cobra.Command{
		Use:   os.Args[0],
		Short: "Generate pfs traffic.",
		Long:  "Generate pfs traffic.",
		Run: pkgcobra.RunFixedArgs(0, func(args []string) error {
			rand.Seed(time.Now().UnixNano())
			client, err := client.NewFromAddress("0.0.0.0:30650")
			if err != nil {
				return err
			}
			for i := 0; i < commits; i++ {
				if _, err := pfs.StartCommit(client, "data", "", "master"); err != nil {
					return err
				}
				if _, err := pfs.PutFile(client, "data", "master", "sales", 0, &reader{lines}); err != nil {
					return err
				}
				if err := pfs.FinishCommit(client, "data", "master"); err != nil {
					return err
				}
			}
			return nil
		}),
	}
	cmd.Flags().IntVarP(&commits, "commits", "c", 0, "commits to write")
	cmd.Flags().IntVarP(&lines, "lines", "l", 0, "lines to write for each commit")
	cmd.Execute()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/bunsenapp/go-selenium"
)

func main() {
	// Create a capabilities object.
	capabilities := goselenium.Capabilities{}

	// Populate it with the browser you wish to use.
	capabilities.SetBrowser(goselenium.FirefoxBrowser())

	// Initialise a new web driver.
	driver, err := goselenium.NewSeleniumWebDriver("http://localhost:4444/wd/hub", capabilities)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Create a session.
	_, err = driver.CreateSession()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Navigate to Google.
	_, err = driver.Go("https://www.google.com")
	if err != nil {
		fmt.Println(err)
	}

	// Hooray, we navigated to Google!
	fmt.Println("Successfully navigated to Google!")

	// Delete the session to ensure the Selenium hub doesn't fall over.
	_, err = driver.DeleteSession()
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>Change to defer<commit_after>package main

import (
	"fmt"

	"github.com/bunsenapp/go-selenium"
)

func main() {
	// Create a capabilities object.
	capabilities := goselenium.Capabilities{}

	// Populate it with the browser you wish to use.
	capabilities.SetBrowser(goselenium.FirefoxBrowser())

	// Initialise a new web driver.
	driver, err := goselenium.NewSeleniumWebDriver("http://localhost:4444/wd/hub", capabilities)
	if err != nil {
		fmt.Println(err)
		return
	}

	// Create a session.
	_, err = driver.CreateSession()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Defer the deletion of the session.
	defer driver.DeleteSession()

	// Navigate to Google.
	_, err = driver.Go("https://www.google.com")
	if err != nil {
		fmt.Println(err)
	}

	// Hooray, we navigated to Google!
	fmt.Println("Successfully navigated to Google!")
}
<|endoftext|>


<commit_before>package router

import (
	"net/http"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	kapierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"

	routeclientset "github.com/openshift/client-go/route/clientset/versioned"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/url"
)

var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()
	var (
		host, ns string
		oc       *exutil.CLI

		configPath = exutil.FixturePath("testdata", "ingress.yaml")
	)

	// this hook must be registered before the framework namespace teardown
	// hook
	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			exutil.DumpPodLogsStartingWithInNamespace("router", "default", oc.AsAdmin())
			selector, err := labels.Parse("router=router")
			if err != nil {
				panic(err)
			}
			exutil.DumpPodsCommand(oc.AdminKubeClient(), "default", selector, "cat /var/lib/haproxy/router/routes.json /var/lib//var/lib/haproxy/conf/haproxy.config")
		}
	})

	oc = exutil.NewCLI("router-stress", exutil.KubeConfigPath())

	g.BeforeEach(func() {
		var err error
		host, err = waitForFirstRouterEndpointIP(oc)
		if kapierrs.IsNotFound(err) {
			g.Skip("no router installed on the cluster")
			return
		}
		o.Expect(err).NotTo(o.HaveOccurred())

		ns = oc.KubeFramework().Namespace.Name
	})

	g.Describe("The HAProxy router", func() {
		g.It("should respond with 503 to unrecognized hosts", func() {
			t := url.NewTester(oc.AdminKubeClient(), ns)
			defer t.Close()
			t.Within(
				time.Minute,
				url.Expect("GET", "https://www.google.com").Through(host).SkipTLSVerification().HasStatusCode(503),
				url.Expect("GET", "http://www.google.com").Through(host).HasStatusCode(503),
			)
		})

		g.It("should serve routes that were created from an ingress", func() {
			g.By("deploying an ingress rule")
			err := oc.Run("create").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("waiting for the ingress rule to be converted to routes")
			client := routeclientset.NewForConfigOrDie(oc.AdminConfig())
			err = wait.Poll(time.Second, time.Minute, func() (bool, error) {
				routes, err := client.Route().Routes(ns).List(metav1.ListOptions{})
				if err != nil {
					return false, err
				}
				return len(routes.Items) == 4, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("verifying the router reports the correct behavior")
			t := url.NewTester(oc.AdminKubeClient(), ns)
			defer t.Close()
			t.Within(
				3*time.Minute,
				url.Expect("GET", "http://1.ingress-test.com/test").Through(host).HasStatusCode(200),
				url.Expect("GET", "http://1.ingress-test.com/other/deep").Through(host).HasStatusCode(200),
				url.Expect("GET", "http://1.ingress-test.com/").Through(host).HasStatusCode(503),
				url.Expect("GET", "http://2.ingress-test.com/").Through(host).HasStatusCode(200),
				url.Expect("GET", "https://3.ingress-test.com/").Through(host).SkipTLSVerification().HasStatusCode(200),
				url.Expect("GET", "http://3.ingress-test.com/").Through(host).RedirectsTo("https://3.ingress-test.com/", http.StatusFound),
			)
		})
	})
})

func waitForFirstRouterEndpointIP(oc *exutil.CLI) (string, error) {
	_, ns, err := exutil.GetRouterPodTemplate(oc)
	if err != nil {
		return "", err
	}

	// wait for at least one router endpoint to be up router endpoints to show up
	var host string
	err = wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) {
		epts, err := oc.AdminKubeClient().CoreV1().Endpoints(ns).Get("router", metav1.GetOptions{})
		o.Expect(err).NotTo(o.HaveOccurred())
		if len(epts.Subsets) == 0 || len(epts.Subsets[0].Addresses) == 0 {
			return false, nil
		}
		host = epts.Subsets[0].Addresses[0].IP
		return true, nil
	})
	return host, err
}
<commit_msg>[WIP] Update extended tests for ingress operator<commit_after>package router

import (
	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"
	"net/http"
	"time"

	kapierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/wait"

	routeclientset "github.com/openshift/client-go/route/clientset/versioned"
	exutil "github.com/openshift/origin/test/extended/util"
	"github.com/openshift/origin/test/extended/util/url"
)

var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()
	var (
		host, ns string
		oc       *exutil.CLI

		configPath = exutil.FixturePath("testdata", "ingress.yaml")
	)

	// this hook must be registered before the framework namespace teardown
	// hook
	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			currlabel := "router=router"
			for _, ns := range []string{"default", "openshift-ingress", "tectonic-ingress"} {
				//Search the router by label
				if ns == "openshift-ingress" {
					currlabel = "router=router-default"
				}
				exutil.DumpPodLogsStartingWithInNamespace("router", ns, oc.AsAdmin())
				selector, err := labels.Parse(currlabel)
				if err != nil {
					panic(err)
				}
				exutil.DumpPodsCommand(oc.AdminKubeClient(), ns, selector, "cat /var/lib/haproxy/router/routes.json /var/lib//var/lib/haproxy/conf/haproxy.config")
			}
		}
	})

	oc = exutil.NewCLI("router-stress", exutil.KubeConfigPath())

	g.BeforeEach(func() {
		var err error
		host, err = waitForFirstRouterEndpointIP(oc)
		if kapierrs.IsNotFound(err) {
			g.Skip("no router installed on the cluster")
			return
		}
		o.Expect(err).NotTo(o.HaveOccurred())

		ns = oc.KubeFramework().Namespace.Name
	})

	g.Describe("The HAProxy router", func() {
		g.It("should respond with 503 to unrecognized hosts", func() {
			t := url.NewTester(oc.AdminKubeClient(), ns)
			defer t.Close()
			t.Within(
				time.Minute,
				url.Expect("GET", "https://www.google.com").Through(host).SkipTLSVerification().HasStatusCode(503),
				url.Expect("GET", "http://www.google.com").Through(host).HasStatusCode(503),
			)
		})

		g.It("should serve routes that were created from an ingress", func() {
			g.By("deploying an ingress rule")
			err := oc.Run("create").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("waiting for the ingress rule to be converted to routes")
			client := routeclientset.NewForConfigOrDie(oc.AdminConfig())
			err = wait.Poll(time.Second, time.Minute, func() (bool, error) {
				routes, err := client.Route().Routes(ns).List(metav1.ListOptions{})
				if err != nil {
					return false, err
				}
				return len(routes.Items) == 4, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred())

			g.By("verifying the router reports the correct behavior")
			t := url.NewTester(oc.AdminKubeClient(), ns)
			defer t.Close()
			t.Within(
				3*time.Minute,
				url.Expect("GET", "http://1.ingress-test.com/test").Through(host).HasStatusCode(200),
				url.Expect("GET", "http://1.ingress-test.com/other/deep").Through(host).HasStatusCode(200),
				url.Expect("GET", "http://1.ingress-test.com/").Through(host).HasStatusCode(503),
				url.Expect("GET", "http://2.ingress-test.com/").Through(host).HasStatusCode(200),
				url.Expect("GET", "https://3.ingress-test.com/").Through(host).SkipTLSVerification().HasStatusCode(200),
				url.Expect("GET", "http://3.ingress-test.com/").Through(host).RedirectsTo("https://3.ingress-test.com/", http.StatusFound),
			)
		})
	})
})

func waitForFirstRouterEndpointIP(oc *exutil.CLI) (string, error) {
	// wait for at least one router endpoint to be up router endpoints to show up
	var host string
	err := wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) {
		currlabel := "router=router"
		for _, ns := range []string{"default", "openshift-ingress", "tectonic-ingress"} {
			//Search the router by label
			if ns == "openshift-ingress" {
				currlabel = "router=router-default"
			}
			podList, err := oc.AdminKubeClient().CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: currlabel})
			if podList.Items != nil && len(podList.Items) > 0 {
				host = podList.Items[0].Status.PodIP
				return true, nil
			}
			o.Expect(err).NotTo(o.HaveOccurred())
		}
		return false, nil
	})
	return host, err
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	_ "github.com/elastic/apm-server/include"
	"github.com/elastic/apm-server/processor"
)

func main() {

	err := generate()
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
}

func generate() error {
	filename := "payload.json"
	basepath := "tests/data/valid"
	outputPath := "docs/data/elasticsearch/"

	processors := processor.Registry.Processors()

	var checked = map[string]struct{}{}

	for _, p := range processors {

		// Remove version from name and and s at the end
		name := p.Name()

		// Create path to test docs
		path := filepath.Join(basepath, name, filename)

		f, err := os.Open(path)
		if err != nil {
			return err
		}

		data, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}

		err = p.Validate(data)
		if err != nil {
			return err
		}

		events, err := p.Transform(data)

		if err != nil {
			return err
		}

		for _, d := range events {
			n, err := d.GetValue("processor.name")
			if err != nil {
				return err
			}
			event, err := d.GetValue("processor.event")
			if err != nil {
				return err
			}

			key := n.(string) + "-" + event.(string)

			if _, ok := checked[key]; !ok {
				checked[key] = struct{}{}
				file := filepath.Join(outputPath, event.(string)+".json")

				output, err := json.MarshalIndent(d.Fields, "", "    ")
				err = ioutil.WriteFile(file, output, 0644)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<commit_msg>Fix swallowed error in output_data script (#164)<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	_ "github.com/elastic/apm-server/include"
	"github.com/elastic/apm-server/processor"
)

func main() {

	err := generate()
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %s\n", err)
		os.Exit(1)
	}
}

func generate() error {
	filename := "payload.json"
	basepath := "tests/data/valid"
	outputPath := "docs/data/elasticsearch/"

	processors := processor.Registry.Processors()

	var checked = map[string]struct{}{}

	for _, p := range processors {

		// Remove version from name and and s at the end
		name := p.Name()

		// Create path to test docs
		path := filepath.Join(basepath, name, filename)

		f, err := os.Open(path)
		if err != nil {
			return err
		}

		data, err := ioutil.ReadAll(f)
		if err != nil {
			return err
		}

		err = p.Validate(data)
		if err != nil {
			return err
		}

		events, err := p.Transform(data)

		if err != nil {
			return err
		}

		for _, d := range events {
			n, err := d.GetValue("processor.name")
			if err != nil {
				return err
			}
			event, err := d.GetValue("processor.event")
			if err != nil {
				return err
			}

			key := n.(string) + "-" + event.(string)

			if _, ok := checked[key]; !ok {
				checked[key] = struct{}{}
				file := filepath.Join(outputPath, event.(string)+".json")

				output, err := json.MarshalIndent(d.Fields, "", "    ")
				if err != nil {
					return err
				}
				err = ioutil.WriteFile(file, output, 0644)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxdclient

import (
	"fmt"
	"os"
	"path"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/lxc/lxd"
)

// The LXD repo does not expose any consts for these values.
// TODO(ericsnow) Expose these consts in the LXD repo?
const (
	// see https://github.com/lxc/lxd/blob/master/config.go
	configDefaultFile = "config.yml"
	// see https://github.com/lxc/lxd/blob/master/client.go (readMyCert)
	configCertFile = "client.crt"
	configKeyFile  = "client.key"
)

// Config contains the config values used for a connection to the LXD API.
type Config struct {
	// Namespace identifies the namespace to associate with containers
	// and other resources with which the client interacts. If may be
	// blank.
	Namespace string

	// Dirname identifies where the client will find config files.
	// default: "$HOME/.config/lxc"
	Dirname string

	// Remote identifies the remote server to which the client should
	// connect. For the default "remote" use Local.
	Remote Remote
}

// ConfigPath returns the full path to Juju LXC config directory.
// This is here so we do not have to import lxc/lxd into the provider.
func ConfigPath(namespace string) string {
	// Here we use the same path as lxc for convention, but we could use
	// any juju specific path.
	return os.ExpandEnv(path.Join("$HOME/.config/lxc", namespace))
}

// WithDefaults updates a copy of the config with default values
// where needed.
func (cfg Config) WithDefaults() (Config, error) {
	// We leave a blank namespace alone.
	// Also, note that cfg is a value receiver, so it is an implicit copy.

	if cfg.Dirname == "" {
		// TODO(ericsnow) Switch to filepath as soon as LXD does,
		dirname := path.Dir(ConfigPath("DUMMY"))
		cfg.Dirname = path.Clean(dirname)
		fmt.Println("cleaned config %s %s", dirname, cfg.Dirname)
	}

	var err error
	cfg.Remote, err = cfg.Remote.WithDefaults()
	if err != nil {
		return cfg, errors.Trace(err)
	}

	return cfg, nil
}

// Validate checks the client's fields for invalid values.
func (cfg Config) Validate() error {
	// TODO(ericsnow) Check cfg.Namespace (if provided)?

	// TODO(ericsnow) Check cfg.Dirname (if provided)?

	// TODO(ericsnow) Check cfg.Filename (if provided)?

	if err := cfg.Remote.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Write writes all the various files for this config.
func (cfg Config) Write() error {
	if err := cfg.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := cfg.write(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// UsingTCPRemote converts the config into a "non-local" version. An
// already non-local remote is left alone.
//
// For a "local" remote (see Local), the remote is changed to a one
// with the host set to the IP address of the local lxcbr0 bridge
// interface. The LXD server is also set up for remote access, exposing
// the TCP port and adding a certificate for remote access.
func (cfg Config) UsingTCPRemote() (Config, error) {
	// Note that cfg is a value receiver, so it is an implicit copy.

	if !cfg.Remote.isLocal() {
		return cfg, nil
	}

	remote, err := cfg.Remote.UsingTCP()
	if err != nil {
		return cfg, errors.Trace(err)
	}

	// Update the server config and authorized certs.
	if err := prepareRemote(cfg, *remote.Cert); err != nil {
		return cfg, errors.Trace(err)
	}

	cfg.Remote = remote
	return cfg, nil
}

func prepareRemote(cfg Config, newCert Cert) error {
	client, err := Connect(cfg)
	if err != nil {
		return errors.Trace(err)
	}

	if err := client.SetConfig("core.https_address", "[::]"); err != nil {
		return errors.Trace(err)
	}

	if err := client.AddCert(newCert); err != nil {
		return errors.Trace(err)
	}

	return nil
}

//TODO(wwitzel3) make sure this is idempotent
func initializeConfigDir(cfg Config) error {
	logger.Debugf("initializing config dir %q", cfg.Dirname)

	if err := os.MkdirAll(cfg.Dirname, 0775); err != nil {
		return errors.Trace(err)
	}

	// Force the default config to get written. LoadConfig() returns the
	// default config from memory if there isn't a config file on disk.
	// So we load that and then explicitly save it to disk with a call
	// to SaveConfig().
	config, err := lxd.LoadConfig(cfg.resolve(configDefaultFile))
	if err != nil {
		return errors.Trace(err)
	}

	if err := lxd.SaveConfig(config, cfg.resolve(configDefaultFile)); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) write() error {
	// Ensure the initial config is set up.
	if err := initializeConfigDir(cfg); err != nil {
		return errors.Trace(err)
	}

	// Update config.yml, if necessary.
	if err := cfg.writeConfigFile(); err != nil {
		return errors.Trace(err)
	}

	// Write the cert file and key file, if applicable.
	if err := cfg.writeCert(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeConfigFile() error {
	filename := cfg.resolve(configDefaultFile)
	logger.Debugf("writing config file %q", filename)

	// TODO(ericsnow) Cache the low-level config in Config?
	rawCfg, err := lxd.LoadConfig(cfg.resolve(configDefaultFile))
	if err != nil {
		return errors.Trace(err)
	}

	if !cfg.Remote.isLocal() {
		// Ensure the remote is set correctly.
		remote := cfg.Remote.Name
		delete(rawCfg.Remotes, remote)
		addr := cfg.Remote.Host
		if err := addServer(rawCfg, remote, addr); err != nil {
			return errors.Trace(err)
		}
	}

	// Write out the updated config, if changed.
	// TODO(ericsnow) Check if changed.
	if err := lxd.SaveConfig(rawCfg, cfg.resolve(configDefaultFile)); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeCert() error {
	if cfg.Remote.Cert.isZero() {
		logger.Debugf("not writing empty certificate")
		return nil
	}

	cert := *cfg.Remote.Cert
	if err := cert.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := cfg.writeCertPEM(cert); err != nil {
		return errors.Trace(err)
	}
	if err := cfg.writeKeyPEM(cert); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeCertPEM(cert Cert) error {
	filename := cfg.resolve(configCertFile)
	logger.Debugf("writing cert PEM file %q", filename)

	file, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer file.Close()

	if err := cert.WriteCertPEM(file); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (cfg Config) writeKeyPEM(cert Cert) error {
	filename := cfg.resolve(configKeyFile)
	logger.Debugf("writing key PEM file %q", filename)

	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return errors.Trace(err)
	}
	defer file.Close()

	if err := cert.WriteKeyPEM(file); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (cfg Config) resolve(file string) string {
	return filepath.Join(cfg.Dirname, file)
}
<commit_msg>lxdclient: fix typo<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// +build go1.3

package lxdclient

import (
	"fmt"
	"os"
	"path"
	"path/filepath"

	"github.com/juju/errors"
	"github.com/lxc/lxd"
)

// The LXD repo does not expose any consts for these values.
// TODO(ericsnow) Expose these consts in the LXD repo?
const (
	// see https://github.com/lxc/lxd/blob/master/config.go
	configDefaultFile = "config.yml"
	// see https://github.com/lxc/lxd/blob/master/client.go (readMyCert)
	configCertFile = "client.crt"
	configKeyFile  = "client.key"
)

// Config contains the config values used for a connection to the LXD API.
type Config struct {
	// Namespace identifies the namespace to associate with containers
	// and other resources with which the client interacts. It may be
	// blank.
	Namespace string

	// Dirname identifies where the client will find config files.
	// default: "$HOME/.config/lxc"
	Dirname string

	// Remote identifies the remote server to which the client should
	// connect. For the default "remote" use Local.
	Remote Remote
}

// ConfigPath returns the full path to Juju LXC config directory.
// This is here so we do not have to import lxc/lxd into the provider.
func ConfigPath(namespace string) string {
	// Here we use the same path as lxc for convention, but we could use
	// any juju specific path.
	return os.ExpandEnv(path.Join("$HOME/.config/lxc", namespace))
}

// WithDefaults updates a copy of the config with default values
// where needed.
func (cfg Config) WithDefaults() (Config, error) {
	// We leave a blank namespace alone.
	// Also, note that cfg is a value receiver, so it is an implicit copy.

	if cfg.Dirname == "" {
		// TODO(ericsnow) Switch to filepath as soon as LXD does,
		dirname := path.Dir(ConfigPath("DUMMY"))
		cfg.Dirname = path.Clean(dirname)
		fmt.Println("cleaned config %s %s", dirname, cfg.Dirname)
	}

	var err error
	cfg.Remote, err = cfg.Remote.WithDefaults()
	if err != nil {
		return cfg, errors.Trace(err)
	}

	return cfg, nil
}

// Validate checks the client's fields for invalid values.
func (cfg Config) Validate() error {
	// TODO(ericsnow) Check cfg.Namespace (if provided)?

	// TODO(ericsnow) Check cfg.Dirname (if provided)?

	// TODO(ericsnow) Check cfg.Filename (if provided)?

	if err := cfg.Remote.Validate(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// Write writes all the various files for this config.
func (cfg Config) Write() error {
	if err := cfg.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := cfg.write(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

// UsingTCPRemote converts the config into a "non-local" version. An
// already non-local remote is left alone.
//
// For a "local" remote (see Local), the remote is changed to a one
// with the host set to the IP address of the local lxcbr0 bridge
// interface. The LXD server is also set up for remote access, exposing
// the TCP port and adding a certificate for remote access.
func (cfg Config) UsingTCPRemote() (Config, error) {
	// Note that cfg is a value receiver, so it is an implicit copy.

	if !cfg.Remote.isLocal() {
		return cfg, nil
	}

	remote, err := cfg.Remote.UsingTCP()
	if err != nil {
		return cfg, errors.Trace(err)
	}

	// Update the server config and authorized certs.
	if err := prepareRemote(cfg, *remote.Cert); err != nil {
		return cfg, errors.Trace(err)
	}

	cfg.Remote = remote
	return cfg, nil
}

func prepareRemote(cfg Config, newCert Cert) error {
	client, err := Connect(cfg)
	if err != nil {
		return errors.Trace(err)
	}

	if err := client.SetConfig("core.https_address", "[::]"); err != nil {
		return errors.Trace(err)
	}

	if err := client.AddCert(newCert); err != nil {
		return errors.Trace(err)
	}

	return nil
}

//TODO(wwitzel3) make sure this is idempotent
func initializeConfigDir(cfg Config) error {
	logger.Debugf("initializing config dir %q", cfg.Dirname)

	if err := os.MkdirAll(cfg.Dirname, 0775); err != nil {
		return errors.Trace(err)
	}

	// Force the default config to get written. LoadConfig() returns the
	// default config from memory if there isn't a config file on disk.
	// So we load that and then explicitly save it to disk with a call
	// to SaveConfig().
	config, err := lxd.LoadConfig(cfg.resolve(configDefaultFile))
	if err != nil {
		return errors.Trace(err)
	}

	if err := lxd.SaveConfig(config, cfg.resolve(configDefaultFile)); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) write() error {
	// Ensure the initial config is set up.
	if err := initializeConfigDir(cfg); err != nil {
		return errors.Trace(err)
	}

	// Update config.yml, if necessary.
	if err := cfg.writeConfigFile(); err != nil {
		return errors.Trace(err)
	}

	// Write the cert file and key file, if applicable.
	if err := cfg.writeCert(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeConfigFile() error {
	filename := cfg.resolve(configDefaultFile)
	logger.Debugf("writing config file %q", filename)

	// TODO(ericsnow) Cache the low-level config in Config?
	rawCfg, err := lxd.LoadConfig(cfg.resolve(configDefaultFile))
	if err != nil {
		return errors.Trace(err)
	}

	if !cfg.Remote.isLocal() {
		// Ensure the remote is set correctly.
		remote := cfg.Remote.Name
		delete(rawCfg.Remotes, remote)
		addr := cfg.Remote.Host
		if err := addServer(rawCfg, remote, addr); err != nil {
			return errors.Trace(err)
		}
	}

	// Write out the updated config, if changed.
	// TODO(ericsnow) Check if changed.
	if err := lxd.SaveConfig(rawCfg, cfg.resolve(configDefaultFile)); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeCert() error {
	if cfg.Remote.Cert.isZero() {
		logger.Debugf("not writing empty certificate")
		return nil
	}

	cert := *cfg.Remote.Cert
	if err := cert.Validate(); err != nil {
		return errors.Trace(err)
	}

	if err := cfg.writeCertPEM(cert); err != nil {
		return errors.Trace(err)
	}
	if err := cfg.writeKeyPEM(cert); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (cfg Config) writeCertPEM(cert Cert) error {
	filename := cfg.resolve(configCertFile)
	logger.Debugf("writing cert PEM file %q", filename)

	file, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer file.Close()

	if err := cert.WriteCertPEM(file); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (cfg Config) writeKeyPEM(cert Cert) error {
	filename := cfg.resolve(configKeyFile)
	logger.Debugf("writing key PEM file %q", filename)

	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return errors.Trace(err)
	}
	defer file.Close()

	if err := cert.WriteKeyPEM(file); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (cfg Config) resolve(file string) string {
	return filepath.Join(cfg.Dirname, file)
}
<|endoftext|>


<commit_before>package mallory

import (
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os/user"
	"sync"

	"golang.org/x/crypto/ssh"
)

//
type SSH struct {
	// global config file
	Cfg *Config
	// connect URL
	URL *url.URL
	// SSH client
	Client *ssh.Client
	// SSH client config
	CliCfg *ssh.ClientConfig
	// direct fetcher
	Direct *Direct
	// atomic Dial
	mutex sync.RWMutex
}

// Create and initialize
func NewSSH(c *Config) (self *SSH, err error) {
	self = &SSH{
		Cfg:    c,
		CliCfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(c.File.RemoteServer)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.CliCfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.CliCfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := c.File.PrivateKey
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			L.Printf("ReadFile %s failed:%s\n", c.File.PrivateKey, err)
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			L.Printf("ParsePrivateKey %s failed:%s\n", c.File.PrivateKey, err)
			break
		}
		self.CliCfg.Auth = append(self.CliCfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.CliCfg.Auth = append(self.CliCfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.CliCfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	// first time to dial to remote server, make sure it is available
	self.Client, err = ssh.Dial("tcp", self.URL.Host, self.CliCfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (c net.Conn, err error) {
		for i := 0; i < 8; i++ {
			self.mutex.RLock()
			saveClient := self.Client
			if self.Client != nil && err == nil {
				c, err = self.Client.Dial(network, addr)
			}
			self.mutex.RUnlock()
			if self.Client != nil && err == nil {
				break // success
			}
			self.mutex.Lock()
			if saveClient == self.Client { // the thread to reconnect
				if self.Client != nil {
					self.Client.Close()
				}
				L.Printf("reconnecting %s...\n", self.URL.Host)
				self.Client, err = ssh.Dial("tcp", self.URL.Host, self.CliCfg)
			}
			self.mutex.Unlock()
		}
		return
	}

	self.Direct = &Direct{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *SSH) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	self.Direct.ServeHTTP(w, r)
}

func (self *SSH) Connect(w http.ResponseWriter, r *http.Request) {
	self.Direct.Connect(w, r)
}
<commit_msg>fix close<commit_after>package mallory

import (
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os/user"
	"sync"

	"golang.org/x/crypto/ssh"
)

//
type SSH struct {
	// global config file
	Cfg *Config
	// connect URL
	URL *url.URL
	// SSH client
	Client *ssh.Client
	// SSH client config
	CliCfg *ssh.ClientConfig
	// direct fetcher
	Direct *Direct
	// atomic Dial
	dialMutex sync.RWMutex
	// dialed conns
	conns []net.Conn
	// connsMutex
	connsMutex sync.RWMutex
}

// Create and initialize
func NewSSH(c *Config) (self *SSH, err error) {
	self = &SSH{
		Cfg:    c,
		CliCfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(c.File.RemoteServer)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.CliCfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.CliCfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := c.File.PrivateKey
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			L.Printf("ReadFile %s failed:%s\n", c.File.PrivateKey, err)
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			L.Printf("ParsePrivateKey %s failed:%s\n", c.File.PrivateKey, err)
			break
		}
		self.CliCfg.Auth = append(self.CliCfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.CliCfg.Auth = append(self.CliCfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.CliCfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	// first time to dial to remote server, make sure it is available
	self.Client, err = ssh.Dial("tcp", self.URL.Host, self.CliCfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (c net.Conn, err error) {
		for i := 0; i < 8; i++ {
			self.dialMutex.RLock()
			saveClient := self.Client
			if self.Client != nil && err == nil {
				c, err = self.Client.Dial(network, addr)
			}
			self.dialMutex.RUnlock()

			if self.Client != nil && err == nil {
				self.connsMutex.Lock()
				self.conns = append(self.conns, c)
				self.connsMutex.Unlock()
				break // success
			}

			self.dialMutex.Lock()
			if saveClient == self.Client { // the thread to reconnect
				if self.Client != nil {
					self.Client.Close()
				}

				self.connsMutex.RLock()
				for _, d := range self.conns {
					err1 := d.Close()
					if err1 != nil {
						L.Println(err1)
					}
				}
				self.connsMutex.RUnlock()

				self.connsMutex.Lock()
				self.conns = self.conns[:0]
				self.connsMutex.Unlock()

				L.Printf("reconnecting %s: %s\n", self.URL.Host, err)
				self.Client, err = ssh.Dial("tcp", self.URL.Host, self.CliCfg)
			}
			self.dialMutex.Unlock()
		}
		return
	}

	self.Direct = &Direct{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *SSH) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	self.Direct.ServeHTTP(w, r)
}

func (self *SSH) Connect(w http.ResponseWriter, r *http.Request) {
	self.Direct.Connect(w, r)
}
<|endoftext|>


<commit_before>package unifi

type Sta struct {
	Ap_mac             string
	Assoc_time         int //Timestamp
	Auth_time          int //Timestamp
	Authorized         bool
	Bssid              string
	Bytes              int
	Bytes_d            int `json:"bytes.d"`
	Bytes_r            int `json:"bytes.r"`
	Ccq                int
	Channel            int
	Dhcpend_time       int
	Dhcpstart_time     int
	Essid              string
	First_seen         int //Timestamp
	Hostname           string
	Idletime           int
	Ip                 string
	Is_11a             bool
	Is_11ac            bool
	Is_11b             bool
	Is_11n             bool
	Is_guest           bool
	Last_seen          int //Timestamp
	Mac                string
	Map_id             string
	Noise              int
	Oui                string
	Powersave_enabled  bool
	Qos_policy_applied bool
	Radio              string
	Roam_count         int
	Rssi               int
	Rx_bytes           int
	Rx_bytes_d         int `json:"rx_bytes.d"`
	Rx_bytes_r         int `json:"rx_bytes.r"`
	Rx_crypts          int
	Rx_crypts_d        int `json:"rx_crytps.d"`
	Rx_crypts_r        int `json:"rx_crytps.r"`
	Rx_dropped         int
	Rx_dropped_d       int `json:"rx_dropped.d"`
	Rx_dropped_r       int `json:"rx_dropped.r"`
	Rx_errors          int
	Rx_errors_d        int `json:"rx_errors.d"`
	Rx_errors_r        int `json:"rx_errors.r"`
	Rx_frags           int
	Rx_frags_d         int `json:"rx_frags.d"`
	Rx_frags_r         int `json:"rx_frags.r"`
	Rx_packets         int
	Rx_packets_d       int `json:"rx_packets.d"`
	Rx_packets_r       int `json:"rx_packets.r"`
	Rx_rate            int
	Signal             int
	State              int
	State_ht           bool
	State_pwrmgt       bool
	T                  string
	Tx_bytes           int
	Tx_bytes_d         int `json:"tx_bytes.d"`
	Tx_bytes_r         int `json:"tx_bytes.r"`
	Tx_dropped         int
	Tx_dropped_d       int `json:"tx_dropped.d"`
	Tx_dropped_r       int `json:"tx_dropped.r"`
	Tx_errors          int
	Tx_errors_d        int `json:"tx_errors.d"`
	Tx_errors_r        int `json:"tx_errors.r"`
	Tx_packets         int
	Tx_packets_d       int `json:"tx_packets.d"`
	Tx_packets_r       int `json:"tx_packets.r"`
	Tx_power           int
	Tx_rate            int
	Tx_retries         int
	Tx_retries_d       int `json:"tx_retries.d"`
	Tx_retries_r       int `json:"tx_retries.r"`
	Uptime             int
	User_id            string
}
<commit_msg>add GetName<commit_after>package unifi

type Sta struct {
	Ap_mac             string
	Assoc_time         int //Timestamp
	Auth_time          int //Timestamp
	Authorized         bool
	Bssid              string
	Bytes              int
	Bytes_d            int `json:"bytes.d"`
	Bytes_r            int `json:"bytes.r"`
	Ccq                int
	Channel            int
	Dhcpend_time       int
	Dhcpstart_time     int
	Essid              string
	First_seen         int //Timestamp
	Hostname           string
	Idletime           int
	Ip                 string
	Is_11a             bool
	Is_11ac            bool
	Is_11b             bool
	Is_11n             bool
	Is_guest           bool
	Last_seen          int //Timestamp
	Mac                string
	Map_id             string
	Noise              int
	Oui                string
	Powersave_enabled  bool
	Qos_policy_applied bool
	Radio              string
	Roam_count         int
	Rssi               int
	Rx_bytes           int
	Rx_bytes_d         int `json:"rx_bytes.d"`
	Rx_bytes_r         int `json:"rx_bytes.r"`
	Rx_crypts          int
	Rx_crypts_d        int `json:"rx_crytps.d"`
	Rx_crypts_r        int `json:"rx_crytps.r"`
	Rx_dropped         int
	Rx_dropped_d       int `json:"rx_dropped.d"`
	Rx_dropped_r       int `json:"rx_dropped.r"`
	Rx_errors          int
	Rx_errors_d        int `json:"rx_errors.d"`
	Rx_errors_r        int `json:"rx_errors.r"`
	Rx_frags           int
	Rx_frags_d         int `json:"rx_frags.d"`
	Rx_frags_r         int `json:"rx_frags.r"`
	Rx_packets         int
	Rx_packets_d       int `json:"rx_packets.d"`
	Rx_packets_r       int `json:"rx_packets.r"`
	Rx_rate            int
	Signal             int
	State              int
	State_ht           bool
	State_pwrmgt       bool
	T                  string
	Tx_bytes           int
	Tx_bytes_d         int `json:"tx_bytes.d"`
	Tx_bytes_r         int `json:"tx_bytes.r"`
	Tx_dropped         int
	Tx_dropped_d       int `json:"tx_dropped.d"`
	Tx_dropped_r       int `json:"tx_dropped.r"`
	Tx_errors          int
	Tx_errors_d        int `json:"tx_errors.d"`
	Tx_errors_r        int `json:"tx_errors.r"`
	Tx_packets         int
	Tx_packets_d       int `json:"tx_packets.d"`
	Tx_packets_r       int `json:"tx_packets.r"`
	Tx_power           int
	Tx_rate            int
	Tx_retries         int
	Tx_retries_d       int `json:"tx_retries.d"`
	Tx_retries_r       int `json:"tx_retries.r"`
	Uptime             int
	User_id            string
}

func (s Sta) GetName() string {
	if s.Hostname != "" {
		return s.Hostname
	}
	if s.Ip != "" {
		return s.Ip
	}
	return s.Mac
}
<|endoftext|>


<commit_before>package osc

import (
	"bytes"
	"fmt"
	"net"
)

// TCPListener is an OSC server based on TCP.
type TCPListener struct {
	*net.TCPListener
}

// ListenTCP creates a new TCP server.
func ListenTCP(network string, laddr *net.TCPAddr) (*TCPListener, error) {
	listener, err := net.ListenTCP(network, laddr)
	if err != nil {
		return nil, err
	}
	return &TCPListener{TCPListener: listener}, nil
}

// Serve starts dispatching OSC.
func (listener *TCPListener) Serve(dispatcher Dispatcher) error {
	if dispatcher == nil {
		return ErrNilDispatcher
	}

	for addr := range dispatcher {
		if err := validateAddress(addr); err != nil {
			return err
		}
	}

	for {
		if err := listener.serve(dispatcher); err != nil {
			return err
		}
		break
	}

	return nil
}

// serve retrieves OSC packets.
func (listener *TCPListener) serve(dispatcher Dispatcher) error {
	buf := make([]byte, 65536)
	for {
		conn, err := listener.Accept()
		if err != nil {
			return err
		}
		for _, err := conn.Read(buf); true; _, err = conn.Read(buf) {
			if err != nil {
				return err
			}
			switch buf[0] {
			case messageChar:
				msg, err := parseMessage(buf)
				if err != nil {
					fmt.Printf("========> error parsing message %s\n", err)
					return err
				}
				go func() { _ = dispatcher.DispatchMessage(msg) }()
			case bundleChar:
				bundle, err := parseBundle(buf)
				if err != nil {
					return err
				}
				go func() { _ = dispatcher.DispatchBundle(bundle) }()
			default:
				return ErrParse
			}
		}
	}
}

// TCPConn is an OSC connection over TCP.
type TCPConn struct {
	*net.TCPConn
}

// DialTCP creates a new OSC connection over TCP.
func DialTCP(network string, laddr, raddr *net.TCPAddr) (*TCPConn, error) {
	conn, err := net.DialTCP(network, laddr, raddr)
	if err != nil {
		return nil, err
	}
	return &TCPConn{TCPConn: conn}, nil
}

// Send sends an OSC message over UDP.
func (conn *TCPConn) Send(p Packet) (int64, error) {
	buf := &bytes.Buffer{}
	if _, err := p.WriteTo(buf); err != nil {
		return 0, err
	}
	n, err := conn.Write(buf.Bytes())
	if err != nil {
		return 0, err
	}
	return int64(n), nil
}
<commit_msg>Revert "fix go vet error"<commit_after>package osc

import (
	"bytes"
	"fmt"
	"net"
)

// TCPListener is an OSC server based on TCP.
type TCPListener struct {
	*net.TCPListener
}

// ListenTCP creates a new TCP server.
func ListenTCP(network string, laddr *net.TCPAddr) (*TCPListener, error) {
	listener, err := net.ListenTCP(network, laddr)
	if err != nil {
		return nil, err
	}
	return &TCPListener{TCPListener: listener}, nil
}

// Serve starts dispatching OSC.
func (listener *TCPListener) Serve(dispatcher Dispatcher) error {
	if dispatcher == nil {
		return ErrNilDispatcher
	}

	for addr := range dispatcher {
		if err := validateAddress(addr); err != nil {
			return err
		}
	}

	for {
		if err := listener.serve(dispatcher); err != nil {
			return err
		}
		break
	}

	return nil
}

// serve retrieves OSC packets.
func (listener *TCPListener) serve(dispatcher Dispatcher) error {
	buf := make([]byte, 65536)
	for {
		conn, err := listener.Accept()
		if err != nil {
			return err
		}
		for _, err := conn.Read(buf); true; _, err = conn.Read(buf) {
			if err != nil {
				return err
			}
			switch buf[0] {
			case messageChar:
				msg, err := parseMessage(buf)
				if err != nil {
					fmt.Printf("========> error parsing message %s\n", err)
					return err
				}
				go func() { _ = dispatcher.DispatchMessage(msg) }()
			case bundleChar:
				bundle, err := parseBundle(buf)
				if err != nil {
					return err
				}
				go func() { _ = dispatcher.DispatchBundle(bundle) }()
			default:
				return ErrParse
			}
		}
	}
	return nil
}

// TCPConn is an OSC connection over TCP.
type TCPConn struct {
	*net.TCPConn
}

// DialTCP creates a new OSC connection over TCP.
func DialTCP(network string, laddr, raddr *net.TCPAddr) (*TCPConn, error) {
	conn, err := net.DialTCP(network, laddr, raddr)
	if err != nil {
		return nil, err
	}
	return &TCPConn{TCPConn: conn}, nil
}

// Send sends an OSC message over UDP.
func (conn *TCPConn) Send(p Packet) (int64, error) {
	buf := &bytes.Buffer{}
	if _, err := p.WriteTo(buf); err != nil {
		return 0, err
	}
	n, err := conn.Write(buf.Bytes())
	if err != nil {
		return 0, err
	}
	return int64(n), nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"log"
	"math/big"
	"net"
	"net/http"
	"os"
	"runtime"
	"sync"
)

// Intercept TLS (HTTPS) connections.

var certFile = flag.String("tls-cert", "", "path to certificate for serving HTTPS")
var keyFile = flag.String("tls-key", "", "path to TLS certificate key")
var sslBypassFile = newActiveFlag("tls-bypass", "", "path to list of sites that bypass SSLBump", readBypassFile)
var tlsVerbose = flag.Bool("tls-verbose", false, "log all intercepted HTTPS connections")

var tlsCert tls.Certificate
var parsedTLSCert *x509.Certificate
var tlsReady bool

var tlsBypass = map[string]bool{}
var tlsBypassRanges []*net.IPNet

// unverifiedClientConfig is a TLS configuration that doesn't verify server
// certificates.
var unverifiedClientConfig = &tls.Config{
	InsecureSkipVerify: true,
}

// loadCertificate loads the TLS certificate specified by certFile and keyFile
// into tlsCert.
func loadCertificate() {
	if *certFile != "" && *keyFile != "" {
		cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
		if err != nil {
			log.Println("Error loading TLS certificate:", err)
			return
		}
		tlsCert = cert
		parsed, err := x509.ParseCertificate(cert.Certificate[0])
		if err != nil {
			log.Println("Error parsing X509 certificate:", err)
			return
		}
		parsedTLSCert = parsed
		tlsReady = true

		go cacheCertificates()
	}
}

func shouldBypass(host string) bool {
	if tlsBypass[host] {
		return true
	}
	addr := net.ParseIP(host)
	if addr == nil {
		return false
	}
	for _, subnet := range tlsBypassRanges {
		if subnet.Contains(addr) {
			return true
		}
	}
	return false
}

// SSLBump performs a man-in-the-middle attack on conn, to filter the HTTPS
// traffic. serverAddr is the address (host:port) of the server the client was
// trying to connect to.
func SSLBump(conn net.Conn, serverAddr string) {
	defer func() {
		if err := recover(); err != nil {
			buf := make([]byte, 4096)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("SSLBump: panic serving connection to %s: %v\n%s", serverAddr, err, buf)
			conn.Close()
		}
	}()

	if host, _, err := net.SplitHostPort(serverAddr); err == nil && shouldBypass(host) {
		// Just connect directly instead of bumping this connection.
		serverConn, err := net.Dial("tcp", serverAddr)
		if err != nil {
			log.Printf("error with pass-through of SSL connection to %s: %s", serverAddr, err)
			return
		}

		go func() {
			io.Copy(conn, serverConn)
			conn.Close()
		}()
		io.Copy(serverConn, conn)
		serverConn.Close()
		return
	}

	if *tlsVerbose {
		log.Printf("intercepting TLS connection from %s to %s", conn.RemoteAddr(), serverAddr)
	}

	cert, err := getCertificate(serverAddr)
	if err != nil {
		// Since it doesn't seem to be an HTTPS server, just connect directly.
		log.Printf("Could not generate a TLS certificate for %s (%s); letting the client connect directly", serverAddr, err)
		serverConn, err := net.Dial("tcp", serverAddr)
		if err != nil {
			log.Printf("Could not connect to %v: %v", serverAddr, err)
			return
		}

		go func() {
			io.Copy(conn, serverConn)
			conn.Close()
		}()
		io.Copy(serverConn, conn)
		serverConn.Close()
		return
	}

	config := &tls.Config{
		NextProtos:   []string{"http/1.1"},
		Certificates: []tls.Certificate{cert, tlsCert},
	}

	tlsConn := tls.Server(conn, config)
	err = tlsConn.Handshake()
	if err != nil {
		log.Printf("Error in TLS handshake for SSLBump connection from %v to %v: %v", conn.RemoteAddr(), serverAddr, err)
		return
	}

	_, port, err := net.SplitHostPort(serverAddr)
	if err != nil {
		port = ""
	}
	listener := &singleListener{conn: tlsConn}
	server := http.Server{
		Handler: proxyHandler{
			TLS:         true,
			connectPort: port,
		},
	}
	server.Serve(listener)
}

// A singleListener is a net.Listener that returns a single connection, then
// gives the error io.EOF.
type singleListener struct {
	conn net.Conn
	once sync.Once
}

func (s *singleListener) Accept() (net.Conn, error) {
	var c net.Conn
	s.once.Do(func() {
		c = s.conn
	})
	if c != nil {
		return c, nil
	}
	return nil, io.EOF
}

func (s *singleListener) Close() error {
	s.once.Do(func() {
		s.conn.Close()
	})
	return nil
}

func (s *singleListener) Addr() net.Addr {
	return s.conn.LocalAddr()
}

// maxSerial is the largest serial number to use for a certificate.
var maxSerial = big.NewInt(1<<63 - 1)

// generateCertificate connects to the server at addr, gets its TLS
// certificate, and returns a new certificate to be used when proxying
// connections to that server.
func generateCertificate(addr string) (tls.Certificate, error) {
	conn, err := tls.Dial("tcp", addr, unverifiedClientConfig)
	if err != nil {
		return tls.Certificate{}, err
	}
	defer conn.Close()
	state := conn.ConnectionState()
	serverCert := state.PeerCertificates[0]

	// Avoid duplicate serial numbers (NSS error -8054 in Chrome).
	serverCert.SerialNumber, err = rand.Int(rand.Reader, maxSerial)
	if err != nil {
		return tls.Certificate{}, fmt.Errorf("failed to generate serial number: %s", err)
	}

	priv, err := rsa.GenerateKey(rand.Reader, 1024)
	if err != nil {
		return tls.Certificate{}, fmt.Errorf("failed to generate private key: %s", err)
	}

	intermediates := x509.NewCertPool()
	for _, cert := range state.PeerCertificates[1:] {
		intermediates.AddCert(cert)
	}
	_, err = serverCert.Verify(x509.VerifyOptions{Intermediates: intermediates})
	signingCert := parsedTLSCert
	if _, ok := err.(x509.UnknownAuthorityError); ok {
		// There was a certificate error, so generate a self-signed certificate.
		signingCert = serverCert
	}

	newCertBytes, err := x509.CreateCertificate(rand.Reader, serverCert, signingCert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return tls.Certificate{}, err
	}

	certBuf := new(bytes.Buffer)
	pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: newCertBytes})
	keyBuf := new(bytes.Buffer)
	pem.Encode(keyBuf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})

	return tls.X509KeyPair(certBuf.Bytes(), keyBuf.Bytes())
}

func readBypassFile(filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("could not open %s: %s", filename, err)
	}
	defer f.Close()
	cr := newConfigReader(f)

	for {
		line, err := cr.ReadLine()
		if err != nil {
			break
		}

		if _, subnet, err := net.ParseCIDR(line); err == nil {
			// It's a network range, like 192.168.0.1/24.
			tlsBypassRanges = append(tlsBypassRanges, subnet)
			continue
		}

		tlsBypass[line] = true
	}

	return nil
}
<commit_msg>Allow bypassing a domain and all its subdomains around SSLBump.<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"log"
	"math/big"
	"net"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
)

// Intercept TLS (HTTPS) connections.

var certFile = flag.String("tls-cert", "", "path to certificate for serving HTTPS")
var keyFile = flag.String("tls-key", "", "path to TLS certificate key")
var sslBypassFile = newActiveFlag("tls-bypass", "", "path to list of sites that bypass SSLBump", readBypassFile)
var tlsVerbose = flag.Bool("tls-verbose", false, "log all intercepted HTTPS connections")

var tlsCert tls.Certificate
var parsedTLSCert *x509.Certificate
var tlsReady bool

var tlsBypass = map[string]bool{}
var tlsBypassRanges []*net.IPNet
var tlsBypassDomains []string

// unverifiedClientConfig is a TLS configuration that doesn't verify server
// certificates.
var unverifiedClientConfig = &tls.Config{
	InsecureSkipVerify: true,
}

// loadCertificate loads the TLS certificate specified by certFile and keyFile
// into tlsCert.
func loadCertificate() {
	if *certFile != "" && *keyFile != "" {
		cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
		if err != nil {
			log.Println("Error loading TLS certificate:", err)
			return
		}
		tlsCert = cert
		parsed, err := x509.ParseCertificate(cert.Certificate[0])
		if err != nil {
			log.Println("Error parsing X509 certificate:", err)
			return
		}
		parsedTLSCert = parsed
		tlsReady = true

		go cacheCertificates()
	}
}

func shouldBypass(host string) bool {
	if tlsBypass[host] {
		return true
	}
	for _, domain := range tlsBypassDomains {
		if strings.HasSuffix(host, domain) {
			return true
		}
	}
	addr := net.ParseIP(host)
	if addr == nil {
		return false
	}
	for _, subnet := range tlsBypassRanges {
		if subnet.Contains(addr) {
			return true
		}
	}
	return false
}

// SSLBump performs a man-in-the-middle attack on conn, to filter the HTTPS
// traffic. serverAddr is the address (host:port) of the server the client was
// trying to connect to.
func SSLBump(conn net.Conn, serverAddr string) {
	defer func() {
		if err := recover(); err != nil {
			buf := make([]byte, 4096)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("SSLBump: panic serving connection to %s: %v\n%s", serverAddr, err, buf)
			conn.Close()
		}
	}()

	if host, _, err := net.SplitHostPort(serverAddr); err == nil && shouldBypass(host) {
		// Just connect directly instead of bumping this connection.
		serverConn, err := net.Dial("tcp", serverAddr)
		if err != nil {
			log.Printf("error with pass-through of SSL connection to %s: %s", serverAddr, err)
			return
		}

		go func() {
			io.Copy(conn, serverConn)
			conn.Close()
		}()
		io.Copy(serverConn, conn)
		serverConn.Close()
		return
	}

	if *tlsVerbose {
		log.Printf("intercepting TLS connection from %s to %s", conn.RemoteAddr(), serverAddr)
	}

	cert, err := getCertificate(serverAddr)
	if err != nil {
		// Since it doesn't seem to be an HTTPS server, just connect directly.
		log.Printf("Could not generate a TLS certificate for %s (%s); letting the client connect directly", serverAddr, err)
		serverConn, err := net.Dial("tcp", serverAddr)
		if err != nil {
			log.Printf("Could not connect to %v: %v", serverAddr, err)
			return
		}

		go func() {
			io.Copy(conn, serverConn)
			conn.Close()
		}()
		io.Copy(serverConn, conn)
		serverConn.Close()
		return
	}

	config := &tls.Config{
		NextProtos:   []string{"http/1.1"},
		Certificates: []tls.Certificate{cert, tlsCert},
	}

	tlsConn := tls.Server(conn, config)
	err = tlsConn.Handshake()
	if err != nil {
		log.Printf("Error in TLS handshake for SSLBump connection from %v to %v: %v", conn.RemoteAddr(), serverAddr, err)
		return
	}

	_, port, err := net.SplitHostPort(serverAddr)
	if err != nil {
		port = ""
	}
	listener := &singleListener{conn: tlsConn}
	server := http.Server{
		Handler: proxyHandler{
			TLS:         true,
			connectPort: port,
		},
	}
	server.Serve(listener)
}

// A singleListener is a net.Listener that returns a single connection, then
// gives the error io.EOF.
type singleListener struct {
	conn net.Conn
	once sync.Once
}

func (s *singleListener) Accept() (net.Conn, error) {
	var c net.Conn
	s.once.Do(func() {
		c = s.conn
	})
	if c != nil {
		return c, nil
	}
	return nil, io.EOF
}

func (s *singleListener) Close() error {
	s.once.Do(func() {
		s.conn.Close()
	})
	return nil
}

func (s *singleListener) Addr() net.Addr {
	return s.conn.LocalAddr()
}

// maxSerial is the largest serial number to use for a certificate.
var maxSerial = big.NewInt(1<<63 - 1)

// generateCertificate connects to the server at addr, gets its TLS
// certificate, and returns a new certificate to be used when proxying
// connections to that server.
func generateCertificate(addr string) (tls.Certificate, error) {
	conn, err := tls.Dial("tcp", addr, unverifiedClientConfig)
	if err != nil {
		return tls.Certificate{}, err
	}
	defer conn.Close()
	state := conn.ConnectionState()
	serverCert := state.PeerCertificates[0]

	// Avoid duplicate serial numbers (NSS error -8054 in Chrome).
	serverCert.SerialNumber, err = rand.Int(rand.Reader, maxSerial)
	if err != nil {
		return tls.Certificate{}, fmt.Errorf("failed to generate serial number: %s", err)
	}

	priv, err := rsa.GenerateKey(rand.Reader, 1024)
	if err != nil {
		return tls.Certificate{}, fmt.Errorf("failed to generate private key: %s", err)
	}

	intermediates := x509.NewCertPool()
	for _, cert := range state.PeerCertificates[1:] {
		intermediates.AddCert(cert)
	}
	_, err = serverCert.Verify(x509.VerifyOptions{Intermediates: intermediates})
	signingCert := parsedTLSCert
	if _, ok := err.(x509.UnknownAuthorityError); ok {
		// There was a certificate error, so generate a self-signed certificate.
		signingCert = serverCert
	}

	newCertBytes, err := x509.CreateCertificate(rand.Reader, serverCert, signingCert, &priv.PublicKey, tlsCert.PrivateKey)
	if err != nil {
		return tls.Certificate{}, err
	}

	certBuf := new(bytes.Buffer)
	pem.Encode(certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: newCertBytes})
	keyBuf := new(bytes.Buffer)
	pem.Encode(keyBuf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})

	return tls.X509KeyPair(certBuf.Bytes(), keyBuf.Bytes())
}

func readBypassFile(filename string) error {
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("could not open %s: %s", filename, err)
	}
	defer f.Close()
	cr := newConfigReader(f)

	for {
		line, err := cr.ReadLine()
		if err != nil {
			break
		}

		if strings.HasPrefix(line, ".") {
			// This is a domain that we should bypass subdomains of too.
			tlsBypassDomains = append(tlsBypassDomains, line)
			continue
		}

		if _, subnet, err := net.ParseCIDR(line); err == nil {
			// It's a network range, like 192.168.0.1/24.
			tlsBypassRanges = append(tlsBypassRanges, subnet)
			continue
		}

		tlsBypass[line] = true
	}

	return nil
}
<|endoftext|>