<commit_before>package udpsender

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"time"
)

type RecvService struct {
	key          string
	ch           chan *RecvBlock
	transactions map[string]*RecvTransaction
	sock         *net.UDPConn
	fn           RecvHandleFunc
	disk         bool
	dir          string
}

type RecvTransaction struct {
	StartTime   int64
	num         int
	blocks      map[int]*RecvBlock
	lastTime    int64
	blockSize   int64
	Speed       float64
	completed   bool
	Addr        *net.UDPAddr
	Transaction []byte
}

type RecvBlock struct {
	transaction             []byte
	blockHash               []byte
	fileHash                []byte
	fileData                []byte
	num, id                 int32
	start, end, size, size1 int64
	time1                   int64
	addr                    *net.UDPAddr
}

type RecvHandleFunc func(service *RecvService, transaction *RecvTransaction)

func InitRecvService(key string, url string, fn RecvHandleFunc) (service *RecvService, err error) {
	service = new(RecvService)
	service.key = key
	service.fn = fn
	service.ch = make(chan *RecvBlock, 100)
	service.transactions = map[string]*RecvTransaction{}

	addr, err := net.ResolveUDPAddr("udp4", url)
	if err != nil {
		return nil, err
	}
	service.sock, err = net.ListenUDP("udp4", addr)
	if err != nil {
		return nil, err
	}
	go service.loop()

	return service, nil
}

func InitRecvTransaction(block *RecvBlock) *RecvTransaction {
	transaction := new(RecvTransaction)
	transaction.blocks = map[int]*RecvBlock{}
	transaction.lastTime = time.Now().UnixNano()
	transaction.StartTime = transaction.lastTime
	transaction.blockSize = block.size1
	transaction.num = int(block.num)
	transaction.Addr = block.addr
	transaction.Transaction = block.transaction
	return transaction
}
func (service *RecvService) EnableFileStorage(dir string) {
	service.disk = true
	service.dir = dir
}
func (service *RecvService) processRecvBlock(block *RecvBlock) {
	stransaction := hex.EncodeToString(block.transaction)
	transaction, ok := service.transactions[stransaction]
	if !ok {
		fmt.Println("Creating transaction:", stransaction)
		transaction = InitRecvTransaction(block)
		service.transactions[stransaction] = transaction
	}
	if transaction.num != int(block.num) {
		// error
	}
	block.time1 = time.Now().UnixNano()
	fdtime := float64(block.time1 - transaction.lastTime)
	transaction.lastTime = block.time1
	speed := float64(len(block.fileData)) / fdtime * float64(time.Second)
	transaction.Speed = movingExpAvg(speed, transaction.Speed, fdtime, float64(time.Second))

	_, ok = transaction.blocks[int(block.id)]
	if ok {
		// dupe
		return
	}
	transaction.blocks[int(block.id)] = block
	// fmt.Printf("got %d %d %d\n%s\n", block.id, len(transaction.blocks), transaction.num, string(block.fileData))
	fmt.Printf("got %d %d %d %f\n", block.id, len(transaction.blocks), transaction.num, transaction.Speed)
	if len(transaction.blocks) == transaction.num {
		transaction.completed = true
		service.fn(service, transaction)

		// if service.disk {

		// 	os.MkdirAll(service.dir+"/"+stransaction, 0700)

		// 	f1, err := os.Create(service.dir + "/" + stransaction + "/file.out")
		// 	if err != nil {
		// 		// panic(err)
		// 		return
		// 	}
		// 	// defer f1.Close()

		// 	for i := 1; i <= transaction.num; i++ {
		// 		block1, ok := transaction.blocks[i]
		// 		if !ok {
		// 			// error
		// 			panic(errors.New(fmt.Sprintf("Not all blocks %d", i)))
		// 		}
		// 		f1.Write(block1.fileData)
		// 	}
		// 	f1.Close()
		// 	block1.fileData = nil
		// }

	}
}
func (service *RecvService) WriteTo(transaction *RecvTransaction, w io.Writer) error {
	if service.disk {
		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				return errors.New(fmt.Sprintf("Not all blocks %d", i))
			}
			// w.Write(block1.fileData)

			f1, err := os.Open(service.dir + "/" + hex.EncodeToString(block1.transaction) + "/" + fmt.Sprintf("%d.data", block1.id))
			if err != nil {
				// panic(err)
				return err
			}
			io.Copy(w, f1)
		}
	} else {
		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				return errors.New(fmt.Sprintf("Not all blocks %d", i))
			}
			w.Write(block1.fileData)
		}
	}

	return nil
}
func (service *RecvService) Stop() {
	service.sock.Close()
}
func (service *RecvService) Loop() error {

	for {
		// len, remote, err := sock.ReadFromUDP(buf[:])
		var buf [1 << 16]byte
		len, remote, err := service.sock.ReadFromUDP(buf[:])
		if err != nil {
			// 	(err)
			return err
		}
		fmt.Print("+")
		service.processPacket(remote, buf[:len])
	}
}
func (service *RecvService) loop() {

	ticker := time.Tick(100 * time.Millisecond)
	var stopTickerTill int64

	for {

		select {
		case blck, ok := <-service.ch:
			if ok {
				service.processRecvBlock(blck)
			}
		case <-ticker:
			time1 := time.Now().UnixNano()
			if stopTickerTill > time1 {
				continue
			}
			for _, transaction := range service.transactions {
				if transaction.completed {
					continue
				}

				fdtime := float64(time1 - transaction.lastTime)
				transaction.lastTime = time1
				transaction.Speed = movingExpAvg(0.0, transaction.Speed, fdtime, float64(time.Second))

				//
				// find last_block
				last_block := -1
				var block *RecvBlock
				for i, _block := range transaction.blocks {
					if last_block < i {
						last_block = i
						block = _block
					}
				}
				if last_block == -1 {
					continue
				}

				rangespeed := transaction.Speed

				if rangespeed < 100000 {
					rangespeed = 100000
				}
				if rangespeed > 10000000 {
					rangespeed = 10000000
				}

				// dtime  = Now - last_block.time
				dtime := time1 - block.time1
				// last_block + (dtime-100ms)*speed/blocksize
				shift := last_block + int((float64(dtime)-100.0*float64(time.Millisecond))*rangespeed/float64(transaction.blockSize)/float64(time.Second))
				// get first 100 un recieved blocks

				req := []int{}

				for i := 1; i <= transaction.num; i++ {
					if i > shift {
						break
					}
					_, ok := transaction.blocks[i]
					if !ok {
						req = append(req, i)
						if len(req) >= 100 {
							break
						}
					}
				}

				// fmt.Printf(":%d %d#", len(req), shift)
				fmt.Println("@", shift, last_block, dtime/int64(time.Millisecond), dtime, rangespeed, float64(transaction.blockSize))

				if len(req) > 0 {
					service.rerequestPackets(transaction, req)
					stopTickerTill = time1 + int64(time.Second)
				}
			}
		}

	}

}
func (service *RecvService) rerequestPackets(transaction *RecvTransaction, req []int) error {
	fmt.Printf("rerequestPackets %#v\n", req)
	r1 := make([]byte, 16)
	_, err := rand.Read(r1)
	if err != nil {
		// panic(err)
		return err
	}

	data := append(transaction.Transaction, r1...)

	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, int32(1))
	binary.Write(buffer, binary.LittleEndian, int32(len(req)))
	for _, i := range req {
		binary.Write(buffer, binary.LittleEndian, int32(i))
	}

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		// panic(err)
		return err
	}

	data1 := buffer.Bytes()

	stream := cipher.NewCFBEncrypter(block, r1)
	stream.XORKeyStream(data1, data1)

	data = append(data, data1...)

	service.sock.WriteToUDP(data, transaction.Addr)
	return nil
}
func (service *RecvService) processPacket(addr *net.UDPAddr, buf []byte) error {
	blck := new(RecvBlock)
	blck.transaction = buf[:16]
	blck.addr = addr
	r1 := buf[16:32]
	data2 := buf[32:]

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		// panic(err)
		return err
	}

	stream := cipher.NewCFBDecrypter(block, r1)

	stream.XORKeyStream(data2, data2)

	// fileHash := data2[0:20]

	buffer := bytes.NewBuffer(data2[20:52])
	binary.Read(buffer, binary.LittleEndian, &blck.id)
	binary.Read(buffer, binary.LittleEndian, &blck.num)
	binary.Read(buffer, binary.LittleEndian, &blck.start)
	binary.Read(buffer, binary.LittleEndian, &blck.end)
	binary.Read(buffer, binary.LittleEndian, &blck.size)
	blck.blockHash = data2[52:72]
	blck.size1 = int64(len(data2[72:]))
	bufCopy := make([]byte, blck.size1)
	copy(bufCopy[:], data2[72:])
	blck.fileData = bufCopy
	blockHash := sha1.Sum(blck.fileData)
	verified := bytes.Equal(blck.blockHash, blockHash[:])

	fmt.Println("processPacker:", blck.id, hex.EncodeToString(r1))

	if verified {
		if service.disk {
			os.MkdirAll(service.dir+"/"+hex.EncodeToString(blck.transaction), 0700)
			f1, err := os.Create(service.dir + "/" + hex.EncodeToString(blck.transaction) + "/" + fmt.Sprintf("%d.data", blck.id))
			if err != nil {
				// panic(err)
				return err
			}
			defer f1.Close()
			f1.Write(blck.fileData)
			blck.fileData = nil
		}
		service.ch <- blck
	} else {
		fmt.Println("wrong hash %#v %#v", blck.blockHash, blockHash[:])
	}
	return nil
}
<commit_msg>time optimization<commit_after>package udpsender

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha1"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"time"
)

type RecvService struct {
	key          string
	ch           chan *RecvBlock
	transactions map[string]*RecvTransaction
	sock         *net.UDPConn
	fn           RecvHandleFunc
	disk         bool
	dir          string
}

type RecvTransaction struct {
	StartTime     int64
	talkBackTime  int64
	num           int
	blocks        map[int]*RecvBlock
	lastTime      int64
	blockSize     int64
	Speed, speed1 float64
	completed     bool
	Addr          *net.UDPAddr
	Transaction   []byte
}

type RecvBlock struct {
	transaction             []byte
	blockHash               []byte
	fileHash                []byte
	fileData                []byte
	num, id                 int32
	start, end, size, size1 int64
	time1                   int64
	addr                    *net.UDPAddr
}

type RecvHandleFunc func(service *RecvService, transaction *RecvTransaction)

func InitRecvService(key string, url string, fn RecvHandleFunc) (service *RecvService, err error) {
	service = new(RecvService)
	service.key = key
	service.fn = fn
	service.ch = make(chan *RecvBlock, 100)
	service.transactions = map[string]*RecvTransaction{}

	addr, err := net.ResolveUDPAddr("udp4", url)
	if err != nil {
		return nil, err
	}
	service.sock, err = net.ListenUDP("udp4", addr)
	if err != nil {
		return nil, err
	}
	go service.loop()

	return service, nil
}

func InitRecvTransaction(block *RecvBlock) *RecvTransaction {
	transaction := new(RecvTransaction)
	transaction.blocks = map[int]*RecvBlock{}
	transaction.lastTime = time.Now().UnixNano()
	transaction.StartTime = transaction.lastTime
	transaction.talkBackTime = transaction.lastTime + int64(time.Second)
	transaction.blockSize = block.size1
	transaction.num = int(block.num)
	transaction.Addr = block.addr
	transaction.Transaction = block.transaction
	return transaction
}
func (service *RecvService) EnableFileStorage(dir string) {
	service.disk = true
	service.dir = dir
}
func (service *RecvService) processRecvBlock(block *RecvBlock) {
	stransaction := hex.EncodeToString(block.transaction)
	transaction, ok := service.transactions[stransaction]
	if !ok {
		fmt.Println("Creating transaction:", stransaction)
		transaction = InitRecvTransaction(block)
		service.transactions[stransaction] = transaction
	}
	if transaction.num != int(block.num) {
		// error
	}
	block.time1 = time.Now().UnixNano()
	fdtime := float64(block.time1 - transaction.lastTime)
	transaction.lastTime = block.time1
	speed := float64(len(block.fileData)) / fdtime * float64(time.Second)
	transaction.Speed = movingExpAvg(speed, transaction.Speed, fdtime, float64(time.Second))

	_, ok = transaction.blocks[int(block.id)]
	if ok {
		// dupe
		return
	}
	transaction.blocks[int(block.id)] = block
	// fmt.Printf("got %d %d %d\n%s\n", block.id, len(transaction.blocks), transaction.num, string(block.fileData))
	fmt.Printf("got %d %d %d %f\n", block.id, len(transaction.blocks), transaction.num, transaction.Speed)
	if len(transaction.blocks) == transaction.num {
		transaction.completed = true
		service.fn(service, transaction)

		// if service.disk {

		// 	os.MkdirAll(service.dir+"/"+stransaction, 0700)

		// 	f1, err := os.Create(service.dir + "/" + stransaction + "/file.out")
		// 	if err != nil {
		// 		// panic(err)
		// 		return
		// 	}
		// 	// defer f1.Close()

		// 	for i := 1; i <= transaction.num; i++ {
		// 		block1, ok := transaction.blocks[i]
		// 		if !ok {
		// 			// error
		// 			panic(errors.New(fmt.Sprintf("Not all blocks %d", i)))
		// 		}
		// 		f1.Write(block1.fileData)
		// 	}
		// 	f1.Close()
		// 	block1.fileData = nil
		// }

	}
}
func (service *RecvService) WriteTo(transaction *RecvTransaction, w io.Writer) error {
	if service.disk {
		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				return errors.New(fmt.Sprintf("Not all blocks %d", i))
			}
			// w.Write(block1.fileData)

			f1, err := os.Open(service.dir + "/" + hex.EncodeToString(block1.transaction) + "/" + fmt.Sprintf("%d.data", block1.id))
			if err != nil {
				// panic(err)
				return err
			}
			io.Copy(w, f1)
		}
	} else {
		for i := 1; i <= transaction.num; i++ {
			block1, ok := transaction.blocks[i]
			if !ok {
				// error
				return errors.New(fmt.Sprintf("Not all blocks %d", i))
			}
			w.Write(block1.fileData)
		}
	}

	return nil
}
func (service *RecvService) Stop() {
	service.sock.Close()
}
func (service *RecvService) Loop() error {

	for {
		// len, remote, err := sock.ReadFromUDP(buf[:])
		var buf [1 << 16]byte
		len, remote, err := service.sock.ReadFromUDP(buf[:])
		if err != nil {
			// 	(err)
			return err
		}
		fmt.Print("+")
		service.processPacket(remote, buf[:len])
	}
}
func (service *RecvService) loop() {

	ticker := time.Tick(100 * time.Millisecond)

	for {

		select {
		case blck, ok := <-service.ch:
			if ok {
				service.processRecvBlock(blck)
			}
		case <-ticker:
			time1 := time.Now().UnixNano()

			for _, transaction := range service.transactions {
				if transaction.completed {
					continue
				}

				if time1 < transaction.talkBackTime {
					continue
				}

				if transaction.Speed > transaction.speed1 {
					continue
				}

				transaction.speed1 = transaction.Speed

				fdtime := float64(time1 - transaction.lastTime)
				transaction.lastTime = time1
				transaction.Speed = movingExpAvg(0.0, transaction.Speed, fdtime, float64(time.Second))

				//
				// find last_block
				last_block := -1
				var block *RecvBlock
				for i, _block := range transaction.blocks {
					if last_block < i {
						last_block = i
						block = _block
					}
				}
				if last_block == -1 {
					continue
				}

				rangespeed := transaction.Speed

				if rangespeed < 100000 {
					rangespeed = 100000
				}
				if rangespeed > 10000000 {
					rangespeed = 10000000
				}

				// dtime  = Now - last_block.time
				dtime := time1 - block.time1
				// last_block + (dtime-100ms)*speed/blocksize
				shift := last_block + int((float64(dtime)-100.0*float64(time.Millisecond))*rangespeed/float64(transaction.blockSize)/float64(time.Second))
				// get first 100 un recieved blocks

				req := []int{}

				for i := 1; i <= transaction.num; i++ {
					if i > shift {
						break
					}
					_, ok := transaction.blocks[i]
					if !ok {
						req = append(req, i)
						if len(req) >= 100 {
							break
						}
					}
				}

				// fmt.Printf(":%d %d#", len(req), shift)
				fmt.Println("@", shift, last_block, dtime/int64(time.Millisecond), dtime, rangespeed, float64(transaction.blockSize))

				if len(req) > 0 {
					service.rerequestPackets(transaction, req)
					transaction.talkBackTime = time1 + int64(300*time.Millisecond)
				}
			}
		}

	}

}
func (service *RecvService) rerequestPackets(transaction *RecvTransaction, req []int) error {
	fmt.Printf("rerequestPackets %#v\n", req)
	r1 := make([]byte, 16)
	_, err := rand.Read(r1)
	if err != nil {
		// panic(err)
		return err
	}

	data := append(transaction.Transaction, r1...)

	buffer := new(bytes.Buffer)
	binary.Write(buffer, binary.LittleEndian, int32(1))
	binary.Write(buffer, binary.LittleEndian, int32(len(req)))
	for _, i := range req {
		binary.Write(buffer, binary.LittleEndian, int32(i))
	}

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		// panic(err)
		return err
	}

	data1 := buffer.Bytes()

	stream := cipher.NewCFBEncrypter(block, r1)
	stream.XORKeyStream(data1, data1)

	data = append(data, data1...)

	service.sock.WriteToUDP(data, transaction.Addr)
	return nil
}
func (service *RecvService) processPacket(addr *net.UDPAddr, buf []byte) error {
	blck := new(RecvBlock)
	blck.transaction = buf[:16]
	blck.addr = addr
	r1 := buf[16:32]
	data2 := buf[32:]

	block, err := aes.NewCipher([]byte(service.key))
	if err != nil {
		// panic(err)
		return err
	}

	stream := cipher.NewCFBDecrypter(block, r1)

	stream.XORKeyStream(data2, data2)

	// fileHash := data2[0:20]

	buffer := bytes.NewBuffer(data2[20:52])
	binary.Read(buffer, binary.LittleEndian, &blck.id)
	binary.Read(buffer, binary.LittleEndian, &blck.num)
	binary.Read(buffer, binary.LittleEndian, &blck.start)
	binary.Read(buffer, binary.LittleEndian, &blck.end)
	binary.Read(buffer, binary.LittleEndian, &blck.size)
	blck.blockHash = data2[52:72]
	blck.size1 = int64(len(data2[72:]))
	bufCopy := make([]byte, blck.size1)
	copy(bufCopy[:], data2[72:])
	blck.fileData = bufCopy
	blockHash := sha1.Sum(blck.fileData)
	verified := bytes.Equal(blck.blockHash, blockHash[:])

	fmt.Println("processPacker:", blck.id, hex.EncodeToString(r1))

	if verified {
		if service.disk {
			os.MkdirAll(service.dir+"/"+hex.EncodeToString(blck.transaction), 0700)
			f1, err := os.Create(service.dir + "/" + hex.EncodeToString(blck.transaction) + "/" + fmt.Sprintf("%d.data", blck.id))
			if err != nil {
				// panic(err)
				return err
			}
			defer f1.Close()
			f1.Write(blck.fileData)
			blck.fileData = nil
		}
		service.ch <- blck
	} else {
		fmt.Println("wrong hash %#v %#v", blck.blockHash, blockHash[:])
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/fzzy/radix/redis"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"time"
)

var redis_location = flag.String("redis", "127.0.0.1:6379", "Location of redis instance")
var server_bind = flag.String("bind", "127.0.0.1:5000", "Location server should listen at")
var handler_mount = flag.String("mount", "/", "Relative path where handler should be at")

var client *redis.Client

type RequestInfo struct {
	Query     string
	UserAgent string
	Time      int64
}

func initStorage(db int) {
	var err error
	client, err = redis.DialTimeout("tcp", *redis_location, time.Duration(10)*time.Second)
	errHndlr(err)
	client.Cmd("SELECT", db)
}

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		os.Exit(1)
	}
}

func httpStore(res http.ResponseWriter, req *http.Request) {
	ri := RequestInfo{
		Query:     req.URL.RawQuery,
		UserAgent: req.UserAgent(),
		Time:      time.Now().Unix(),
	}

	if ri.Query != "" {
		b, err := json.Marshal(ri)
		errHndlr(err)
		client.Cmd("RPUSH", "incoming", string(b))
	}

	_, filename, _, _ := runtime.Caller(0)
	beacon, err := ioutil.ReadFile(path.Join(path.Dir(filename), "../assets/1x1.gif"))
	errHndlr(err)

	res.Header().Set("Content-Type", "image/gif")
	res.Write(beacon)
}

func main() {
	flag.Parse()
	initStorage(1)
	http.HandleFunc(*handler_mount, httpStore)
	err := http.ListenAndServe(*server_bind, nil)
	errHndlr(err)
	defer client.Close()
}
<commit_msg>Correcting path to assets<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/fzzy/radix/redis"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"runtime"
	"time"
)

var redis_location = flag.String("redis", "127.0.0.1:6379", "Location of redis instance")
var server_bind = flag.String("bind", "127.0.0.1:5000", "Location server should listen at")
var handler_mount = flag.String("mount", "/", "Relative path where handler should be at")

var client *redis.Client

type RequestInfo struct {
	Query     string
	UserAgent string
	Time      int64
}

func initStorage(db int) {
	var err error
	client, err = redis.DialTimeout("tcp", *redis_location, time.Duration(10)*time.Second)
	errHndlr(err)
	client.Cmd("SELECT", db)
}

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		os.Exit(1)
	}
}

func httpStore(res http.ResponseWriter, req *http.Request) {
	ri := RequestInfo{
		Query:     req.URL.RawQuery,
		UserAgent: req.UserAgent(),
		Time:      time.Now().Unix(),
	}

	if ri.Query != "" {
		b, err := json.Marshal(ri)
		errHndlr(err)
		client.Cmd("RPUSH", "incoming", string(b))
	}

	_, filename, _, _ := runtime.Caller(0)
	beacon, err := ioutil.ReadFile(path.Join(path.Dir(filename), "assets/1x1.gif"))
	errHndlr(err)

	res.Header().Set("Content-Type", "image/gif")
	res.Write(beacon)
}

func main() {
	flag.Parse()
	initStorage(1)
	http.HandleFunc(*handler_mount, httpStore)
	err := http.ListenAndServe(*server_bind, nil)
	errHndlr(err)
	defer client.Close()
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     July 03, 2019
;       related version of root zone:     2019070301
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>auto-update<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     July 25, 2019
;       related version of root zone:     2019072501
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>package rope

import (
	"bytes"
	"io"
	"math"
	"strings"
	"sync"
	"sync/atomic"
	"unicode/utf8"
)

// Key -> *Rope
//TODO eviction
var cache sync.Map

type Key struct {
	left    *Rope
	right   *Rope
	content string
}

type Rope struct {
	left     *Rope
	right    *Rope
	content  []byte
	serial   int64
	height   int
	weight   int
	balanced bool
}

var nextSerial int64

var MaxLengthPerNode = 128

func NewFromReader(r io.Reader) (ret *Rope, err error) {
	slots := make([]*Rope, 64)

	for {
		buf := make([]byte, MaxLengthPerNode)
		l, err := r.Read(buf)
		if l == 0 {
			if err == io.EOF {
				break
			}
			if err != nil {
				return nil, err
			}
		}

		key := Key{
			content: string(buf[:l]),
		}
		var rope *Rope
		if v, ok := cache.Load(key); ok {
			rope = v.(*Rope)
		} else {
			rope = &Rope{
				content:  buf[:l],
				serial:   atomic.AddInt64(&nextSerial, 1),
				height:   1,
				weight:   l,
				balanced: l == len(buf),
			}
			cache.Store(key, rope)
		}

		slotIndex := 0
		for slots[slotIndex] != nil {
			key := Key{
				left:  slots[slotIndex],
				right: rope,
			}
			if v, ok := cache.Load(key); ok {
				rope = v.(*Rope)
			} else {
				rope = &Rope{
					left:     slots[slotIndex],
					right:    rope,
					serial:   atomic.AddInt64(&nextSerial, 1),
					height:   slotIndex + 2,
					weight:   slots[slotIndex].Len(),
					balanced: true,
				}
				cache.Store(key, rope)
			}
			slots[slotIndex] = nil
			slotIndex++
		}
		slots[slotIndex] = rope

		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
	}

	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func NewFromString(s string) *Rope {
	r, err := NewFromReader(strings.NewReader(s))
	if err != nil {
		panic(err)
	}
	return r
}

func NewFromBytes(bs []byte) *Rope {
	r, err := NewFromReader(bytes.NewReader(bs))
	if err != nil {
		panic(err)
	}
	return r
}

func (r *Rope) Index(i int) byte {
	if i >= r.weight {
		return r.right.Index(i - r.weight)
	}
	if r.left != nil { // non leaf
		return r.left.Index(i)
	}
	// leaf
	return r.content[i]
}

func (r *Rope) Len() int {
	if r == nil {
		return 0
	}
	return r.weight + r.right.Len()
}

func (r *Rope) Bytes() []byte {
	ret := make([]byte, r.Len())
	i := 0
	r.Iter(0, func(bs []byte) bool {
		copy(ret[i:], bs)
		i += len(bs)
		return true
	})
	return ret
}

func (r *Rope) Concat(r2 *Rope) (ret *Rope) {
	key := Key{
		left:  r,
		right: r2,
	}
	if v, ok := cache.Load(key); ok {
		return v.(*Rope)
	}
	ret = &Rope{
		left:   r,
		right:  r2,
		serial: atomic.AddInt64(&nextSerial, 1),
		weight: r.Len(),
	}
	if ret.left != nil {
		ret.height = ret.left.height
	}
	if ret.right != nil && ret.right.height > ret.height {
		ret.height = ret.right.height
	}
	if ret.left != nil && ret.left.balanced &&
		ret.right != nil && ret.right.balanced &&
		ret.left.height == ret.right.height {
		ret.balanced = true
	}
	ret.height++
	// check and rebalance
	if !ret.balanced {
		l := int((math.Ceil(math.Log2(float64((ret.Len()/MaxLengthPerNode)+1))) + 1) * 1.5)
		if ret.height > l {
			ret = ret.rebalance()
		}
	}
	cache.Store(key, ret)
	return
}

func (r *Rope) rebalance() (ret *Rope) {
	var currentBytes []byte
	slots := make([]*Rope, 32)
	r.iterNodes(func(node *Rope) bool {
		var balancedNode *Rope
		iterSubNodes := true
		if len(currentBytes) == 0 && node.balanced { // balanced, insert to slots
			balancedNode = node
			iterSubNodes = false
		} else { // collect bytes
			currentBytes = append(currentBytes, node.content...)
			if len(currentBytes) >= MaxLengthPerNode { // a full leaf
				key := Key{
					content: string(currentBytes[:MaxLengthPerNode]),
				}
				if v, ok := cache.Load(key); ok {
					balancedNode = v.(*Rope)
				} else {
					balancedNode = &Rope{
						content:  currentBytes[:MaxLengthPerNode],
						serial:   atomic.AddInt64(&nextSerial, 1),
						height:   1,
						weight:   MaxLengthPerNode,
						balanced: true,
					}
					cache.Store(key, balancedNode)
				}
				currentBytes = currentBytes[MaxLengthPerNode:]
			}
		}
		if balancedNode != nil {
			slotIndex := balancedNode.height - 1
			for slots[slotIndex] != nil {
				key := Key{
					left:  slots[slotIndex],
					right: balancedNode,
				}
				if v, ok := cache.Load(key); ok {
					balancedNode = v.(*Rope)
				} else {
					balancedNode = &Rope{
						left:     slots[slotIndex],
						right:    balancedNode,
						serial:   atomic.AddInt64(&nextSerial, 1),
						height:   balancedNode.height + 1,
						weight:   slots[slotIndex].Len(),
						balanced: true,
					}
					cache.Store(key, balancedNode)
				}
				slots[slotIndex] = nil
				slotIndex++
			}
			slots[slotIndex] = balancedNode
		}
		return iterSubNodes
	})
	if len(currentBytes) > 0 {
		key := Key{
			content: string(currentBytes),
		}
		if v, ok := cache.Load(key); ok {
			ret = v.(*Rope)
		} else {
			ret = &Rope{
				content:  currentBytes,
				serial:   atomic.AddInt64(&nextSerial, 1),
				height:   1,
				weight:   len(currentBytes),
				balanced: false,
			}
			cache.Store(key, ret)
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Split(n int) (out1, out2 *Rope) {
	if r == nil {
		return
	}
	if len(r.content) > 0 { // leaf
		if n > len(r.content) { // offset overflow
			n = len(r.content)
		}
		out1 = NewFromBytes(r.content[:n])
		out2 = NewFromBytes(r.content[n:])
	} else { // non leaf
		var r1 *Rope
		if n >= r.weight { // at right subtree
			r1, out2 = r.right.Split(n - r.weight)
			out1 = r.left.Concat(r1)
		} else { // at left subtree
			out1, r1 = r.left.Split(n)
			out2 = r1.Concat(r.right)
		}
	}
	return
}

func (r *Rope) Insert(n int, bs []byte) *Rope {
	r1, r2 := r.Split(n)
	return r1.Concat(NewFromBytes(bs)).Concat(r2)
}

func (r *Rope) Delete(n, l int) *Rope {
	r1, r2 := r.Split(n)
	_, r2 = r2.Split(l)
	return r1.Concat(r2)
}

func (r *Rope) Sub(n, l int) []byte {
	ret := make([]byte, l)
	i := 0
	r.Iter(n, func(bs []byte) bool {
		if l >= len(bs) {
			copy(ret[i:], bs)
			i += len(bs)
			l -= len(bs)
			return true
		} else {
			copy(ret[i:], bs[:l])
			i += l
			return false
		}
	})
	return ret[:i]
}

func (r *Rope) Iter(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		if offset < len(r.content) {
			if !fn(r.content[offset:]) {
				return false
			}
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.Iter(offset-r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.Iter(offset, fn) {
				return false
			}
			if !r.right.Iter(0, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) IterBackward(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		content := r.content[:offset]
		if len(content) == 0 {
			return true
		}
		bs := reversedBytes(content)
		if !fn(bs) {
			return false
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.IterBackward(offset-r.weight, fn) {
				return false
			}
			if !r.left.IterBackward(r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.IterBackward(offset, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) iterNodes(fn func(*Rope) bool) {
	if r == nil {
		return
	}
	if fn(r) {
		r.left.iterNodes(fn)
		r.right.iterNodes(fn)
	}
}

func (r *Rope) IterRune(offset int, fn func(rune, int) bool) {
	var bs []byte
	stopped := false
	r.Iter(offset, func(slice []byte) bool {
		bs = append(bs, slice...)
		for len(bs) >= 4 {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru == utf8.RuneError {
				return false
			}
			if !fn(ru, l) {
				stopped = true
				return false
			}
		}
		return true
	})
	if !stopped && len(bs) > 0 {
		for {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru != utf8.RuneError {
				fn(ru, l)
			} else {
				break
			}
		}
	}
}
<commit_msg>fix slot size<commit_after>package rope

import (
	"bytes"
	"io"
	"math"
	"strings"
	"sync"
	"sync/atomic"
	"unicode/utf8"
)

// Key -> *Rope
//TODO eviction
var cache sync.Map

type Key struct {
	left    *Rope
	right   *Rope
	content string
}

type Rope struct {
	left     *Rope
	right    *Rope
	content  []byte
	serial   int64
	height   int
	weight   int
	balanced bool
}

var nextSerial int64

var MaxLengthPerNode = 128

func NewFromReader(r io.Reader) (ret *Rope, err error) {
	slots := make([]*Rope, 32)

	for {
		buf := make([]byte, MaxLengthPerNode)
		l, err := r.Read(buf)
		if l == 0 {
			if err == io.EOF {
				break
			}
			if err != nil {
				return nil, err
			}
		}

		key := Key{
			content: string(buf[:l]),
		}
		var rope *Rope
		if v, ok := cache.Load(key); ok {
			rope = v.(*Rope)
		} else {
			rope = &Rope{
				content:  buf[:l],
				serial:   atomic.AddInt64(&nextSerial, 1),
				height:   1,
				weight:   l,
				balanced: l == len(buf),
			}
			cache.Store(key, rope)
		}

		slotIndex := 0
		for slots[slotIndex] != nil {
			key := Key{
				left:  slots[slotIndex],
				right: rope,
			}
			if v, ok := cache.Load(key); ok {
				rope = v.(*Rope)
			} else {
				rope = &Rope{
					left:     slots[slotIndex],
					right:    rope,
					serial:   atomic.AddInt64(&nextSerial, 1),
					height:   slotIndex + 2,
					weight:   slots[slotIndex].Len(),
					balanced: true,
				}
				cache.Store(key, rope)
			}
			slots[slotIndex] = nil
			slotIndex++
		}
		slots[slotIndex] = rope

		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
	}

	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func NewFromString(s string) *Rope {
	r, err := NewFromReader(strings.NewReader(s))
	if err != nil {
		panic(err)
	}
	return r
}

func NewFromBytes(bs []byte) *Rope {
	r, err := NewFromReader(bytes.NewReader(bs))
	if err != nil {
		panic(err)
	}
	return r
}

func (r *Rope) Index(i int) byte {
	if i >= r.weight {
		return r.right.Index(i - r.weight)
	}
	if r.left != nil { // non leaf
		return r.left.Index(i)
	}
	// leaf
	return r.content[i]
}

func (r *Rope) Len() int {
	if r == nil {
		return 0
	}
	return r.weight + r.right.Len()
}

func (r *Rope) Bytes() []byte {
	ret := make([]byte, r.Len())
	i := 0
	r.Iter(0, func(bs []byte) bool {
		copy(ret[i:], bs)
		i += len(bs)
		return true
	})
	return ret
}

func (r *Rope) Concat(r2 *Rope) (ret *Rope) {
	key := Key{
		left:  r,
		right: r2,
	}
	if v, ok := cache.Load(key); ok {
		return v.(*Rope)
	}
	ret = &Rope{
		left:   r,
		right:  r2,
		serial: atomic.AddInt64(&nextSerial, 1),
		weight: r.Len(),
	}
	if ret.left != nil {
		ret.height = ret.left.height
	}
	if ret.right != nil && ret.right.height > ret.height {
		ret.height = ret.right.height
	}
	if ret.left != nil && ret.left.balanced &&
		ret.right != nil && ret.right.balanced &&
		ret.left.height == ret.right.height {
		ret.balanced = true
	}
	ret.height++
	// check and rebalance
	if !ret.balanced {
		l := int((math.Ceil(math.Log2(float64((ret.Len()/MaxLengthPerNode)+1))) + 1) * 1.5)
		if ret.height > l {
			ret = ret.rebalance()
		}
	}
	cache.Store(key, ret)
	return
}

func (r *Rope) rebalance() (ret *Rope) {
	var currentBytes []byte
	slots := make([]*Rope, 64)
	r.iterNodes(func(node *Rope) bool {
		var balancedNode *Rope
		iterSubNodes := true
		if len(currentBytes) == 0 && node.balanced { // balanced, insert to slots
			balancedNode = node
			iterSubNodes = false
		} else { // collect bytes
			currentBytes = append(currentBytes, node.content...)
			if len(currentBytes) >= MaxLengthPerNode { // a full leaf
				key := Key{
					content: string(currentBytes[:MaxLengthPerNode]),
				}
				if v, ok := cache.Load(key); ok {
					balancedNode = v.(*Rope)
				} else {
					balancedNode = &Rope{
						content:  currentBytes[:MaxLengthPerNode],
						serial:   atomic.AddInt64(&nextSerial, 1),
						height:   1,
						weight:   MaxLengthPerNode,
						balanced: true,
					}
					cache.Store(key, balancedNode)
				}
				currentBytes = currentBytes[MaxLengthPerNode:]
			}
		}
		if balancedNode != nil {
			slotIndex := balancedNode.height - 1
			for slots[slotIndex] != nil {
				key := Key{
					left:  slots[slotIndex],
					right: balancedNode,
				}
				if v, ok := cache.Load(key); ok {
					balancedNode = v.(*Rope)
				} else {
					balancedNode = &Rope{
						left:     slots[slotIndex],
						right:    balancedNode,
						serial:   atomic.AddInt64(&nextSerial, 1),
						height:   balancedNode.height + 1,
						weight:   slots[slotIndex].Len(),
						balanced: true,
					}
					cache.Store(key, balancedNode)
				}
				slots[slotIndex] = nil
				slotIndex++
			}
			slots[slotIndex] = balancedNode
		}
		return iterSubNodes
	})
	if len(currentBytes) > 0 {
		key := Key{
			content: string(currentBytes),
		}
		if v, ok := cache.Load(key); ok {
			ret = v.(*Rope)
		} else {
			ret = &Rope{
				content:  currentBytes,
				serial:   atomic.AddInt64(&nextSerial, 1),
				height:   1,
				weight:   len(currentBytes),
				balanced: false,
			}
			cache.Store(key, ret)
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Split(n int) (out1, out2 *Rope) {
	if r == nil {
		return
	}
	if len(r.content) > 0 { // leaf
		if n > len(r.content) { // offset overflow
			n = len(r.content)
		}
		out1 = NewFromBytes(r.content[:n])
		out2 = NewFromBytes(r.content[n:])
	} else { // non leaf
		var r1 *Rope
		if n >= r.weight { // at right subtree
			r1, out2 = r.right.Split(n - r.weight)
			out1 = r.left.Concat(r1)
		} else { // at left subtree
			out1, r1 = r.left.Split(n)
			out2 = r1.Concat(r.right)
		}
	}
	return
}

func (r *Rope) Insert(n int, bs []byte) *Rope {
	r1, r2 := r.Split(n)
	return r1.Concat(NewFromBytes(bs)).Concat(r2)
}

func (r *Rope) Delete(n, l int) *Rope {
	r1, r2 := r.Split(n)
	_, r2 = r2.Split(l)
	return r1.Concat(r2)
}

func (r *Rope) Sub(n, l int) []byte {
	ret := make([]byte, l)
	i := 0
	r.Iter(n, func(bs []byte) bool {
		if l >= len(bs) {
			copy(ret[i:], bs)
			i += len(bs)
			l -= len(bs)
			return true
		} else {
			copy(ret[i:], bs[:l])
			i += l
			return false
		}
	})
	return ret[:i]
}

func (r *Rope) Iter(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		if offset < len(r.content) {
			if !fn(r.content[offset:]) {
				return false
			}
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.Iter(offset-r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.Iter(offset, fn) {
				return false
			}
			if !r.right.Iter(0, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) IterBackward(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		content := r.content[:offset]
		if len(content) == 0 {
			return true
		}
		bs := reversedBytes(content)
		if !fn(bs) {
			return false
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.IterBackward(offset-r.weight, fn) {
				return false
			}
			if !r.left.IterBackward(r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.IterBackward(offset, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) iterNodes(fn func(*Rope) bool) {
	if r == nil {
		return
	}
	if fn(r) {
		r.left.iterNodes(fn)
		r.right.iterNodes(fn)
	}
}

func (r *Rope) IterRune(offset int, fn func(rune, int) bool) {
	var bs []byte
	stopped := false
	r.Iter(offset, func(slice []byte) bool {
		bs = append(bs, slice...)
		for len(bs) >= 4 {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru == utf8.RuneError {
				return false
			}
			if !fn(ru, l) {
				stopped = true
				return false
			}
		}
		return true
	})
	if !stopped && len(bs) > 0 {
		for {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru != utf8.RuneError {
				fn(ru, l)
			} else {
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	s.log.Debugf("doRetransmit for %x", *msg.ID)
	msg.Retransmissions++
	s.log.Debugf("retransmissions: %d", msg.Retransmissions)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		msg.SURBID = &surbID
		idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend with SURB ID %x", idStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend without SURB")
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(cConstants.RoundTripTimeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				sentWaitChan <- msg
			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}

// BlockingSendReliableMessage sends a message with automatic message retransmission enabled
func (s *Session) BlockingSendReliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// TODO: it would be better to have the message automatically retransmitted a configurable number of times before emitting a failure to this channel
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<commit_msg>Use the ReplyETA to derive slop time<commit_after>// send.go - mixnet client send
// Copyright (C) 2018  David Stainton.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package client

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"github.com/katzenpost/client/utils"
	"io"
	"time"

	cConstants "github.com/katzenpost/client/constants"
	"github.com/katzenpost/core/constants"
	"github.com/katzenpost/core/crypto/rand"
	sConstants "github.com/katzenpost/core/sphinx/constants"
)

var ErrReplyTimeout = errors.New("failure waiting for reply, timeout reached")
var ErrMessageNotSent = errors.New("failure sending message")

func (s *Session) sendNext() {
	msg, err := s.egressQueue.Peek()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Peek from queue")
		return
	}
	if msg == nil {
		s.fatalErrCh <- errors.New("impossible failure, got nil message from queue")
		return
	}
	m := msg.(*Message)
	s.doSend(m)
	_, err = s.egressQueue.Pop()
	if err != nil {
		s.fatalErrCh <- errors.New("impossible failure to Pop from queue")
	}
}

func NewRescheduler(s *Session) *rescheduler {
	r := &rescheduler{s: s}
	s.log.Debugf("Creating TimerQueue")
	r.timerQ = NewTimerQueue(r)
	return r
}

type rescheduler struct {
	s *Session
	timerQ *TimerQueue
}

func (r *rescheduler) Push(i Item) error {
	// rescheduler checks whether a message was ACK'd when the timerQ fires
	// and if it has not, reschedules the message for transmission again
	m := i.(*Message)
	if _, ok := r.s.surbIDMap.Load(*m.SURBID); ok {
		// still waiting for a SURB-ACK that hasn't arrived
		r.s.surbIDMap.Delete(m.SURBID)
		r.s.opCh <- opRetransmit{msg :m}
	}
	return nil
}

func (s *Session) doRetransmit(msg *Message) {
	s.log.Debugf("doRetransmit for %x", *msg.ID)
	msg.Retransmissions++
	s.log.Debugf("retransmissions: %d", msg.Retransmissions)
	s.doSend(msg)
}

func (s *Session) doSend(msg *Message) {
	surbID := [sConstants.SURBIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, surbID[:])
	if err != nil {
		s.fatalErrCh <- fmt.Errorf("impossible failure, failed to generate SURB ID for message ID %x", *msg.ID)
		return
	}
	key := []byte{}
	var eta time.Duration
	if msg.WithSURB {
		msg.SURBID = &surbID
		idStr := fmt.Sprintf("[%v]", hex.EncodeToString(surbID[:]))
		s.log.Debugf("doSend with SURB ID %x", idStr)
		key, eta, err = s.minclient.SendCiphertext(msg.Recipient, msg.Provider, &surbID, msg.Payload)
	} else {
		s.log.Debugf("doSend without SURB")
		err = s.minclient.SendUnreliableCiphertext(msg.Recipient, msg.Provider, msg.Payload)
	}

	// message was sent
	if err == nil {
		msg.SentAt = time.Now()
	}
	// expect a reply
	if msg.WithSURB {
		if err == nil {
			s.log.Debugf("doSend setting ReplyETA to %v", eta)
			// increase the timeout for each retransmission
			msg.ReplyETA = eta * (1 + time.Duration(msg.Retransmissions))
			msg.Key = key
			s.surbIDMap.Store(surbID, msg)
			if msg.Reliable {
				s.log.Debugf("Sending reliable message with retransmissions")
				timeSlop := eta // add a round-trip worth of delay before timing out
				msg.QueuePriority = uint64(msg.SentAt.Add(msg.ReplyETA).Add(timeSlop).UnixNano())
				s.rescheduler.timerQ.Push(msg)
			}
		}
		// write to waiting channel or close channel if message failed to send
		if msg.IsBlocking {
			sentWaitChanRaw, ok := s.sentWaitChanMap.Load(*msg.ID)
			if !ok {
				s.fatalErrCh <- fmt.Errorf("impossible failure, sentWaitChan not found for message ID %x", *msg.ID)
				return
			}
			sentWaitChan := sentWaitChanRaw.(chan *Message)
			if err == nil {
				sentWaitChan <- msg
			} else {
				close(sentWaitChan)
			}
			return
		}
	}
	s.eventCh.In() <- &MessageSentEvent{
		MessageID: msg.ID,
		Err:       err,
		SentAt:    msg.SentAt,
		ReplyETA:  msg.ReplyETA,
	}
}

func (s *Session) sendDropDecoy(loopSvc *utils.ServiceDescriptor) {
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for drop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  false,
		IsDecoy:   true,
	}
	s.doSend(msg)
}

func (s *Session) sendLoopDecoy(loopSvc *utils.ServiceDescriptor) {
	s.log.Info("sending loop decoy")
	payload := make([]byte, constants.UserForwardPayloadLength)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		s.fatalErrCh <- errors.New("failure to generate message ID for loop decoy")
		return
	}
	msg := &Message{
		ID:        &id,
		Recipient: loopSvc.Name,
		Provider:  loopSvc.Provider,
		Payload:   payload[:],
		WithSURB:  true,
		IsDecoy:   true,
	}
	defer s.incrementDecoyLoopTally()
	s.doSend(msg)
}

func (s *Session) composeMessage(recipient, provider string, message []byte, isBlocking bool) (*Message, error) {
	s.log.Debug("SendMessage")
	if len(message) > constants.UserForwardPayloadLength-4 {
		return nil, fmt.Errorf("invalid message size: %v", len(message))
	}
	payload := make([]byte, constants.UserForwardPayloadLength)
	binary.BigEndian.PutUint32(payload[:4], uint32(len(message)))
	copy(payload[4:], message)
	id := [cConstants.MessageIDLength]byte{}
	_, err := io.ReadFull(rand.Reader, id[:])
	if err != nil {
		return nil, err
	}
	var msg = Message{
		ID:         &id,
		Recipient:  recipient,
		Provider:   provider,
		Payload:    payload[:],
		WithSURB:   true,
		IsBlocking: isBlocking,
	}
	return &msg, nil
}

// SendReliableMessage asynchronously sends messages with automatic retransmissiosn.
func (s *Session) SendReliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

// SendUnreliableMessage asynchronously sends message without any automatic retransmissions.
func (s *Session) SendUnreliableMessage(recipient, provider string, message []byte) (*[cConstants.MessageIDLength]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, false)
	if err != nil {
		return nil, err
	}
	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}
	return msg.ID, nil
}

func (s *Session) BlockingSendUnreliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}

// BlockingSendReliableMessage sends a message with automatic message retransmission enabled
func (s *Session) BlockingSendReliableMessage(recipient, provider string, message []byte) ([]byte, error) {
	msg, err := s.composeMessage(recipient, provider, message, true)
	if err != nil {
		return nil, err
	}
	msg.Reliable = true
	sentWaitChan := make(chan *Message)
	s.sentWaitChanMap.Store(*msg.ID, sentWaitChan)
	defer s.sentWaitChanMap.Delete(*msg.ID)

	replyWaitChan := make(chan []byte)
	s.replyWaitChanMap.Store(*msg.ID, replyWaitChan)
	defer s.replyWaitChanMap.Delete(*msg.ID)

	err = s.egressQueue.Push(msg)
	if err != nil {
		return nil, err
	}

	// wait until sent so that we know the ReplyETA for the waiting below
	sentMessage := <-sentWaitChan

	// if the message failed to send we will receive a nil message
	if sentMessage == nil {
		return nil, ErrMessageNotSent
	}

	// TODO: it would be better to have the message automatically retransmitted a configurable number of times before emitting a failure to this channel
	// wait for reply or round trip timeout
	select {
	case reply := <-replyWaitChan:
		return reply, nil
	// these timeouts are often far too aggressive
	case <-time.After(sentMessage.ReplyETA + cConstants.RoundTripTimeSlop):
		return nil, ErrReplyTimeout
	}
	// unreachable
}
<|endoftext|>


<commit_before>// Copyright 2017 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//

package main

import (
	"errors"
	"io"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/moby/moby/pkg/term"
	context "golang.org/x/net/context"

	pb "github.com/kata-containers/agent/protocols/grpc"
)

const sigChanSize = 2048

var sigIgnored = map[syscall.Signal]bool{
	syscall.SIGCHLD:  true,
	syscall.SIGPIPE:  true,
	syscall.SIGWINCH: true,
}

type shim struct {
	containerID string
	execID      string

	ctx   context.Context
	agent *shimAgent
}

func newShim(addr, containerID, execID string) (*shim, error) {
	agent, err := newShimAgent(addr)
	if err != nil {
		return nil, err
	}

	return &shim{containerID: containerID,
		execID: execID,
		ctx:    context.Background(),
		agent:  agent}, nil
}

func (s *shim) proxyStdio(wg *sync.WaitGroup, terminal bool) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(1)
	if !terminal {
		// In case it's not a terminal, we also need to get the output
		// from stderr.
		wg.Add(1)
	}

	inPipe, outPipe, errPipe := shimStdioPipe(s.ctx, s.agent, s.containerID, s.execID)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		_, err2 := s.agent.CloseStdin(s.ctx, &pb.CloseStdinRequest{
			ContainerId: s.containerID,
			ExecId:      s.execID})
		if err1 != nil {
			logger().WithError(err1).Warn("copy stdin failed")
		}
		if err2 != nil {
			logger().WithError(err2).Warn("close stdin failed")
		}
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		logger().WithError(err).Info("copy stdout failed")
		wg.Done()
	}()

	if !terminal {
		go func() {
			_, err := io.Copy(os.Stderr, errPipe)
			logger().WithError(err).Info("copy stderr failed")
			wg.Done()
		}()
	}
}

func (s *shim) forwardAllSignals() chan os.Signal {
	sigc := make(chan os.Signal, sigChanSize)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for sig := range sigc {
			sysSig, ok := sig.(syscall.Signal)
			if !ok {
				err := errors.New("unknown signal")
				logger().WithError(err).WithField("signal", sig.String()).Error("")
				continue
			}

			if fatalSignal(sysSig) {
				logger().WithField("signal", sig).Error("received fatal signal")
				die()
			}

			if sigIgnored[sysSig] {
				//ignore these
				continue
			}
			// forward this signal to container
			_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
				ContainerId: s.containerID,
				ExecId:      s.execID,
				Signal:      uint32(sysSig)})
			if err != nil {
				logger().WithError(err).WithField("signal", sig.String()).Error("forward signal failed")
			}
		}
	}()
	return sigc
}

func (s *shim) resizeTty(fromTty *os.File) error {
	fd := fromTty.Fd()

	ws, err := term.GetWinsize(fd)
	if err != nil {
		logger().WithError(err).WithField("fd", fd).Info("Error getting window size")
		return nil
	}

	_, err = s.agent.TtyWinResize(s.ctx, &pb.TtyWinResizeRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID,
		Row:         uint32(ws.Height),
		Column:      uint32(ws.Width)})
	if err != nil {
		logger().WithError(err).Error("set window size failed")
	}

	return err
}

func (s *shim) monitorTtySize(tty *os.File) {
	s.resizeTty(tty)
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGWINCH)
	go func() {
		for range sigchan {
			s.resizeTty(tty)
		}
	}()
}

func (s *shim) wait() (int32, error) {
	resp, err := s.agent.WaitProcess(s.ctx, &pb.WaitProcessRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID})
	if err != nil {
		return 0, err
	}

	return resp.Status, nil
}
<commit_msg>logging: Add window details if resize fails<commit_after>// Copyright 2017 HyperHQ Inc.
//
// SPDX-License-Identifier: Apache-2.0
//

package main

import (
	"errors"
	"io"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/moby/moby/pkg/term"
	"github.com/sirupsen/logrus"
	context "golang.org/x/net/context"

	pb "github.com/kata-containers/agent/protocols/grpc"
)

const sigChanSize = 2048

var sigIgnored = map[syscall.Signal]bool{
	syscall.SIGCHLD:  true,
	syscall.SIGPIPE:  true,
	syscall.SIGWINCH: true,
}

type shim struct {
	containerID string
	execID      string

	ctx   context.Context
	agent *shimAgent
}

func newShim(addr, containerID, execID string) (*shim, error) {
	agent, err := newShimAgent(addr)
	if err != nil {
		return nil, err
	}

	return &shim{containerID: containerID,
		execID: execID,
		ctx:    context.Background(),
		agent:  agent}, nil
}

func (s *shim) proxyStdio(wg *sync.WaitGroup, terminal bool) {
	// don't wait the copying of the stdin, because `io.Copy(inPipe, os.Stdin)`
	// can't terminate when no input. todo: find a better way.
	wg.Add(1)
	if !terminal {
		// In case it's not a terminal, we also need to get the output
		// from stderr.
		wg.Add(1)
	}

	inPipe, outPipe, errPipe := shimStdioPipe(s.ctx, s.agent, s.containerID, s.execID)
	go func() {
		_, err1 := io.Copy(inPipe, os.Stdin)
		_, err2 := s.agent.CloseStdin(s.ctx, &pb.CloseStdinRequest{
			ContainerId: s.containerID,
			ExecId:      s.execID})
		if err1 != nil {
			logger().WithError(err1).Warn("copy stdin failed")
		}
		if err2 != nil {
			logger().WithError(err2).Warn("close stdin failed")
		}
	}()

	go func() {
		_, err := io.Copy(os.Stdout, outPipe)
		logger().WithError(err).Info("copy stdout failed")
		wg.Done()
	}()

	if !terminal {
		go func() {
			_, err := io.Copy(os.Stderr, errPipe)
			logger().WithError(err).Info("copy stderr failed")
			wg.Done()
		}()
	}
}

func (s *shim) forwardAllSignals() chan os.Signal {
	sigc := make(chan os.Signal, sigChanSize)
	// handle all signals for the process.
	signal.Notify(sigc)
	signal.Ignore(syscall.SIGCHLD, syscall.SIGPIPE)

	go func() {
		for sig := range sigc {
			sysSig, ok := sig.(syscall.Signal)
			if !ok {
				err := errors.New("unknown signal")
				logger().WithError(err).WithField("signal", sig.String()).Error("")
				continue
			}

			if fatalSignal(sysSig) {
				logger().WithField("signal", sig).Error("received fatal signal")
				die()
			}

			if sigIgnored[sysSig] {
				//ignore these
				continue
			}
			// forward this signal to container
			_, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{
				ContainerId: s.containerID,
				ExecId:      s.execID,
				Signal:      uint32(sysSig)})
			if err != nil {
				logger().WithError(err).WithField("signal", sig.String()).Error("forward signal failed")
			}
		}
	}()
	return sigc
}

func (s *shim) resizeTty(fromTty *os.File) error {
	fd := fromTty.Fd()

	ws, err := term.GetWinsize(fd)
	if err != nil {
		logger().WithError(err).WithField("fd", fd).Info("Error getting window size")
		return nil
	}

	_, err = s.agent.TtyWinResize(s.ctx, &pb.TtyWinResizeRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID,
		Row:         uint32(ws.Height),
		Column:      uint32(ws.Width)})
	if err != nil {
		logger().WithError(err).WithFields(logrus.Fields{
			"window-height": ws.Height,
			"window-width":  ws.Width,
		}).Error("set window size failed")
	}

	return err
}

func (s *shim) monitorTtySize(tty *os.File) {
	s.resizeTty(tty)
	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, syscall.SIGWINCH)
	go func() {
		for range sigchan {
			s.resizeTty(tty)
		}
	}()
}

func (s *shim) wait() (int32, error) {
	resp, err := s.agent.WaitProcess(s.ctx, &pb.WaitProcessRequest{
		ContainerId: s.containerID,
		ExecId:      s.execID})
	if err != nil {
		return 0, err
	}

	return resp.Status, nil
}
<|endoftext|>


<commit_before>package gopool

import (
	"golang.org/x/net/context"

	"log"
	"sync"
	"time"
)

// StdLogger is used to log error messages.
type Logger interface {
	Debug(v ...interface{})
	Info(v ...interface{})
	Crit(v ...interface{})
	Fatal(v ...interface{})
}

var PoolLogger GoPoolLogger

type GoPoolLogger struct {
}

func (gpl *GoPoolLogger) Debug(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Info(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Crit(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Fatal(v ...interface{}) {
	log.Println(v)
}

type GoPool struct {
	wg        sync.WaitGroup
	cancelctx context.Context
	canclFunc context.CancelFunc
}

//this is a function which takes a context and exits when the context is done.
type GoPoolFunc func(ctx context.Context, args ...interface{}) error

func NewGoPool(ctx context.Context) *GoPool {
	cancelctx, canclFunc := context.WithCancel(ctx)
	return &GoPool{
		cancelctx: cancelctx,
		canclFunc: canclFunc,
	}
}

func (gp *GoPool) Context() context.Context {
	return gp.cancelctx
}

var (
	PanicHandler = func(err interface{}, method string) {

		switch err.(type) {

		case error:
			PoolLogger.Info("RECOVERED FROM PANIC", "error is ", err.(error).Error(), "details", method)
		case string:
			PoolLogger.Info("RECOVERED FROM PANIC", "error is ", err.(string), "details", method)
		default:
			PoolLogger.Info("RECOVERED FROM PANIC")
		}

	}
)

// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func (gp *GoPool) waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
	c := make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return false // completed normally
	case <-time.After(timeout):
		return true // timed out
	}
}

func (gp *GoPool) ShutDown(waitforver bool, timeout time.Duration) {
	gp.canclFunc()
	if waitforver {
		gp.wg.Wait()
	} else {
		gp.waitTimeout(&gp.wg, timeout)
	}

}

func (gp *GoPool) AddJob(method string, fn GoPoolFunc, args ...interface{}) {
	gp.wg.Add(1)

	go func() {
		defer func() {
			gp.wg.Done()
			handler := PanicHandler
			if handler != nil {
				if err := recover(); err != nil {
					handler(err, method)
				}
			}
		}()
		fn(gp.Context(), args...)
	}()
}
<commit_msg>expose function as function instead of as type<commit_after>package gopool

import (
	"golang.org/x/net/context"

	"log"
	"sync"
	"time"
)

// StdLogger is used to log error messages.
type Logger interface {
	Debug(v ...interface{})
	Info(v ...interface{})
	Crit(v ...interface{})
	Fatal(v ...interface{})
}

var PoolLogger GoPoolLogger

type GoPoolLogger struct {
}

func (gpl *GoPoolLogger) Debug(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Info(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Crit(v ...interface{}) {
	log.Println(v)
}
func (gpl *GoPoolLogger) Fatal(v ...interface{}) {
	log.Println(v)
}

type GoPool struct {
	wg        sync.WaitGroup
	cancelctx context.Context
	canclFunc context.CancelFunc
}

//this is a function which takes a context and exits when the context is done.
//type GoPoolFunc func(ctx context.Context, args ...interface{}) error

func NewGoPool(ctx context.Context) *GoPool {
	cancelctx, canclFunc := context.WithCancel(ctx)
	return &GoPool{
		cancelctx: cancelctx,
		canclFunc: canclFunc,
	}
}

func (gp *GoPool) Context() context.Context {
	return gp.cancelctx
}

var (
	PanicHandler = func(err interface{}, method string) {

		switch err.(type) {

		case error:
			PoolLogger.Info("RECOVERED FROM PANIC", "error is ", err.(error).Error(), "details", method)
		case string:
			PoolLogger.Info("RECOVERED FROM PANIC", "error is ", err.(string), "details", method)
		default:
			PoolLogger.Info("RECOVERED FROM PANIC")
		}

	}
)

// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func (gp *GoPool) waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
	c := make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return false // completed normally
	case <-time.After(timeout):
		return true // timed out
	}
}

func (gp *GoPool) ShutDown(waitforver bool, timeout time.Duration) {
	gp.canclFunc()
	if waitforver {
		gp.wg.Wait()
	} else {
		gp.waitTimeout(&gp.wg, timeout)
	}

}

func (gp *GoPool) AddJob(method string, fn func(ctx context.Context, args ...interface{}) error, args ...interface{}) {
	gp.wg.Add(1)

	go func() {
		defer func() {
			gp.wg.Done()
			handler := PanicHandler
			if handler != nil {
				if err := recover(); err != nil {
					handler(err, method)
				}
			}
		}()
		fn(gp.Context(), args...)
	}()
}
<|endoftext|>


<commit_before>// Package s3 signs HTTP requests for Amazon S3 and compatible services.
//
// See
// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/RESTAuthentication.html.
package s3

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"io"
	"net/http"
	"sort"
	"strings"
	"time"
)

var signParams = map[string]bool{
	"acl":                          true,
	"delete":                       true,
	"lifecycle":                    true,
	"location":                     true,
	"logging":                      true,
	"notification":                 true,
	"partNumber":                   true,
	"policy":                       true,
	"requestPayment":               true,
	"response-cache-control":       true,
	"response-content-disposition": true,
	"response-content-encoding":    true,
	"response-content-language":    true,
	"response-content-type":        true,
	"response-expires":             true,
	"torrent":                      true,
	"uploadId":                     true,
	"uploads":                      true,
	"versionId":                    true,
	"versioning":                   true,
	"versions":                     true,
	"website":                      true,
}

// Keys holds a set of Amazon Security Credentials.
type Keys struct {
	AccessKey string
	SecretKey string
	// Used for temporary security credentials. Leave blank to use
	// standard AWS Account or IAM credentials.
	// See http://docs.aws.amazon.com/AmazonS3/latest/dev/MakingRequests.html#TypesofSecurityCredentials
	SecurityToken string
}

// IdentityBucket returns subdomain as-is.
func IdentityBucket(subdomain string) string {
	return subdomain
}

// AmazonBucket returns up to last section of subdomain.
// Intended to be used with Amazon S3 service: "johnsmith.s3-eu-west-1" => "johnsmith".
func AmazonBucket(subdomain string) string {
	s := strings.Split(subdomain, ".")
	return strings.Join(s[:len(s)-1], ".")
}

// The default Service used by Sign.
var DefaultService = &Service{Domain: "amazonaws.com"}

// Sign signs an HTTP request with the given S3 keys.
//
// This function is shorthand for DefaultService.Sign(r, k).
func Sign(r *http.Request, k Keys) {
	DefaultService.Sign(r, k)
}

func SignURL(r *http.Request, t time.Time, k Keys) {
	DefaultService.SignURL(r, t, k)
}

// Service represents an S3-compatible service.
type Service struct {
	Domain string                        // service root domain, used to extract subdomain from an http.Request and pass it to Bucket
	Bucket func(subdomain string) string // function used to derive a bucket name from subdomain; if nil, AmazonBucket is used
}

// Sign signs an HTTP request with the given S3 keys for use on service s.
func (s *Service) Sign(r *http.Request, k Keys) {
	if k.SecurityToken != "" {
		r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
	}
	h := hmac.New(sha1.New, []byte(k.SecretKey))
	s.writeSigData(h, r)
	sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
	base64.StdEncoding.Encode(sig, h.Sum(nil))
	r.Header.Set("Authorization", "AWS "+k.AccessKey+":"+string(sig))
}

func (s *Service) SignURL(r *http.Request, t time.Time, k Keys) {
	if k.SecurityToken != "" {
		r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
	}
	h := hmac.New(sha1.New, []byte(k.SecretKey))
	s.writeSigURLData(h, r, t)
	sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
	base64.StdEncoding.Encode(sig, h.Sum(nil))
	r.URL.RawQuery = "AWSAccessKeyId="+k.AccessKey+"&Signature="+string(sig)+"&Expires="+string(t.Unix())
}

func (s *Service) writeSigData(w io.Writer, r *http.Request) {
	w.Write([]byte(r.Method))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-md5")))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-type")))
	w.Write([]byte{'\n'})
	if _, ok := r.Header["X-Amz-Date"]; !ok {
		w.Write([]byte(r.Header.Get("date")))
	}
	w.Write([]byte{'\n'})
	writeAmzHeaders(w, r)
	s.writeResource(w, r)
}

func (s *Service) writeSigURLData(w io.Writer, r *http.Request, t time.Time) {
	w.Write([]byte(r.Method))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-md5")))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-type")))
	w.Write([]byte{'\n'})
	w.Write([]byte(string(t.Unix())))
	w.Write([]byte{'\n'})
	writeAmzHeaders(w, r)
	s.writeResource(w, r)
}

func (s *Service) writeResource(w io.Writer, r *http.Request) {
	s.writeVhostBucket(w, strings.ToLower(r.Host))
	path := r.URL.RequestURI()
	if r.URL.RawQuery != "" {
		path = path[:len(path)-len(r.URL.RawQuery)-1]
	}
	w.Write([]byte(path))
	s.writeSubResource(w, r)
}

func (s *Service) writeVhostBucket(w io.Writer, host string) {
	if i := strings.Index(host, ":"); i != -1 {
		host = host[:i]
	}

	if host == s.Domain {
		// no vhost - do nothing
	} else if strings.HasSuffix(host, "."+s.Domain) {
		// vhost - bucket may be in prefix
		b := s.Bucket
		if b == nil {
			b = AmazonBucket
		}
		bucket := b(host[:len(host)-len(s.Domain)-1])

		if bucket != "" {
			w.Write([]byte{'/'})
			w.Write([]byte(bucket))
		}
	} else {
		// cname - bucket is host
		w.Write([]byte{'/'})
		w.Write([]byte(host))
	}
}

func (s *Service) writeSubResource(w io.Writer, r *http.Request) {
	var a []string
	for k, vs := range r.URL.Query() {
		if signParams[k] {
			for _, v := range vs {
				if v == "" {
					a = append(a, k)
				} else {
					a = append(a, k+"="+v)
				}
			}
		}
	}
	sort.Strings(a)
	var p byte = '?'
	for _, s := range a {
		w.Write([]byte{p})
		w.Write([]byte(s))
		p = '&'
	}
}

func writeAmzHeaders(w io.Writer, r *http.Request) {
	var keys []string
	for k, _ := range r.Header {
		if strings.HasPrefix(strings.ToLower(k), "x-amz-") {
			keys = append(keys, k)
		}
	}

	sort.Strings(keys)
	var a []string
	for _, k := range keys {
		v := r.Header[k]
		a = append(a, strings.ToLower(k)+":"+strings.Join(v, ","))
	}
	for _, h := range a {
		w.Write([]byte(h))
		w.Write([]byte{'\n'})
	}
}
<commit_msg>move over to SignQuery.<commit_after>// Package s3 signs HTTP requests for Amazon S3 and compatible services.
//
// See
// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/RESTAuthentication.html.
package s3

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"io"
	"net/http"
	"sort"
	"strings"
	"time"
)

var signParams = map[string]bool{
	"acl":                          true,
	"delete":                       true,
	"lifecycle":                    true,
	"location":                     true,
	"logging":                      true,
	"notification":                 true,
	"partNumber":                   true,
	"policy":                       true,
	"requestPayment":               true,
	"response-cache-control":       true,
	"response-content-disposition": true,
	"response-content-encoding":    true,
	"response-content-language":    true,
	"response-content-type":        true,
	"response-expires":             true,
	"torrent":                      true,
	"uploadId":                     true,
	"uploads":                      true,
	"versionId":                    true,
	"versioning":                   true,
	"versions":                     true,
	"website":                      true,
}

// Keys holds a set of Amazon Security Credentials.
type Keys struct {
	AccessKey string
	SecretKey string
	// Used for temporary security credentials. Leave blank to use
	// standard AWS Account or IAM credentials.
	// See http://docs.aws.amazon.com/AmazonS3/latest/dev/MakingRequests.html#TypesofSecurityCredentials
	SecurityToken string
}

// IdentityBucket returns subdomain as-is.
func IdentityBucket(subdomain string) string {
	return subdomain
}

// AmazonBucket returns up to last section of subdomain.
// Intended to be used with Amazon S3 service: "johnsmith.s3-eu-west-1" => "johnsmith".
func AmazonBucket(subdomain string) string {
	s := strings.Split(subdomain, ".")
	return strings.Join(s[:len(s)-1], ".")
}

// The default Service used by Sign.
var DefaultService = &Service{Domain: "amazonaws.com"}

// Sign signs an HTTP request with the given S3 keys.
//
// This function is shorthand for DefaultService.Sign(r, k).
func Sign(r *http.Request, k Keys) {
	DefaultService.Sign(r, k)
}

func SignQuery(r *http.Request, t time.Time, k Keys) {
	DefaultService.SignQuery(r, t, k)
}

// Service represents an S3-compatible service.
type Service struct {
	Domain string                        // service root domain, used to extract subdomain from an http.Request and pass it to Bucket
	Bucket func(subdomain string) string // function used to derive a bucket name from subdomain; if nil, AmazonBucket is used
}

// Sign signs an HTTP request with the given S3 keys for use on service s.
func (s *Service) Sign(r *http.Request, k Keys) {
	if k.SecurityToken != "" {
		r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
	}
	h := hmac.New(sha1.New, []byte(k.SecretKey))
	s.writeSigData(h, r)
	sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
	base64.StdEncoding.Encode(sig, h.Sum(nil))
	r.Header.Set("Authorization", "AWS "+k.AccessKey+":"+string(sig))
}

func (s *Service) SignQuery(r *http.Request, t time.Time, k Keys) {
	if k.SecurityToken != "" {
		r.Header.Set("X-Amz-Security-Token", k.SecurityToken)
	}
	h := hmac.New(sha1.New, []byte(k.SecretKey))
	s.writeSigQueryData(h, r, t)
	sig := make([]byte, base64.StdEncoding.EncodedLen(h.Size()))
	base64.StdEncoding.Encode(sig, h.Sum(nil))
	r.URL.RawQuery = "AWSAccessKeyId="+k.AccessKey+"&Signature="+string(sig)+"&Expires="+string(t.Unix())
}

func (s *Service) writeSigData(w io.Writer, r *http.Request) {
	w.Write([]byte(r.Method))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-md5")))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-type")))
	w.Write([]byte{'\n'})
	if _, ok := r.Header["X-Amz-Date"]; !ok {
		w.Write([]byte(r.Header.Get("date")))
	}
	w.Write([]byte{'\n'})
	writeAmzHeaders(w, r)
	s.writeResource(w, r)
}

func (s *Service) writeSigQueryData(w io.Writer, r *http.Request, t time.Time) {
	w.Write([]byte(r.Method))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-md5")))
	w.Write([]byte{'\n'})
	w.Write([]byte(r.Header.Get("content-type")))
	w.Write([]byte{'\n'})
	w.Write([]byte(string(t.Unix())))
	w.Write([]byte{'\n'})
	writeAmzHeaders(w, r)
	s.writeResource(w, r)
}

func (s *Service) writeResource(w io.Writer, r *http.Request) {
	s.writeVhostBucket(w, strings.ToLower(r.Host))
	path := r.URL.RequestURI()
	if r.URL.RawQuery != "" {
		path = path[:len(path)-len(r.URL.RawQuery)-1]
	}
	w.Write([]byte(path))
	s.writeSubResource(w, r)
}

func (s *Service) writeVhostBucket(w io.Writer, host string) {
	if i := strings.Index(host, ":"); i != -1 {
		host = host[:i]
	}

	if host == s.Domain {
		// no vhost - do nothing
	} else if strings.HasSuffix(host, "."+s.Domain) {
		// vhost - bucket may be in prefix
		b := s.Bucket
		if b == nil {
			b = AmazonBucket
		}
		bucket := b(host[:len(host)-len(s.Domain)-1])

		if bucket != "" {
			w.Write([]byte{'/'})
			w.Write([]byte(bucket))
		}
	} else {
		// cname - bucket is host
		w.Write([]byte{'/'})
		w.Write([]byte(host))
	}
}

func (s *Service) writeSubResource(w io.Writer, r *http.Request) {
	var a []string
	for k, vs := range r.URL.Query() {
		if signParams[k] {
			for _, v := range vs {
				if v == "" {
					a = append(a, k)
				} else {
					a = append(a, k+"="+v)
				}
			}
		}
	}
	sort.Strings(a)
	var p byte = '?'
	for _, s := range a {
		w.Write([]byte{p})
		w.Write([]byte(s))
		p = '&'
	}
}

func writeAmzHeaders(w io.Writer, r *http.Request) {
	var keys []string
	for k, _ := range r.Header {
		if strings.HasPrefix(strings.ToLower(k), "x-amz-") {
			keys = append(keys, k)
		}
	}

	sort.Strings(keys)
	var a []string
	for _, k := range keys {
		v := r.Header[k]
		a = append(a, strings.ToLower(k)+":"+strings.Join(v, ","))
	}
	for _, h := range a {
		w.Write([]byte(h))
		w.Write([]byte{'\n'})
	}
}
<|endoftext|>