<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"syscall"
	"time"

	"github.com/cascades-fbp/cascades/components/utils"
	"github.com/cascades-fbp/cascades/runtime"
	zmq "github.com/pebbe/zmq4"
)

var (
	// Flags
	optionsEndpoint = flag.String("port.options", "", "Component's options port endpoint")
	inputEndpoint   = flag.String("port.in", "", "Component's input port endpoint")
	outputEndpoint  = flag.String("port.out", "", "Component's output port endpoint")
	jsonFlag        = flag.Bool("json", false, "Print component documentation in JSON")
	debug           = flag.Bool("debug", false, "Enable debug mode")

	// Internal
	optionsPort, inPort, outPort *zmq.Socket
	inCh, outCh                  chan bool
	err                          error
)

func validateArgs() {
	if *optionsEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *inputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *outputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
}

func openPorts() {
	optionsPort, err = utils.CreateInputPort("tcp/server.options", *optionsEndpoint, nil)
	utils.AssertError(err)

	inPort, err = utils.CreateInputPort("tcp/server.in", *inputEndpoint, inCh)
	utils.AssertError(err)
}

func closePorts() {
	optionsPort.Close()
	inPort.Close()
	if outPort != nil {
		outPort.Close()
	}
	zmq.Term()
}

func main() {
	flag.Parse()

	if *jsonFlag {
		doc, _ := registryEntry.JSON()
		fmt.Println(string(doc))
		os.Exit(0)
	}

	log.SetFlags(0)
	if *debug {
		log.SetOutput(os.Stdout)
	} else {
		log.SetOutput(ioutil.Discard)
	}

	validateArgs()

	service := NewService()
	ch := utils.HandleInterruption()
	inCh = make(chan bool)
	outCh = make(chan bool)

	openPorts()
	defer closePorts()

	go func() {
		outPort, err = utils.CreateOutputPort("tcp/server.out", *outputEndpoint, outCh)
		utils.AssertError(err)
		for data := range service.Output {
			outPort.SendMessage(runtime.NewOpenBracket())
			outPort.SendMessage(runtime.NewPacket(data[0]))
			outPort.SendMessage(runtime.NewPacket(data[1]))
			outPort.SendMessage(runtime.NewCloseBracket())
		}
	}()

	waitCh := make(chan bool)
	go func() {
		total := 0
		for {
			select {
			case v := <-inCh:
				if !v {
					log.Println("IN port is closed. Interrupting execution")
					ch <- syscall.SIGTERM
				} else {
					total++
				}
			case v := <-outCh:
				if !v {
					log.Println("OUT port is closed. Interrupting execution")
					ch <- syscall.SIGTERM
				} else {
					total++
				}
			}
			if total >= 2 && waitCh != nil {
				waitCh <- true
			}
		}
	}()

	log.Println("Waiting for port connections to establish... ")
	select {
	case <-waitCh:
		log.Println("Ports connected")
		waitCh = nil
	case <-time.Tick(30 * time.Second):
		log.Println("Timeout: port connections were not established within provided interval")
		os.Exit(1)
	}

	// Wait for the configuration on the options port
	log.Println("Waiting for configuration...")
	var bindAddr string
	for {
		ip, err := optionsPort.RecvMessageBytes(0)
		if err != nil {
			log.Println("Error receiving IP:", err.Error())
			continue
		}
		if !runtime.IsValidIP(ip) || !runtime.IsPacket(ip) {
			continue
		}
		bindAddr = string(ip[1])
		break
	}
	optionsPort.Close()

	// Create binding address listener
	laddr, err := net.ResolveTCPAddr("tcp", bindAddr)
	if err != nil {
		log.Fatalln(err)
	}
	listener, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("Listening on", listener.Addr())

	go service.Serve(listener)

	log.Println("Started...")
	var (
		connID string
		data   []byte
	)
	for {
		ip, err := inPort.RecvMessageBytes(0)
		if err != nil {
			log.Println("Error receiving message:", err.Error())
			continue
		}
		if !runtime.IsValidIP(ip) {
			continue
		}
		switch {
		case runtime.IsOpenBracket(ip):
			connID = ""
			data = nil
		case runtime.IsPacket(ip):
			if connID == "" {
				connID = string(ip[1])
			} else {
				data = ip[1]
			}
		case runtime.IsCloseBracket(ip):
			service.Dispatch(connID, data)
		}
	}
}
<commit_msg>Refactored life-cycle management<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/cascades-fbp/cascades/components/utils"
	"github.com/cascades-fbp/cascades/runtime"
	zmq "github.com/pebbe/zmq4"
)

var (
	// Flags
	optionsEndpoint = flag.String("port.options", "", "Component's options port endpoint")
	inputEndpoint   = flag.String("port.in", "", "Component's input port endpoint")
	outputEndpoint  = flag.String("port.out", "", "Component's output port endpoint")
	jsonFlag        = flag.Bool("json", false, "Print component documentation in JSON")
	debug           = flag.Bool("debug", false, "Enable debug mode")

	// Internal
	optionsPort, inPort, outPort *zmq.Socket
	inCh, outCh                  chan bool
	exitCh                       chan os.Signal
	err                          error
)

func main() {
	flag.Parse()

	if *jsonFlag {
		doc, _ := registryEntry.JSON()
		fmt.Println(string(doc))
		os.Exit(0)
	}

	log.SetFlags(0)
	if *debug {
		log.SetOutput(os.Stdout)
	} else {
		log.SetOutput(ioutil.Discard)
	}

	validateArgs()

	// Communication channels
	inCh = make(chan bool)
	outCh = make(chan bool)
	exitCh = make(chan os.Signal, 1)

	// Start the communication & processing logic
	go mainLoop()

	// Wait for the end...
	signal.Notify(exitCh, os.Interrupt, syscall.SIGTERM)
	<-exitCh

	log.Println("Done")
}

// mainLoop initiates all ports and handles the traffic
func mainLoop() {
	service := NewService()

	openPorts()
	defer closePorts()

	go func() {
		outPort, err = utils.CreateOutputPort("tcp/server.out", *outputEndpoint, outCh)
		utils.AssertError(err)
		for data := range service.Output {
			outPort.SendMessage(runtime.NewOpenBracket())
			outPort.SendMessage(runtime.NewPacket(data[0]))
			outPort.SendMessage(runtime.NewPacket(data[1]))
			outPort.SendMessage(runtime.NewCloseBracket())
		}
	}()

	waitCh := make(chan bool)
	go func() {
		total := 0
		for {
			select {
			case v := <-inCh:
				if !v {
					log.Println("IN port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			case v := <-outCh:
				if !v {
					log.Println("OUT port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			}
			if total >= 2 && waitCh != nil {
				waitCh <- true
			}
		}
	}()

	log.Println("Waiting for port connections to establish... ")
	select {
	case <-waitCh:
		log.Println("Ports connected")
		waitCh = nil
	case <-time.Tick(30 * time.Second):
		log.Println("Timeout: port connections were not established within provided interval")
		exitCh <- syscall.SIGTERM
		return
	}

	// Wait for the configuration on the options port
	log.Println("Waiting for configuration...")
	var bindAddr string
	for {
		ip, err := optionsPort.RecvMessageBytes(0)
		if err != nil {
			log.Println("Error receiving IP:", err.Error())
			continue
		}
		if !runtime.IsValidIP(ip) || !runtime.IsPacket(ip) {
			continue
		}
		bindAddr = string(ip[1])
		break
	}
	optionsPort.Close()

	// Create binding address listener
	laddr, err := net.ResolveTCPAddr("tcp", bindAddr)
	if err != nil {
		log.Fatalln(err)
	}
	listener, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("Listening on", listener.Addr())

	go service.Serve(listener)

	log.Println("Started...")
	var (
		connID string
		data   []byte
	)
	for {
		ip, err := inPort.RecvMessageBytes(0)
		if err != nil {
			log.Println("Error receiving message:", err.Error())
			continue
		}
		if !runtime.IsValidIP(ip) {
			continue
		}
		switch {
		case runtime.IsOpenBracket(ip):
			connID = ""
			data = nil
		case runtime.IsPacket(ip):
			if connID == "" {
				connID = string(ip[1])
			} else {
				data = ip[1]
			}
		case runtime.IsCloseBracket(ip):
			service.Dispatch(connID, data)
		}
	}
}

// validateArgs checks all required flags
func validateArgs() {
	if *optionsEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *inputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *outputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
}

// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
	optionsPort, err = utils.CreateInputPort("tcp/server.options", *optionsEndpoint, nil)
	utils.AssertError(err)

	inPort, err = utils.CreateInputPort("tcp/server.in", *inputEndpoint, inCh)
	utils.AssertError(err)
}

// closePorts closes all active ports and terminates ZMQ context
func closePorts() {
	log.Println("Closing ports...")
	optionsPort.Close()
	inPort.Close()
	if outPort != nil {
		outPort.Close()
	}
	zmq.Term()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
)

var traceCommunications = flag.Bool("trace_comm", false, "Wypisz na standardowe wyjście diagnostyczne informację o każdej wysłanej i odebranej wiadomości")

const magic = 1736434764
const recvResponseMagic = magic + 1
const sendOpType = 3
const recvOpType = 4

type header struct {
	Magic     uint32
	NodeCount int32
	NodeID    int32
}

type recvResponse struct {
	RecvResponseMagic uint32
	SourceID          int32
	Length            int32
	// Message []byte
}

type recvHeader struct {
	// OpType byte
	SourceID int32
}

type sendHeader struct {
	// OpType byte
	TargetID int32
	Length   int32
	// Message []byte
}

const MessageCountLimit = 1000
const MessageSizeLimit = 8 * 1024 * 1024

type Message struct {
	Source  int
	Target  int
	Message []byte
}

func (i *Instance) PutMessage(message Message) {
	i.queues[message.Source].Put() <- message
}

func (i *Instance) sendMessage(targetID int, message []byte) error {
	i.messagesSent++
	if i.messagesSent > MessageCountLimit {
		return fmt.Errorf("przekroczony limit (%d) liczby wysłanych wiadomości", MessageCountLimit)
	}
	i.messageBytesSent += len(message)
	if i.messageBytesSent > MessageSizeLimit {
		return fmt.Errorf("przekroczony limit (%d bajtów) sumarycznego rozmiaru wysłanych wiadomości", MessageSizeLimit)
	}
	i.outgoingMessages <- Message{Source: i.id, Target: targetID, Message: message}
	return nil
}

func (i *Instance) receiveMessage(sourceID int) Message {
	// XXX unblocking when exiting
	message := (<-i.queues[sourceID].Get()).(Message)
	return message
}

func (i *Instance) receiveAnyMessage() Message {
	// XXX unblocking when exiting
	mq := <-i.selector
	message := (<-mq.Get()).(Message)
	return message
}

func (i *Instance) communicate(r io.Reader, w io.Writer) error {
	h := header{
		Magic:     magic,
		NodeCount: int32(i.totalInstances),
		NodeID:    int32(i.id),
	}
	if err := binary.Write(w, binary.LittleEndian, &h); err != nil {
		return err
	}
	for {
		var opType [1]byte
		if _, err := r.Read(opType[:]); err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		switch opType[0] {
		case sendOpType:
			var sh sendHeader
			if err := binary.Read(r, binary.LittleEndian, &sh); err != nil {
				return err
			}
			if sh.Length < 0 || sh.Length > MessageSizeLimit {
				return fmt.Errorf("invalid size of a message to be sent: %d", sh.Length)
			}
			if sh.TargetID < 0 || sh.TargetID >= 100 {
				return fmt.Errorf("invalid target instance in a send request: %d", sh.TargetID)
			}
			if *traceCommunications {
				log.Printf("Instancja %d wysyła %d bajtów do instancji %d.", i.id, sh.Length, sh.TargetID)
			}
			message := make([]byte, sh.Length)
			if _, err := io.ReadFull(r, message); err != nil {
				return err
			}
			i.sendMessage(int(sh.TargetID), message)
		case recvOpType:
			var rh recvHeader
			if err := binary.Read(r, binary.LittleEndian, &rh); err != nil {
				return err
			}
			if rh.SourceID < -1 || rh.SourceID >= 100 {
				return fmt.Errorf("invalid source instance in a receive request: %d", rh.SourceID)
			}
			var message Message
			if rh.SourceID == -1 {
				if *traceCommunications {
					log.Printf("Instancja %d czeka na wiadomość od dowolnej innej instancji.", i.id)
				}
				message = i.receiveAnyMessage()
			} else {
				if *traceCommunications {
					log.Printf("Instancja %d czeka na wiadomość od instancji %d.", i.id, rh.SourceID)
				}
				message = i.receiveMessage(int(rh.SourceID))
			}
			if *traceCommunications {
				log.Printf("Instancja %d odebrała wiadomość od instancji %d.", i.id, message.Source)
			}
			rr := recvResponse{
				RecvResponseMagic: recvResponseMagic,
				SourceID:          int32(message.Source),
				Length:            int32(len(message.Message)),
			}
			if err := binary.Write(w, binary.LittleEndian, &rr); err != nil {
				return err
			}
			if n, err := w.Write(message.Message); n < len(message.Message) {
				if err == nil {
					err = io.ErrShortWrite
				}
				return err
			}
		default:
			return fmt.Errorf("invalid operation type %x", opType[0])
		}
	}
}
<commit_msg>added names for limit exceeded errors<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
)

var traceCommunications = flag.Bool("trace_comm", false, "Wypisz na standardowe wyjście diagnostyczne informację o każdej wysłanej i odebranej wiadomości")

const magic = 1736434764
const recvResponseMagic = magic + 1
const sendOpType = 3
const recvOpType = 4

type header struct {
	Magic     uint32
	NodeCount int32
	NodeID    int32
}

type recvResponse struct {
	RecvResponseMagic uint32
	SourceID          int32
	Length            int32
	// Message []byte
}

type recvHeader struct {
	// OpType byte
	SourceID int32
}

type sendHeader struct {
	// OpType byte
	TargetID int32
	Length   int32
	// Message []byte
}

const MessageCountLimit = 1000
const MessageSizeLimit = 8 * 1024 * 1024

type Message struct {
	Source  int
	Target  int
	Message []byte
}

func (i *Instance) PutMessage(message Message) {
	i.queues[message.Source].Put() <- message
}

var ErrMessageCount = fmt.Errorf("przekroczony limit (%d) liczby wysłanych wiadomości", MessageCountLimit)
var ErrMessageSize = fmt.Errorf("przekroczony limit (%d bajtów) sumarycznego rozmiaru wysłanych wiadomości", MessageSizeLimit)

func (i *Instance) sendMessage(targetID int, message []byte) error {
	i.messagesSent++
	if i.messagesSent > MessageCountLimit {
		return ErrMessageCount
	}
	i.messageBytesSent += len(message)
	if i.messageBytesSent > MessageSizeLimit {
		return ErrMessageSize
	}
	i.outgoingMessages <- Message{Source: i.id, Target: targetID, Message: message}
	return nil
}

func (i *Instance) receiveMessage(sourceID int) Message {
	// XXX unblocking when exiting
	message := (<-i.queues[sourceID].Get()).(Message)
	return message
}

func (i *Instance) receiveAnyMessage() Message {
	// XXX unblocking when exiting
	mq := <-i.selector
	message := (<-mq.Get()).(Message)
	return message
}

func (i *Instance) communicate(r io.Reader, w io.Writer) error {
	h := header{
		Magic:     magic,
		NodeCount: int32(i.totalInstances),
		NodeID:    int32(i.id),
	}
	if err := binary.Write(w, binary.LittleEndian, &h); err != nil {
		return err
	}
	for {
		var opType [1]byte
		if _, err := r.Read(opType[:]); err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		switch opType[0] {
		case sendOpType:
			var sh sendHeader
			if err := binary.Read(r, binary.LittleEndian, &sh); err != nil {
				return err
			}
			if sh.Length < 0 || sh.Length > MessageSizeLimit {
				return fmt.Errorf("invalid size of a message to be sent: %d", sh.Length)
			}
			if sh.TargetID < 0 || sh.TargetID >= 100 {
				return fmt.Errorf("invalid target instance in a send request: %d", sh.TargetID)
			}
			if *traceCommunications {
				log.Printf("Instancja %d wysyła %d bajtów do instancji %d.", i.id, sh.Length, sh.TargetID)
			}
			message := make([]byte, sh.Length)
			if _, err := io.ReadFull(r, message); err != nil {
				return err
			}
			i.sendMessage(int(sh.TargetID), message)
		case recvOpType:
			var rh recvHeader
			if err := binary.Read(r, binary.LittleEndian, &rh); err != nil {
				return err
			}
			if rh.SourceID < -1 || rh.SourceID >= 100 {
				return fmt.Errorf("invalid source instance in a receive request: %d", rh.SourceID)
			}
			var message Message
			if rh.SourceID == -1 {
				if *traceCommunications {
					log.Printf("Instancja %d czeka na wiadomość od dowolnej innej instancji.", i.id)
				}
				message = i.receiveAnyMessage()
			} else {
				if *traceCommunications {
					log.Printf("Instancja %d czeka na wiadomość od instancji %d.", i.id, rh.SourceID)
				}
				message = i.receiveMessage(int(rh.SourceID))
			}
			if *traceCommunications {
				log.Printf("Instancja %d odebrała wiadomość od instancji %d.", i.id, message.Source)
			}
			rr := recvResponse{
				RecvResponseMagic: recvResponseMagic,
				SourceID:          int32(message.Source),
				Length:            int32(len(message.Message)),
			}
			if err := binary.Write(w, binary.LittleEndian, &rr); err != nil {
				return err
			}
			if n, err := w.Write(message.Message); n < len(message.Message) {
				if err == nil {
					err = io.ErrShortWrite
				}
				return err
			}
		default:
			return fmt.Errorf("invalid operation type %x", opType[0])
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func dumpStart(d dumpTarget) result {
	t := time.Now()

	var r result
	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Fatal("mongodump could not be found")
		r.Result = "failed"
		return r
	}
	fmt.Printf("mongodump is available at %s\n", path)

	dumpCmd := exec.Command("mongodump", "--host", "10.10.1.103", "--port", "27017", "--archive")
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		r.Result = "failed"
		return r
	}

	if err := dumpCmd.Start(); err != nil {
		r.Result = "failed"
		return r
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String(d.Bucket),
		Key:    aws.String(d.Path + t.Format("2006-01-02T15:04")),
	})
	if err != nil {
		log.Fatalln("Failed to upload", err)
		r.Result = "failed"
		return r
	}

	if err := dumpCmd.Wait(); err != nil {
		log.Fatalln("Failed to dump", err)
		r.Result = "failed"
		return r
	}

	log.Println("Successfully uploaded to", result.Location)
	r.Result = "success"
	return r
}
<commit_msg>make configurable<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func dumpStart(d dumpTarget) result {
	t := time.Now()
	mongoHost := os.Getenv("MONGO_HOST")
	mongoPort := os.Getenv("MONGO_PORT")
	if mongoHost == "" {
		mongoHost = "mongo"
	}
	if mongoPort == "" {
		mongoPort = "27017"
	}

	var r result
	path, err := exec.LookPath("mongodump")
	if err != nil {
		log.Fatal("mongodump could not be found")
		r.Result = "failed"
		return r
	}
	fmt.Printf("mongodump is available at %s\n", path)

	dumpCmd := exec.Command("mongodump", "--host", mongoHost, "--port", mongoPort, "--archive")
	body, err := dumpCmd.StdoutPipe()
	if err != nil {
		r.Result = "failed"
		return r
	}

	if err := dumpCmd.Start(); err != nil {
		r.Result = "failed"
		return r
	}

	// Wrap the pipe to hide the seek methods from the uploader
	var bodyWrap = struct {
		io.Reader
	}{body}

	uploader := s3manager.NewUploader(session.New(&aws.Config{Region: aws.String("us-east-1")}))
	result, err := uploader.Upload(&s3manager.UploadInput{
		Body:   bodyWrap,
		Bucket: aws.String(d.Bucket),
		Key:    aws.String(d.Path + t.Format("2006-01-02T15:04")),
	})
	if err != nil {
		log.Fatalln("Failed to upload", err)
		r.Result = "failed"
		return r
	}

	if err := dumpCmd.Wait(); err != nil {
		log.Fatalln("Failed to dump", err)
		r.Result = "failed"
		return r
	}

	log.Println("Successfully uploaded to", result.Location)
	r.Result = "success"
	return r
}
<|endoftext|>


<commit_before>package aws

import (
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAwsRoute53Record_importBasic(t *testing.T) {
	resourceName := "aws_route53_record.default"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53RecordDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccRoute53RecordConfig,
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"weight"},
			},
		},
	})
}

func TestAccAwsRoute53Record_importUnderscored(t *testing.T) {
	resourceName := "aws_route53_record.underscore"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53RecordDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccRoute53RecordConfigUnderscoreInName,
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"weight"},
			},
		},
	})
}
<commit_msg>resource/aws_route53_record: Fix import acc tests<commit_after>package aws

import (
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccAwsRoute53Record_importBasic(t *testing.T) {
	resourceName := "aws_route53_record.default"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53RecordDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccRoute53RecordConfig,
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"allow_overwrite", "weight"},
			},
		},
	})
}

func TestAccAwsRoute53Record_importUnderscored(t *testing.T) {
	resourceName := "aws_route53_record.underscore"

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckRoute53RecordDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccRoute53RecordConfigUnderscoreInName,
			},
			{
				ResourceName:            resourceName,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"allow_overwrite", "weight"},
			},
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// goconfig is a easy-use comments-support configuration file parser.
package goconfig

import (
	"fmt"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

const (
	// Default section name.
	DEFAULT_SECTION = "DEFAULT"
	// Maximum allowed depth when recursively substituing variable names.
	_DEPTH_VALUES = 200

	// Get Errors.
	SectionNotFound = iota
	KeyNotFound
	// Read Errors.
	BlankSection
	// Get and Read Errors.
	CouldNotParse
)

var (
	LineBreak = "\n"
	// %(variable)s
	varRegExp = regexp.MustCompile(`%\(([a-zA-Z0-9_.\-]+)\)s`)
)

func init() {
	if runtime.GOOS == "windows" {
		LineBreak = "\r\n"
	}
}

// ConfigFile is the representation of configuration settings.
// The public interface is entirely through methods.
type ConfigFile struct {
	lock            sync.RWMutex
	fileNames       []string                     // Support mutil-files.
	data            map[string]map[string]string // Section -> key : value
	sectionList     []string                     // Section list
	keyList         map[string][]string          // Section -> Key list
	sectionComments map[string]string            // Sections comments
	keyComments     map[string]map[string]string // Keys comments
	BlockMode       bool
}

// newConfigFile creates an empty configuration representation.
// This representation can be filled with AddSection and AddKey and then
// saved to a file using SaveConfigFile.
func newConfigFile(fileNames []string) *ConfigFile {
	c := new(ConfigFile)
	c.fileNames = fileNames
	c.data = make(map[string]map[string]string)
	c.keyList = make(map[string][]string)
	c.sectionComments = make(map[string]string)
	c.keyComments = make(map[string]map[string]string)
	c.BlockMode = true
	return c
}

// SetValue adds a new section-key-value to the configuration.
// It returns true if the key and value were inserted, and false if the value was overwritten.
// If the section does not exist in advance, it is created.
func (c *ConfigFile) SetValue(section, key, value string) bool {
	if c.BlockMode {
		c.lock.Lock()
		defer c.lock.Unlock()
	}

	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		// Execute add operation.
		c.data[section] = make(map[string]string)
		// Append section to list.
		c.sectionList = append(c.sectionList, section)
	}

	// Check if key exists.
	_, ok := c.data[section][key]
	c.data[section][key] = value
	if !ok {
		// If not exists, append to key list.
		c.keyList[section] = append(c.keyList[section], key)
	}
	return !ok
}

// DeleteKey deletes the key in given section.
// It returns true if the key was deleted, and false if the section or key didn't exist.
func (c *ConfigFile) DeleteKey(section, key string) bool {
	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		return false
	}

	// Check if key exists
	if _, ok := c.data[section][key]; ok {
		// Execute remove operation
		delete(c.data[section], key)
		// Remove comments of key
		c.SetKeyComments(section, key, "")
		// Get index of key
		i := 0
		for _, keyName := range c.keyList[section] {
			if keyName == key {
				break
			}
			i++
		}
		// Remove from key list
		c.keyList[section] =
			append(c.keyList[section][:i], c.keyList[section][i+1:]...)
		return true
	}
	return false
}

// GetValue returns the value of key available in the given section.
// If the value needs to be unfolded (see e.g. %(google)s example in the GoConfig_test.go),
// then String does this unfolding automatically, up to
// _DEPTH_VALUES number of iterations.
// It returns an error if the section or (default)key does not exist and empty string value.
func (c *ConfigFile) GetValue(section, key string) (string, error) {
	if c.BlockMode {
		c.lock.RLock()
		defer c.lock.RUnlock()
	}

	// Blank section name represents DEFAULT section.
	if len(section) == 0 {
		section = DEFAULT_SECTION
	}

	// Check if section exists
	if _, ok := c.data[section]; !ok {
		// Section does not exist.
		return "", getError{SectionNotFound, section}
	}

	// Section exists.
	// Check if key exists or empty value.
	value, ok := c.data[section][key]
	if !ok || len(value) == 0 {
		// Check if it is a sub-section.
		if i := strings.LastIndex(section, "."); i > -1 {
			return c.GetValue(section[:i], key)
		}

		// Return empty value.
		return "", getError{KeyNotFound, key}
	}

	// Key exists.
	var i int
	for i = 0; i < _DEPTH_VALUES; i++ {
		vr := varRegExp.FindString(value)
		if len(vr) == 0 {
			break
		}

		// Take off leading '%(' and trailing ')s'
		noption := strings.TrimLeft(vr, "%(")
		noption = strings.TrimRight(noption, ")s")

		// Search variable in default section
		nvalue, _ := c.GetValue(DEFAULT_SECTION, noption)
		// Search in the same section
		if _, ok := c.data[section][noption]; ok {
			nvalue = c.data[section][noption]
		}

		// substitute by new value and take off leading '%(' and trailing ')s'
		value = strings.Replace(value, vr, nvalue, -1)
	}
	return value, nil
}

// Bool returns bool type value.
func (c *ConfigFile) Bool(section, key string) (bool, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return false, err
	}

	// Convert type.
	return strconv.ParseBool(value)
}

// Float64 returns float64 type value.
func (c *ConfigFile) Float64(section, key string) (float64, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0.0, err
	}

	// Convert type.
	return strconv.ParseFloat(value, 64)
}

// Int returns int type value.
func (c *ConfigFile) Int(section, key string) (int, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0, err
	}

	// Convert type.
	return strconv.Atoi(value)
}

// Int64 returns int64 type value.
func (c *ConfigFile) Int64(section, key string) (int64, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0, err
	}

	// Convert type.
	return strconv.ParseInt(value, 10, 64)
}

// MustValue always returns value without error,
// it returns empty string if error occurs.
func (c *ConfigFile) MustValue(section, key string, defaultVal ...string) string {
	value, err := c.GetValue(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustBool always returns value without error,
// it returns false if error occurs.
func (c *ConfigFile) MustBool(section, key string, defaultVal ...bool) bool {
	value, err := c.Bool(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustFloat64 always returns value without error,
// it returns 0.0 if error occurs.
func (c *ConfigFile) MustFloat64(section, key string, defaultVal ...float64) float64 {
	value, err := c.Float64(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustInt always returns value without error,
// it returns 0 if error occurs.
func (c *ConfigFile) MustInt(section, key string, defaultVal ...int) int {
	value, err := c.Int(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustInt64 always returns value without error,
// it returns 0 if error occurs.
func (c *ConfigFile) MustInt64(section, key string, defaultVal ...int64) int64 {
	value, err := c.Int64(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// GetSectionList returns the list of all sections
// in the same order in the file.
func (c *ConfigFile) GetSectionList() []string {
	list := make([]string, len(c.sectionList))
	copy(list, c.sectionList)
	return list
}

// GetKeyList returns the list of all key in give section
// in the same order in the file.
func (c *ConfigFile) GetKeyList(section string) []string {
	list := make([]string, len(c.keyList[section])-1)
	copy(list, c.keyList[section][1:])
	return list
}

// DeleteSection deletes the entire section by given name.
// It returns true if the section was deleted, and false if the section didn't exist.
func (c *ConfigFile) DeleteSection(section string) bool {
	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		return false
	}

	// Execute remove operation
	delete(c.data, section)
	// Remove comments of section
	c.SetSectionComments(section, "")
	// Get index of section
	i := 0
	for _, secName := range c.sectionList {
		if secName == section {
			break
		}
		i++
	}
	// Remove from section list
	c.sectionList =
		append(c.sectionList[:i], c.sectionList[i+1:]...)
	return true
}

// GetSection returns key-value pairs in given section.
// It section does not exist, returns nil and error.
func (c *ConfigFile) GetSection(section string) (map[string]string, error) {
	// Check if section exists
	if _, ok := c.data[section]; !ok {
		// Section does not exist.
		return nil, getError{SectionNotFound, section}
	}

	// Remove pre-defined key.
	secMap := c.data[section]
	delete(c.data[section], " ")

	// Section exists.
	return secMap, nil
}

// SetSectionComments adds new section comments to the configuration.
// If comments are empty(0 length), it will remove its section comments!
// It returns true if the comments were inserted or removed, and false if the comments were overwritten.
func (c *ConfigFile) SetSectionComments(section, comments string) bool {
	// Check length of comments
	if len(comments) == 0 {
		// Check if section exists
		if _, ok := c.sectionComments[section]; ok {
			// Execute remove operation
			delete(c.sectionComments, section)
		}

		// Not exists can be seen as remove
		return true
	}

	// Check if comments exists
	_, ok := c.sectionComments[section]
	if comments[0] != '#' && comments[0] != ';' {
		comments = "; " + comments
	}
	c.sectionComments[section] = comments
	return !ok
}

// SetKeyComments adds new section-key comments to the configuration.
// If comments are empty(0 length), it will remove its section-key comments!
// It returns true if the comments were inserted or removed, and false if the comments were overwritten.
// If the section does not exist in advance, it is created.
func (c *ConfigFile) SetKeyComments(section, key, comments string) bool {
	// Check if section exists
	if _, ok := c.keyComments[section]; ok {
		// Section exists
		// Check length of comments
		if len(comments) == 0 {
			// Check if key exists
			if _, ok := c.keyComments[section][key]; ok {
				// Execute remove operation
				delete(c.keyComments[section], key)
			}

			// Not exists can be seen as remove
			return true
		}
	} else {
		// Section not exists
		// Check length of comments
		if len(comments) == 0 {
			// Not exists can be seen as remove
			return true
		} else {
			// Execute add operation
			c.keyComments[section] = make(map[string]string)
		}
	}

	// Check if key exists
	_, ok := c.keyComments[section][key]
	if comments[0] != '#' && comments[0] != ';' {
		comments = "; " + comments
	}
	c.keyComments[section][key] = comments
	return !ok
}

// GetSectionComments returns the comments in the given section.
// It returns an empty string(0 length) if the comments do not exist
func (c *ConfigFile) GetSectionComments(section string) (comments string) {
	return c.sectionComments[section]
}

// GetKeyComments returns the comments of key in the given section.
// It returns an empty string(0 length) if the comments do not exist
func (c *ConfigFile) GetKeyComments(section, key string) (comments string) {
	// Check if section exists
	if _, ok := c.keyComments[section]; ok {
		// Exists
		return c.keyComments[section][key]
	}

	// Not exists
	return ""
}

// getError occurs when get value in configuration file with invalid parameter
type getError struct {
	Reason int // Error reason
	Name   string
}

// Implement Error method
func (err getError) Error() string {
	switch err.Reason {
	case SectionNotFound:
		return fmt.Sprintf("section '%s' not found", string(err.Name))
	}

	return "invalid get error"
}
<commit_msg>Mirror update<commit_after>// Copyright 2013-2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// goconfig is a easy-use comments-support configuration file parser.
package goconfig

import (
	"fmt"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

const (
	// Default section name.
	DEFAULT_SECTION = "DEFAULT"
	// Maximum allowed depth when recursively substituing variable names.
	_DEPTH_VALUES = 200

	// Get Errors.
	SectionNotFound = iota
	KeyNotFound
	// Read Errors.
	BlankSection
	// Get and Read Errors.
	CouldNotParse
)

var (
	LineBreak = "\n"
	// %(variable)s
	varRegExp = regexp.MustCompile(`%\(([a-zA-Z0-9_.\-]+)\)s`)
)

func init() {
	if runtime.GOOS == "windows" {
		LineBreak = "\r\n"
	}
}

// ConfigFile is the representation of configuration settings.
// The public interface is entirely through methods.
type ConfigFile struct {
	lock            sync.RWMutex
	fileNames       []string                     // Support mutil-files.
	data            map[string]map[string]string // Section -> key : value
	sectionList     []string                     // Section list
	keyList         map[string][]string          // Section -> Key list
	sectionComments map[string]string            // Sections comments
	keyComments     map[string]map[string]string // Keys comments
	BlockMode       bool
}

// newConfigFile creates an empty configuration representation.
// This representation can be filled with AddSection and AddKey and then
// saved to a file using SaveConfigFile.
func newConfigFile(fileNames []string) *ConfigFile {
	c := new(ConfigFile)
	c.fileNames = fileNames
	c.data = make(map[string]map[string]string)
	c.keyList = make(map[string][]string)
	c.sectionComments = make(map[string]string)
	c.keyComments = make(map[string]map[string]string)
	c.BlockMode = true
	return c
}

// SetValue adds a new section-key-value to the configuration.
// It returns true if the key and value were inserted, and false if the value was overwritten.
// If the section does not exist in advance, it is created.
func (c *ConfigFile) SetValue(section, key, value string) bool {
	if c.BlockMode {
		c.lock.Lock()
		defer c.lock.Unlock()
	}

	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		// Execute add operation.
		c.data[section] = make(map[string]string)
		// Append section to list.
		c.sectionList = append(c.sectionList, section)
	}

	// Check if key exists.
	_, ok := c.data[section][key]
	c.data[section][key] = value
	if !ok {
		// If not exists, append to key list.
		c.keyList[section] = append(c.keyList[section], key)
	}
	return !ok
}

// DeleteKey deletes the key in given section.
// It returns true if the key was deleted, and false if the section or key didn't exist.
func (c *ConfigFile) DeleteKey(section, key string) bool {
	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		return false
	}

	// Check if key exists
	if _, ok := c.data[section][key]; ok {
		// Execute remove operation
		delete(c.data[section], key)
		// Remove comments of key
		c.SetKeyComments(section, key, "")
		// Get index of key
		i := 0
		for _, keyName := range c.keyList[section] {
			if keyName == key {
				break
			}
			i++
		}
		// Remove from key list
		c.keyList[section] =
			append(c.keyList[section][:i], c.keyList[section][i+1:]...)
		return true
	}
	return false
}

// GetValue returns the value of key available in the given section.
// If the value needs to be unfolded (see e.g. %(google)s example in the GoConfig_test.go),
// then String does this unfolding automatically, up to
// _DEPTH_VALUES number of iterations.
// It returns an error if the section or (default)key does not exist and empty string value.
func (c *ConfigFile) GetValue(section, key string) (string, error) {
	if c.BlockMode {
		c.lock.RLock()
		defer c.lock.RUnlock()
	}

	// Blank section name represents DEFAULT section.
	if len(section) == 0 {
		section = DEFAULT_SECTION
	}

	// Check if section exists
	if _, ok := c.data[section]; !ok {
		// Section does not exist.
		return "", getError{SectionNotFound, section}
	}

	// Section exists.
	// Check if key exists or empty value.
	value, ok := c.data[section][key]
	if !ok || len(value) == 0 {
		// Check if it is a sub-section.
		if i := strings.LastIndex(section, "."); i > -1 {
			return c.GetValue(section[:i], key)
		}

		// Return empty value.
		return "", getError{KeyNotFound, key}
	}

	// Key exists.
	var i int
	for i = 0; i < _DEPTH_VALUES; i++ {
		vr := varRegExp.FindString(value)
		if len(vr) == 0 {
			break
		}

		// Take off leading '%(' and trailing ')s'
		noption := strings.TrimLeft(vr, "%(")
		noption = strings.TrimRight(noption, ")s")

		// Search variable in default section
		nvalue, err := c.GetValue(DEFAULT_SECTION, noption)
		if err != nil && section != DEFAULT_SECTION {
			nvalue, _ = c.GetValue(section, noption)
		}
		// Search in the same section
		if _, ok := c.data[section][noption]; ok {
			nvalue = c.data[section][noption]
		}

		// substitute by new value and take off leading '%(' and trailing ')s'
		value = strings.Replace(value, vr, nvalue, -1)
	}
	return value, nil
}

// Bool returns bool type value.
func (c *ConfigFile) Bool(section, key string) (bool, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return false, err
	}

	// Convert type.
	return strconv.ParseBool(value)
}

// Float64 returns float64 type value.
func (c *ConfigFile) Float64(section, key string) (float64, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0.0, err
	}

	// Convert type.
	return strconv.ParseFloat(value, 64)
}

// Int returns int type value.
func (c *ConfigFile) Int(section, key string) (int, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0, err
	}

	// Convert type.
	return strconv.Atoi(value)
}

// Int64 returns int64 type value.
func (c *ConfigFile) Int64(section, key string) (int64, error) {
	// Get string format value.
	value, err := c.GetValue(section, key)
	if err != nil {
		return 0, err
	}

	// Convert type.
	return strconv.ParseInt(value, 10, 64)
}

// MustValue always returns value without error,
// it returns empty string if error occurs.
func (c *ConfigFile) MustValue(section, key string, defaultVal ...string) string {
	value, err := c.GetValue(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustBool always returns value without error,
// it returns false if error occurs.
func (c *ConfigFile) MustBool(section, key string, defaultVal ...bool) bool {
	value, err := c.Bool(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustFloat64 always returns value without error,
// it returns 0.0 if error occurs.
func (c *ConfigFile) MustFloat64(section, key string, defaultVal ...float64) float64 {
	value, err := c.Float64(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustInt always returns value without error,
// it returns 0 if error occurs.
func (c *ConfigFile) MustInt(section, key string, defaultVal ...int) int {
	value, err := c.Int(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// MustInt64 always returns value without error,
// it returns 0 if error occurs.
func (c *ConfigFile) MustInt64(section, key string, defaultVal ...int64) int64 {
	value, err := c.Int64(section, key)
	if err != nil && len(defaultVal) > 0 {
		return defaultVal[0]
	}
	return value
}

// GetSectionList returns the list of all sections
// in the same order in the file.
func (c *ConfigFile) GetSectionList() []string {
	list := make([]string, len(c.sectionList))
	copy(list, c.sectionList)
	return list
}

// GetKeyList returns the list of all key in give section
// in the same order in the file.
func (c *ConfigFile) GetKeyList(section string) []string {
	list := make([]string, len(c.keyList[section])-1)
	copy(list, c.keyList[section][1:])
	return list
}

// DeleteSection deletes the entire section by given name.
// It returns true if the section was deleted, and false if the section didn't exist.
func (c *ConfigFile) DeleteSection(section string) bool {
	// Check if section exists.
	if _, ok := c.data[section]; !ok {
		// Section not exists.
		return false
	}

	// Execute remove operation
	delete(c.data, section)
	// Remove comments of section
	c.SetSectionComments(section, "")
	// Get index of section
	i := 0
	for _, secName := range c.sectionList {
		if secName == section {
			break
		}
		i++
	}
	// Remove from section list
	c.sectionList =
		append(c.sectionList[:i], c.sectionList[i+1:]...)
	return true
}

// GetSection returns key-value pairs in given section.
// It section does not exist, returns nil and error.
func (c *ConfigFile) GetSection(section string) (map[string]string, error) {
	// Check if section exists
	if _, ok := c.data[section]; !ok {
		// Section does not exist.
		return nil, getError{SectionNotFound, section}
	}

	// Remove pre-defined key.
	secMap := c.data[section]
	delete(c.data[section], " ")

	// Section exists.
	return secMap, nil
}

// SetSectionComments adds new section comments to the configuration.
// If comments are empty(0 length), it will remove its section comments!
// It returns true if the comments were inserted or removed, and false if the comments were overwritten.
func (c *ConfigFile) SetSectionComments(section, comments string) bool {
	// Check length of comments
	if len(comments) == 0 {
		// Check if section exists
		if _, ok := c.sectionComments[section]; ok {
			// Execute remove operation
			delete(c.sectionComments, section)
		}

		// Not exists can be seen as remove
		return true
	}

	// Check if comments exists
	_, ok := c.sectionComments[section]
	if comments[0] != '#' && comments[0] != ';' {
		comments = "; " + comments
	}
	c.sectionComments[section] = comments
	return !ok
}

// SetKeyComments adds new section-key comments to the configuration.
// If comments are empty(0 length), it will remove its section-key comments!
// It returns true if the comments were inserted or removed, and false if the comments were overwritten.
// If the section does not exist in advance, it is created.
func (c *ConfigFile) SetKeyComments(section, key, comments string) bool {
	// Check if section exists
	if _, ok := c.keyComments[section]; ok {
		// Section exists
		// Check length of comments
		if len(comments) == 0 {
			// Check if key exists
			if _, ok := c.keyComments[section][key]; ok {
				// Execute remove operation
				delete(c.keyComments[section], key)
			}

			// Not exists can be seen as remove
			return true
		}
	} else {
		// Section not exists
		// Check length of comments
		if len(comments) == 0 {
			// Not exists can be seen as remove
			return true
		} else {
			// Execute add operation
			c.keyComments[section] = make(map[string]string)
		}
	}

	// Check if key exists
	_, ok := c.keyComments[section][key]
	if comments[0] != '#' && comments[0] != ';' {
		comments = "; " + comments
	}
	c.keyComments[section][key] = comments
	return !ok
}

// GetSectionComments returns the comments in the given section.
// It returns an empty string(0 length) if the comments do not exist
func (c *ConfigFile) GetSectionComments(section string) (comments string) {
	return c.sectionComments[section]
}

// GetKeyComments returns the comments of key in the given section.
// It returns an empty string(0 length) if the comments do not exist
func (c *ConfigFile) GetKeyComments(section, key string) (comments string) {
	// Check if section exists
	if _, ok := c.keyComments[section]; ok {
		// Exists
		return c.keyComments[section][key]
	}

	// Not exists
	return ""
}

// getError occurs when get value in configuration file with invalid parameter
type getError struct {
	Reason int // Error reason
	Name   string
}

// Implement Error method
func (err getError) Error() string {
	switch err.Reason {
	case SectionNotFound:
		return fmt.Sprintf("section '%s' not found", string(err.Name))
	}

	return "invalid get error"
}
<|endoftext|>


<commit_before>package registry

import (
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/docker/reference"
	"github.com/docker/docker/registry"
	"github.com/docker/engine-api/types"
	imageParser "github.com/docker/engine-api/types/reference"
	"github.com/nhurel/dim/lib/utils"
	t "github.com/nhurel/dim/types"
	"golang.org/x/net/context"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

// Client defines method to interact with a registry
type Client interface {
	client.Registry
	NewRepository(parsedName reference.Named) (Repository, error)
	Search(query, advanced string, offset, maxResults int) (*t.SearchResults, error)
	WalkRepositories(repositories chan<- Repository) error
	PrintImageInfo(out io.Writer, parsedName reference.Named, tpl *template.Template) error
	DeleteImage(parsedName reference.Named) error
}

// RegistryClient implements Client interface
type registryClient struct {
	client.Registry
	transport   http.RoundTripper
	registryURL string
}

var ctx = context.Background()

// New creates a registry client. Handles getting right credentials from user
func New(registryAuth *types.AuthConfig, registryURL string) (Client, error) {
	var err error
	var reg client.Registry

	if registryURL == "" {
		return nil, fmt.Errorf("No registry URL given")
	}

	transport := http.DefaultTransport

	if registryAuth != nil {
		transport = registry.AuthTransport(transport, registryAuth, true)
	}

	if reg, err = client.NewRegistry(ctx, registryURL, transport); err != nil {
		return nil, err
	}

	repos := make([]string, 1)
	for _, err = reg.Repositories(ctx, repos, ""); err != nil && err != io.EOF; _, err = reg.Repositories(ctx, repos, "") {
		logrus.Debugln("Prompting for credentials")
		if registryAuth == nil {
			registryAuth = &types.AuthConfig{}
		}
		utils.ReadCredentials(registryAuth)
		transport = registry.AuthTransport(transport, registryAuth, true)
		if reg, err = client.NewRegistry(ctx, registryURL, transport); err != nil {
			return nil, err
		}
	}

	return &registryClient{reg, transport, registryURL}, nil
}

// NewRepository creates a Repository object to query the registry about a specific repository
func (c *registryClient) NewRepository(parsedName reference.Named) (Repository, error) {
	logrus.WithField("name", parsedName).Debugln("Creating new repository")

	var repo distribution.Repository
	var err error
	if repo, err = client.NewRepository(ctx, parsedName, c.registryURL, c.transport); err != nil {
		return &registryRepository{}, err
	}

	return &registryRepository{Repository: repo, client: c}, nil
}

// Search runs a search against the registry, handling dim advanced querying option
func (c *registryClient) Search(query, advanced string, offset, maxResults int) (*t.SearchResults, error) {
	q := strings.TrimSpace(query)
	a := strings.TrimSpace(advanced)
	var err error

	var resp *http.Response

	values := url.Values{}
	if a != "" {
		values.Set("a", a)
	}
	if q != "" {
		values.Set("q", q)
	}
	values.Set("f", "full")
	values.Set("offset", strconv.Itoa(offset))
	values.Set("maxResults", strconv.Itoa(maxResults))

	httpClient := http.Client{Transport: c.transport}
	if resp, err = httpClient.PostForm(strings.Join([]string{c.registryURL, "/v1/search"}, ""), values); err != nil {
		return nil, fmt.Errorf("Failed to send request : %v", err)
	}
	defer resp.Body.Close()
	var b []byte
	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
		results := &t.SearchResults{}
		if err := json.NewDecoder(resp.Body).Decode(results); err != nil {
			return nil, fmt.Errorf("Failed to parse response : %v", err)
		}

		return results, nil
	}

	b, _ = ioutil.ReadAll(resp.Body)
	return nil, fmt.Errorf("Server returned an error : %s", string(b))
}

// WalkRepositories walks through all repositories and send them in the given channel
func (c *registryClient) WalkRepositories(repositories chan<- Repository) error {
	return WalkRepositories(c, repositories)
}

// WalkRepositories walks through all repositories and ssend them in the given channel
func WalkRepositories(c Client, repositories chan<- Repository) error {
	var err error

	var n int
	registries := make([]string, 20)
	defer close(repositories)
	last := ""
	for stop := false; !stop; {

		if n, err = c.Repositories(nil, registries, last); err != nil && err != io.EOF {
			logrus.WithField("n", n).WithError(err).Errorln("Failed to get repostories")
			return err
		}

		stop = (err == io.EOF)

		for i := 0; i < n; i++ {
			last = registries[i]

			var parsedName reference.Named

			l := logrus.WithField("repository", last)
			l.Infoln("Indexing repository")
			if parsedName, err = reference.ParseNamed(last); err != nil {
				logrus.WithError(err).WithField("name", last).Errorln("Failed to parse repository name")
				continue
			}

			var repository Repository

			if repository, err = c.NewRepository(parsedName); err != nil {
				logrus.WithError(err).WithField("name", last).Errorln("Failed to fetch repository info")
				continue
			}
			repositories <- repository
		}

	}
	return nil

}

// PrintImageInfo prints the info about an image available on the remote registry
func (c *registryClient) PrintImageInfo(w io.Writer, parsedName reference.Named, tpl *template.Template) error {
	var repository Repository
	var err error
	name, _ := reference.ParseNamed(parsedName.Name()[strings.Index(parsedName.Name(), "/")+1:])
	if repository, err = c.NewRepository(name); err != nil {
		logrus.WithError(err).Errorln("Failed to fetch repository info")
		return err
	}

	var tag string
	switch parsedName := parsedName.(type) {
	case reference.NamedTagged:
		tag = parsedName.Tag()
	default:
		tag = "latest"
	}

	var image *Image
	if image, err = repository.Image(tag); err != nil {
		logrus.WithError(err).Errorln("Failed to fetch image info")
		return err
	}

	info := &types.ImageInspect{
		RepoTags: []string{image.Tag},
		ID:       image.ImageID(),
		Config:   image.Config,
	}

	return tpl.Execute(w, info)
}

// DeleteImage deletes the image on the remote registry
func (c *registryClient) DeleteImage(parsedName reference.Named) error {
	logrus.WithField("parsedName", parsedName.String()).Debugln("Entering DeleteImage")
	var repo Repository
	var err error
	name, _ := reference.ParseNamed(parsedName.Name()[strings.Index(parsedName.Name(), "/")+1:])
	if repo, err = c.NewRepository(name); err != nil {
		return err
	}

	tag := imageParser.GetTagFromNamedRef(parsedName)

	if tag == "" {
		tag = "latest"
	}

	logrus.Debugln("Deleting image")
	if err = repo.DeleteImage(tag); err != nil {
		logrus.WithError(err).Errorln("Failed to delete image on the remote registry")
		return err
	}

	return nil
}
<commit_msg>Returns an error if the registry url is wrong instead of prompting credentials indefinitely #patch<commit_after>package registry

import (
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution"
	"github.com/docker/distribution/registry/client"
	"github.com/docker/docker/reference"
	"github.com/docker/docker/registry"
	"github.com/docker/engine-api/types"
	imageParser "github.com/docker/engine-api/types/reference"
	"github.com/nhurel/dim/lib/utils"
	t "github.com/nhurel/dim/types"
	"golang.org/x/net/context"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"text/template"
)

// Client defines method to interact with a registry
type Client interface {
	client.Registry
	NewRepository(parsedName reference.Named) (Repository, error)
	Search(query, advanced string, offset, maxResults int) (*t.SearchResults, error)
	WalkRepositories(repositories chan<- Repository) error
	PrintImageInfo(out io.Writer, parsedName reference.Named, tpl *template.Template) error
	DeleteImage(parsedName reference.Named) error
}

// RegistryClient implements Client interface
type registryClient struct {
	client.Registry
	transport   http.RoundTripper
	registryURL string
}

var ctx = context.Background()

// New creates a registry client. Handles getting right credentials from user
func New(registryAuth *types.AuthConfig, registryURL string) (Client, error) {
	var err error
	var reg client.Registry

	if registryURL == "" {
		return nil, fmt.Errorf("No registry URL given")
	}

	transport := http.DefaultTransport

	if registryAuth != nil {
		transport = registry.AuthTransport(transport, registryAuth, true)
	}

	if reg, err = client.NewRegistry(ctx, registryURL, transport); err != nil {
		return nil, err
	}

	repos := make([]string, 1)
	for _, err = reg.Repositories(ctx, repos, ""); err != nil && err != io.EOF; _, err = reg.Repositories(ctx, repos, "") {
		switch err.(type) {
		case *client.UnexpectedHTTPStatusError, *url.Error:
			return nil, fmt.Errorf("Failed to join the registry : %v", err)
		}
		logrus.Debugln("Prompting for credentials")
		if registryAuth == nil {
			registryAuth = &types.AuthConfig{}
		}
		utils.ReadCredentials(registryAuth)
		transport = registry.AuthTransport(transport, registryAuth, true)
		if reg, err = client.NewRegistry(ctx, registryURL, transport); err != nil {
			return nil, err
		}
	}

	return &registryClient{reg, transport, registryURL}, nil
}

// NewRepository creates a Repository object to query the registry about a specific repository
func (c *registryClient) NewRepository(parsedName reference.Named) (Repository, error) {
	logrus.WithField("name", parsedName).Debugln("Creating new repository")

	var repo distribution.Repository
	var err error
	if repo, err = client.NewRepository(ctx, parsedName, c.registryURL, c.transport); err != nil {
		return &registryRepository{}, err
	}

	return &registryRepository{Repository: repo, client: c}, nil
}

// Search runs a search against the registry, handling dim advanced querying option
func (c *registryClient) Search(query, advanced string, offset, maxResults int) (*t.SearchResults, error) {
	q := strings.TrimSpace(query)
	a := strings.TrimSpace(advanced)
	var err error

	var resp *http.Response

	values := url.Values{}
	if a != "" {
		values.Set("a", a)
	}
	if q != "" {
		values.Set("q", q)
	}
	values.Set("f", "full")
	values.Set("offset", strconv.Itoa(offset))
	values.Set("maxResults", strconv.Itoa(maxResults))

	httpClient := http.Client{Transport: c.transport}
	if resp, err = httpClient.PostForm(strings.Join([]string{c.registryURL, "/v1/search"}, ""), values); err != nil {
		return nil, fmt.Errorf("Failed to send request : %v", err)
	}
	defer resp.Body.Close()
	var b []byte
	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
		results := &t.SearchResults{}
		if err := json.NewDecoder(resp.Body).Decode(results); err != nil {
			return nil, fmt.Errorf("Failed to parse response : %v", err)
		}

		return results, nil
	}

	b, _ = ioutil.ReadAll(resp.Body)
	return nil, fmt.Errorf("Server returned an error : %s", string(b))
}

// WalkRepositories walks through all repositories and send them in the given channel
func (c *registryClient) WalkRepositories(repositories chan<- Repository) error {
	return WalkRepositories(c, repositories)
}

// WalkRepositories walks through all repositories and ssend them in the given channel
func WalkRepositories(c Client, repositories chan<- Repository) error {
	var err error

	var n int
	registries := make([]string, 20)
	defer close(repositories)
	last := ""
	for stop := false; !stop; {

		if n, err = c.Repositories(nil, registries, last); err != nil && err != io.EOF {
			logrus.WithField("n", n).WithError(err).Errorln("Failed to get repostories")
			return err
		}

		stop = (err == io.EOF)

		for i := 0; i < n; i++ {
			last = registries[i]

			var parsedName reference.Named

			l := logrus.WithField("repository", last)
			l.Infoln("Indexing repository")
			if parsedName, err = reference.ParseNamed(last); err != nil {
				logrus.WithError(err).WithField("name", last).Errorln("Failed to parse repository name")
				continue
			}

			var repository Repository

			if repository, err = c.NewRepository(parsedName); err != nil {
				logrus.WithError(err).WithField("name", last).Errorln("Failed to fetch repository info")
				continue
			}
			repositories <- repository
		}

	}
	return nil

}

// PrintImageInfo prints the info about an image available on the remote registry
func (c *registryClient) PrintImageInfo(w io.Writer, parsedName reference.Named, tpl *template.Template) error {
	var repository Repository
	var err error
	name, _ := reference.ParseNamed(parsedName.Name()[strings.Index(parsedName.Name(), "/")+1:])
	if repository, err = c.NewRepository(name); err != nil {
		logrus.WithError(err).Errorln("Failed to fetch repository info")
		return err
	}

	var tag string
	switch parsedName := parsedName.(type) {
	case reference.NamedTagged:
		tag = parsedName.Tag()
	default:
		tag = "latest"
	}

	var image *Image
	if image, err = repository.Image(tag); err != nil {
		logrus.WithError(err).Errorln("Failed to fetch image info")
		return err
	}

	info := &types.ImageInspect{
		RepoTags: []string{image.Tag},
		ID:       image.ImageID(),
		Config:   image.Config,
	}

	return tpl.Execute(w, info)
}

// DeleteImage deletes the image on the remote registry
func (c *registryClient) DeleteImage(parsedName reference.Named) error {
	logrus.WithField("parsedName", parsedName.String()).Debugln("Entering DeleteImage")
	var repo Repository
	var err error
	name, _ := reference.ParseNamed(parsedName.Name()[strings.Index(parsedName.Name(), "/")+1:])
	if repo, err = c.NewRepository(name); err != nil {
		return err
	}

	tag := imageParser.GetTagFromNamedRef(parsedName)

	if tag == "" {
		tag = "latest"
	}

	logrus.Debugln("Deleting image")
	if err = repo.DeleteImage(tag); err != nil {
		logrus.WithError(err).Errorln("Failed to delete image on the remote registry")
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package utp

/*
#include "utp.h"
#include <stdbool.h>

struct utp_process_udp_args {
	const byte *buf;
	size_t len;
	const struct sockaddr *sa;
	socklen_t sal;
};

void process_received_messages(utp_context *ctx, struct utp_process_udp_args *args, size_t argslen)
{
	bool gotUtp = false;
	size_t i;
	for (i = 0; i < argslen; i++) {
		struct utp_process_udp_args *a = &args[i];
		//if (!a->len) continue;
		if (utp_process_udp(ctx, a->buf, a->len, a->sa, a->sal)) {
			gotUtp = true;
		}
	}
	if (gotUtp) {
		utp_issue_deferred_acks(ctx);
		utp_check_timeouts(ctx);
	}
}
*/
import "C"
import (
	"context"
	"errors"
	"net"
	"runtime/pprof"
	"time"

	"github.com/anacrolix/missinggo/inproc"
	"github.com/anacrolix/mmsg"
)

type Socket struct {
	pc            net.PacketConn
	ctx           *C.utp_context
	backlog       chan *Conn
	closed        bool
	conns         map[*C.utp_socket]*Conn
	nonUtpReads   chan packet
	writeDeadline time.Time
	readDeadline  time.Time
}

var (
	_               net.PacketConn = (*Socket)(nil)
	_               net.Listener   = (*Socket)(nil)
	errSocketClosed                = errors.New("Socket closed")
)

type packet struct {
	b    []byte
	from net.Addr
}

func listenPacket(network, addr string) (pc net.PacketConn, err error) {
	if network == "inproc" {
		return inproc.ListenPacket(network, addr)
	}
	return net.ListenPacket(network, addr)
}

func NewSocket(network, addr string) (*Socket, error) {
	pc, err := listenPacket(network, addr)
	if err != nil {
		return nil, err
	}
	mu.Lock()
	defer mu.Unlock()
	ctx := C.utp_init(2)
	if ctx == nil {
		panic(ctx)
	}
	ctx.setCallbacks()
	if utpLogging {
		ctx.setOption(C.UTP_LOG_NORMAL, 1)
		ctx.setOption(C.UTP_LOG_MTU, 1)
		ctx.setOption(C.UTP_LOG_DEBUG, 1)
	}
	s := &Socket{
		pc:          pc,
		ctx:         ctx,
		backlog:     make(chan *Conn, 5),
		conns:       make(map[*C.utp_socket]*Conn),
		nonUtpReads: make(chan packet, 100),
	}
	libContextToSocket[ctx] = s
	go s.timeoutChecker()
	go s.packetReader()
	return s, nil
}

func (s *Socket) onLibSocketDestroyed(ls *C.utp_socket) {
	delete(s.conns, ls)
}

func (s *Socket) newConn(us *C.utp_socket) *Conn {
	c := &Conn{
		s:         s,
		us:        us,
		localAddr: s.pc.LocalAddr(),
	}
	c.cond.L = &mu
	s.conns[us] = c
	c.writeDeadlineTimer = time.AfterFunc(-1, c.cond.Broadcast)
	c.readDeadlineTimer = time.AfterFunc(-1, c.cond.Broadcast)
	return c
}

const maxNumBuffers = 16

func (s *Socket) packetReader() {
	mc := mmsg.NewConn(s.pc)
	// Increasing the messages increases the memory use, but also means we can
	// reduces utp_issue_deferred_acks and syscalls which should improve
	// efficiency. On the flip side, not all OSs implement batched reads.
	ms := make([]mmsg.Message, func() int {
		if mc.Err() == nil {
			return maxNumBuffers
		} else {
			return 1
		}
	}())
	for i := range ms {
		// The IPv4 UDP limit is allegedly about 64 KiB, and this message has
		// been seen on receiving on Windows with just 0x1000: wsarecvfrom: A
		// message sent on a datagram socket was larger than the internal
		// message buffer or some other network limit, or the buffer used to
		// receive a datagram into was smaller than the datagram itself.
		ms[i].Buffers = [][]byte{make([]byte, 0x10000)}
	}
	// Some crap OSs like Windoze will raise errors in Reads that don't
	// actually mean we should stop.
	consecutiveErrors := 0
	for {
		// In C, all the reads are processed and when it threatens to block,
		// we're supposed to call utp_issue_deferred_acks.
		n, err := mc.RecvMsgs(ms)
		if n == 1 {
			singleMsgRecvs.Add(1)
		}
		if n > 1 {
			multiMsgRecvs.Add(1)
		}
		if err != nil {
			mu.Lock()
			closed := s.closed
			mu.Unlock()
			if closed {
				// We don't care.
				return
			}
			// See https://github.com/anacrolix/torrent/issues/83. If we get
			// an endless stream of errors (such as the PacketConn being
			// Closed outside of our control, this work around may need to be
			// reconsidered.
			Logger.Printf("ignoring socket read error: %s", err)
			consecutiveErrors++
			if consecutiveErrors >= 100 {
				Logger.Print("too many consecutive errors, closing socket")
				s.Close()
				return
			}
			continue
		}
		consecutiveErrors = 0
		expMap.Add("successful mmsg receive calls", 1)
		expMap.Add("received messages", int64(n))
		s.processReceivedMessages(ms[:n])
	}
}

func (s *Socket) processReceivedMessages(ms []mmsg.Message) {
	mu.Lock()
	defer mu.Unlock()
	if s.closed {
		return
	}
	if processPacketsInC {
		var args [maxNumBuffers]C.struct_utp_process_udp_args
		for i, m := range ms {
			a := &args[i]
			a.buf = (*C.byte)(&m.Buffers[0][0])
			a.len = C.size_t(m.N)
			a.sa, a.sal = netAddrToLibSockaddr(m.Addr)
		}
		C.process_received_messages(s.ctx, &args[0], C.size_t(len(ms)))
	} else {
		gotUtp := false
		for _, m := range ms {
			gotUtp = s.processReceivedMessage(m.Buffers[0][:m.N], m.Addr) || gotUtp
		}
		if gotUtp {
			s.afterReceivingUtpMessages()
		}
	}
}

func (s *Socket) afterReceivingUtpMessages() {
	pprof.Do(context.Background(), pprof.Labels("go-libutp", "afterReceivingUtpMessages"), func(context.Context) {
		C.utp_issue_deferred_acks(s.ctx)
		// TODO: When is this done in C?
		C.utp_check_timeouts(s.ctx)
	})
}

func (s *Socket) processReceivedMessage(b []byte, addr net.Addr) (utp bool) {
	if s.utpProcessUdp(b, addr) {
		socketUtpPacketsReceived.Add(1)
		return true
	} else {
		s.onReadNonUtp(b, addr)
		return false
	}
}

// Process packet batches entirely from C, reducing CGO overhead. Currently
// requires GODEBUG=cgocheck=0.
const processPacketsInC = false

// Wraps libutp's utp_process_udp, returning relevant information.
func (s *Socket) utpProcessUdp(b []byte, addr net.Addr) (utp bool) {
	sa, sal := netAddrToLibSockaddr(addr)
	if len(b) == 0 {
		// The implementation of utp_process_udp rejects null buffers, and
		// anything smaller than the UTP header size. It's also prone to
		// assert on those, which we don't want to trigger.
		return false
	}
	ret := C.utp_process_udp(s.ctx, (*C.byte)(&b[0]), C.size_t(len(b)), sa, sal)
	switch ret {
	case 1:
		return true
	case 0:
		return false
	default:
		panic(ret)
	}
}

func (s *Socket) timeoutChecker() {
	for {
		mu.Lock()
		if s.closed {
			mu.Unlock()
			return
		}
		// C.utp_issue_deferred_acks(s.ctx)
		C.utp_check_timeouts(s.ctx)
		mu.Unlock()
		time.Sleep(500 * time.Millisecond)
	}
}

func (s *Socket) Close() error {
	mu.Lock()
	defer mu.Unlock()
	if s.closed {
		return nil
	}
	// Calling this deletes the pointer. It must not be referred to after
	// this.
	C.utp_destroy(s.ctx)
	s.ctx = nil
	s.pc.Close()
	close(s.backlog)
	close(s.nonUtpReads)
	s.closed = true
	return nil
}

func (s *Socket) Addr() net.Addr {
	return s.pc.LocalAddr()
}

func (s *Socket) LocalAddr() net.Addr {
	return s.pc.LocalAddr()
}

func (s *Socket) Accept() (net.Conn, error) {
	nc, ok := <-s.backlog
	if !ok {
		return nil, errors.New("closed")
	}
	return nc, nil
}

func (s *Socket) Dial(addr string) (net.Conn, error) {
	return s.DialTimeout(addr, 0)
}

func (s *Socket) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) {
	ctx := context.Background()
	if timeout != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, timeout)
		defer cancel()
	}
	return s.DialContext(ctx, "", addr)
}

func (s *Socket) resolveAddr(network, addr string) (net.Addr, error) {
	if network == "" {
		network = s.Addr().Network()
	}
	return resolveAddr(network, addr)
}

func resolveAddr(network, addr string) (net.Addr, error) {
	switch network {
	case "inproc":
		return inproc.ResolveAddr(network, addr)
	default:
		return net.ResolveUDPAddr(network, addr)
	}
}

// Passing an empty network will use the network of the Socket's listener.
func (s *Socket) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
	c, err := s.NewConn()
	if err != nil {
		return nil, err
	}
	err = c.Connect(ctx, network, addr)
	if err != nil {
		c.Close()
		return nil, err
	}
	return c, nil
}

func (s *Socket) NewConn() (*Conn, error) {
	mu.Lock()
	defer mu.Unlock()
	if s.closed {
		return nil, errors.New("socket closed")
	}
	return s.newConn(C.utp_create_socket(s.ctx)), nil
}

func (s *Socket) pushBacklog(c *Conn) {
	select {
	case s.backlog <- c:
	default:
		c.close()
	}
}

func (s *Socket) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	p, ok := <-s.nonUtpReads
	if !ok {
		err = errors.New("closed")
		return
	}
	n = copy(b, p.b)
	addr = p.from
	return
}

func (s *Socket) onReadNonUtp(b []byte, from net.Addr) {
	socketNonUtpPacketsReceived.Add(1)
	select {
	case s.nonUtpReads <- packet{append([]byte(nil), b...), from}:
	default:
		// log.Printf("dropped non utp packet: no room in buffer")
		nonUtpPacketsDropped.Add(1)
	}
}

func (s *Socket) SetReadDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) SetWriteDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) SetDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) WriteTo(b []byte, addr net.Addr) (int, error) {
	return s.pc.WriteTo(b, addr)
}

func (s *Socket) ReadBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_get_option(s.ctx, C.UTP_RCVBUF))
}

func (s *Socket) WriteBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_get_option(s.ctx, C.UTP_SNDBUF))
}

func (s *Socket) SetWriteBufferLen(len int) {
	mu.Lock()
	defer mu.Unlock()
	i := C.utp_context_set_option(s.ctx, C.UTP_SNDBUF, C.int(len))
	if i != 0 {
		panic(i)
	}
}

func (s *Socket) SetOption(opt Option, val int) int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_set_option(s.ctx, opt, C.int(val)))
}
<commit_msg>Add Socket.closeLocked<commit_after>package utp

/*
#include "utp.h"
#include <stdbool.h>

struct utp_process_udp_args {
	const byte *buf;
	size_t len;
	const struct sockaddr *sa;
	socklen_t sal;
};

void process_received_messages(utp_context *ctx, struct utp_process_udp_args *args, size_t argslen)
{
	bool gotUtp = false;
	size_t i;
	for (i = 0; i < argslen; i++) {
		struct utp_process_udp_args *a = &args[i];
		//if (!a->len) continue;
		if (utp_process_udp(ctx, a->buf, a->len, a->sa, a->sal)) {
			gotUtp = true;
		}
	}
	if (gotUtp) {
		utp_issue_deferred_acks(ctx);
		utp_check_timeouts(ctx);
	}
}
*/
import "C"
import (
	"context"
	"errors"
	"net"
	"runtime/pprof"
	"time"

	"github.com/anacrolix/missinggo/inproc"
	"github.com/anacrolix/mmsg"
)

type Socket struct {
	pc            net.PacketConn
	ctx           *C.utp_context
	backlog       chan *Conn
	closed        bool
	conns         map[*C.utp_socket]*Conn
	nonUtpReads   chan packet
	writeDeadline time.Time
	readDeadline  time.Time
}

var (
	_               net.PacketConn = (*Socket)(nil)
	_               net.Listener   = (*Socket)(nil)
	errSocketClosed                = errors.New("Socket closed")
)

type packet struct {
	b    []byte
	from net.Addr
}

func listenPacket(network, addr string) (pc net.PacketConn, err error) {
	if network == "inproc" {
		return inproc.ListenPacket(network, addr)
	}
	return net.ListenPacket(network, addr)
}

func NewSocket(network, addr string) (*Socket, error) {
	pc, err := listenPacket(network, addr)
	if err != nil {
		return nil, err
	}
	mu.Lock()
	defer mu.Unlock()
	ctx := C.utp_init(2)
	if ctx == nil {
		panic(ctx)
	}
	ctx.setCallbacks()
	if utpLogging {
		ctx.setOption(C.UTP_LOG_NORMAL, 1)
		ctx.setOption(C.UTP_LOG_MTU, 1)
		ctx.setOption(C.UTP_LOG_DEBUG, 1)
	}
	s := &Socket{
		pc:          pc,
		ctx:         ctx,
		backlog:     make(chan *Conn, 5),
		conns:       make(map[*C.utp_socket]*Conn),
		nonUtpReads: make(chan packet, 100),
	}
	libContextToSocket[ctx] = s
	go s.timeoutChecker()
	go s.packetReader()
	return s, nil
}

func (s *Socket) onLibSocketDestroyed(ls *C.utp_socket) {
	delete(s.conns, ls)
}

func (s *Socket) newConn(us *C.utp_socket) *Conn {
	c := &Conn{
		s:         s,
		us:        us,
		localAddr: s.pc.LocalAddr(),
	}
	c.cond.L = &mu
	s.conns[us] = c
	c.writeDeadlineTimer = time.AfterFunc(-1, c.cond.Broadcast)
	c.readDeadlineTimer = time.AfterFunc(-1, c.cond.Broadcast)
	return c
}

const maxNumBuffers = 16

func (s *Socket) packetReader() {
	mc := mmsg.NewConn(s.pc)
	// Increasing the messages increases the memory use, but also means we can
	// reduces utp_issue_deferred_acks and syscalls which should improve
	// efficiency. On the flip side, not all OSs implement batched reads.
	ms := make([]mmsg.Message, func() int {
		if mc.Err() == nil {
			return maxNumBuffers
		} else {
			return 1
		}
	}())
	for i := range ms {
		// The IPv4 UDP limit is allegedly about 64 KiB, and this message has
		// been seen on receiving on Windows with just 0x1000: wsarecvfrom: A
		// message sent on a datagram socket was larger than the internal
		// message buffer or some other network limit, or the buffer used to
		// receive a datagram into was smaller than the datagram itself.
		ms[i].Buffers = [][]byte{make([]byte, 0x10000)}
	}
	// Some crap OSs like Windoze will raise errors in Reads that don't
	// actually mean we should stop.
	consecutiveErrors := 0
	for {
		// In C, all the reads are processed and when it threatens to block,
		// we're supposed to call utp_issue_deferred_acks.
		n, err := mc.RecvMsgs(ms)
		if n == 1 {
			singleMsgRecvs.Add(1)
		}
		if n > 1 {
			multiMsgRecvs.Add(1)
		}
		if err != nil {
			mu.Lock()
			closed := s.closed
			mu.Unlock()
			if closed {
				// We don't care.
				return
			}
			// See https://github.com/anacrolix/torrent/issues/83. If we get
			// an endless stream of errors (such as the PacketConn being
			// Closed outside of our control, this work around may need to be
			// reconsidered.
			Logger.Printf("ignoring socket read error: %s", err)
			consecutiveErrors++
			if consecutiveErrors >= 100 {
				Logger.Print("too many consecutive errors, closing socket")
				s.Close()
				return
			}
			continue
		}
		consecutiveErrors = 0
		expMap.Add("successful mmsg receive calls", 1)
		expMap.Add("received messages", int64(n))
		s.processReceivedMessages(ms[:n])
	}
}

func (s *Socket) processReceivedMessages(ms []mmsg.Message) {
	mu.Lock()
	defer mu.Unlock()
	if s.closed {
		return
	}
	if processPacketsInC {
		var args [maxNumBuffers]C.struct_utp_process_udp_args
		for i, m := range ms {
			a := &args[i]
			a.buf = (*C.byte)(&m.Buffers[0][0])
			a.len = C.size_t(m.N)
			a.sa, a.sal = netAddrToLibSockaddr(m.Addr)
		}
		C.process_received_messages(s.ctx, &args[0], C.size_t(len(ms)))
	} else {
		gotUtp := false
		for _, m := range ms {
			gotUtp = s.processReceivedMessage(m.Buffers[0][:m.N], m.Addr) || gotUtp
		}
		if gotUtp {
			s.afterReceivingUtpMessages()
		}
	}
}

func (s *Socket) afterReceivingUtpMessages() {
	pprof.Do(context.Background(), pprof.Labels("go-libutp", "afterReceivingUtpMessages"), func(context.Context) {
		C.utp_issue_deferred_acks(s.ctx)
		// TODO: When is this done in C?
		C.utp_check_timeouts(s.ctx)
	})
}

func (s *Socket) processReceivedMessage(b []byte, addr net.Addr) (utp bool) {
	if s.utpProcessUdp(b, addr) {
		socketUtpPacketsReceived.Add(1)
		return true
	} else {
		s.onReadNonUtp(b, addr)
		return false
	}
}

// Process packet batches entirely from C, reducing CGO overhead. Currently
// requires GODEBUG=cgocheck=0.
const processPacketsInC = false

// Wraps libutp's utp_process_udp, returning relevant information.
func (s *Socket) utpProcessUdp(b []byte, addr net.Addr) (utp bool) {
	sa, sal := netAddrToLibSockaddr(addr)
	if len(b) == 0 {
		// The implementation of utp_process_udp rejects null buffers, and
		// anything smaller than the UTP header size. It's also prone to
		// assert on those, which we don't want to trigger.
		return false
	}
	ret := C.utp_process_udp(s.ctx, (*C.byte)(&b[0]), C.size_t(len(b)), sa, sal)
	switch ret {
	case 1:
		return true
	case 0:
		return false
	default:
		panic(ret)
	}
}

func (s *Socket) timeoutChecker() {
	for {
		mu.Lock()
		if s.closed {
			mu.Unlock()
			return
		}
		// C.utp_issue_deferred_acks(s.ctx)
		C.utp_check_timeouts(s.ctx)
		mu.Unlock()
		time.Sleep(500 * time.Millisecond)
	}
}

func (s *Socket) Close() error {
	mu.Lock()
	defer mu.Unlock()
	return s.closeLocked()
}

func (s *Socket) closeLocked() error {
	if s.closed {
		return nil
	}
	// Calling this deletes the pointer. It must not be referred to after
	// this.
	C.utp_destroy(s.ctx)
	s.ctx = nil
	s.pc.Close()
	close(s.backlog)
	close(s.nonUtpReads)
	s.closed = true
	return nil
}

func (s *Socket) Addr() net.Addr {
	return s.pc.LocalAddr()
}

func (s *Socket) LocalAddr() net.Addr {
	return s.pc.LocalAddr()
}

func (s *Socket) Accept() (net.Conn, error) {
	nc, ok := <-s.backlog
	if !ok {
		return nil, errors.New("closed")
	}
	return nc, nil
}

func (s *Socket) Dial(addr string) (net.Conn, error) {
	return s.DialTimeout(addr, 0)
}

func (s *Socket) DialTimeout(addr string, timeout time.Duration) (net.Conn, error) {
	ctx := context.Background()
	if timeout != 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, timeout)
		defer cancel()
	}
	return s.DialContext(ctx, "", addr)
}

func (s *Socket) resolveAddr(network, addr string) (net.Addr, error) {
	if network == "" {
		network = s.Addr().Network()
	}
	return resolveAddr(network, addr)
}

func resolveAddr(network, addr string) (net.Addr, error) {
	switch network {
	case "inproc":
		return inproc.ResolveAddr(network, addr)
	default:
		return net.ResolveUDPAddr(network, addr)
	}
}

// Passing an empty network will use the network of the Socket's listener.
func (s *Socket) DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
	c, err := s.NewConn()
	if err != nil {
		return nil, err
	}
	err = c.Connect(ctx, network, addr)
	if err != nil {
		c.Close()
		return nil, err
	}
	return c, nil
}

func (s *Socket) NewConn() (*Conn, error) {
	mu.Lock()
	defer mu.Unlock()
	if s.closed {
		return nil, errors.New("socket closed")
	}
	return s.newConn(C.utp_create_socket(s.ctx)), nil
}

func (s *Socket) pushBacklog(c *Conn) {
	select {
	case s.backlog <- c:
	default:
		c.close()
	}
}

func (s *Socket) ReadFrom(b []byte) (n int, addr net.Addr, err error) {
	p, ok := <-s.nonUtpReads
	if !ok {
		err = errors.New("closed")
		return
	}
	n = copy(b, p.b)
	addr = p.from
	return
}

func (s *Socket) onReadNonUtp(b []byte, from net.Addr) {
	socketNonUtpPacketsReceived.Add(1)
	select {
	case s.nonUtpReads <- packet{append([]byte(nil), b...), from}:
	default:
		// log.Printf("dropped non utp packet: no room in buffer")
		nonUtpPacketsDropped.Add(1)
	}
}

func (s *Socket) SetReadDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) SetWriteDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) SetDeadline(t time.Time) error {
	panic("not implemented")
}

func (s *Socket) WriteTo(b []byte, addr net.Addr) (int, error) {
	return s.pc.WriteTo(b, addr)
}

func (s *Socket) ReadBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_get_option(s.ctx, C.UTP_RCVBUF))
}

func (s *Socket) WriteBufferLen() int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_get_option(s.ctx, C.UTP_SNDBUF))
}

func (s *Socket) SetWriteBufferLen(len int) {
	mu.Lock()
	defer mu.Unlock()
	i := C.utp_context_set_option(s.ctx, C.UTP_SNDBUF, C.int(len))
	if i != 0 {
		panic(i)
	}
}

func (s *Socket) SetOption(opt Option, val int) int {
	mu.Lock()
	defer mu.Unlock()
	return int(C.utp_context_set_option(s.ctx, opt, C.int(val)))
}
<|endoftext|>


<commit_before>package commonmark

import (
	"bytes"
	"log"
)

// Block represents a node in the parse tree.
//
// "We can think of a document as a sequence of blocks—structural elements like
// paragraphs, block quotations, lists, headers, rules, and code blocks. Blocks
// can contain other blocks, or they can contain inline content: words, spaces,
// links, emphasized text, images, and inline code."
type Block interface {
	// Children returns the list of child blocks, in order.
	Children() []Block

	// AppendChild appends a child block to the list of children.
	AppendChild(Block)

	// AppendLine appends the given line to the list of lines.
	AppendLine([]byte)

	// AcceptsLines returns whether blocks of this type can contain lines.
	AcceptsLines() bool
}

// block implements the common part of the Block interface.
type block struct {
	children      []Block
	content       []byte
	inlineContent Inline
}

func (b *block) Children() []Block {
	return b.children
}

func (b *block) AppendChild(child Block) {
	b.children = append(b.children, child)
}

func (b *block) AppendLine(line []byte) {
	b.content = append(b.content, line...)
}

// document is the root node of the parse tree.
type document struct {
	block
}

func (d *document) AcceptsLines() bool {
	return false
}

// paragraph represents a paragraph of text.
//
// "A sequence of non-blank lines that cannot be interpreted as other kinds of
// blocks forms a paragraph."
type paragraph struct {
	block
}

func (p *paragraph) AppendLine(line []byte) {
	p.block.AppendLine(bytes.TrimLeft(line, " "))
}

func (p *paragraph) AcceptsLines() bool {
	return true
}

// indentedCodeBlock represents an indented code block.
//
// "An indented code block is composed of one or more indented chunks separated
// by blank lines. An indented chunk is a sequence of non-blank lines, each
// indented four or more spaces."
type indentedCodeBlock struct {
	block
}

func (c *indentedCodeBlock) AcceptsLines() bool {
	return true
}

func parseBlocks(data []byte) (*document, error) {
	scanner := newScanner(data)
	doc := &document{}
	openBlocks := []Block{doc}
	for scanner.Scan() {
		line := scanner.Bytes()
		line = tabsToSpaces(line)
		line = append(line, '\n')

		indent := indentation(line)
		blank := line[indent] == '\n'

		var openBlock Block
		var i int
		for i, openBlock = range openBlocks {
			allMatched := true
			switch openBlock.(type) {
			case *indentedCodeBlock:
				if indent >= 4 || blank {
					if len(line) > 4 {
						line = line[4:]
					} else {
						line = line[len(line)-1:]
					}
				} else {
					allMatched = false
				}
			case *paragraph:
				// TODO close paragraph on blank line
				break
			}
			if !allMatched {
				assertf(i > 0, "allMatched should not become false at the document root")
				openBlock = openBlocks[i-1]
				break
			}
		}

		if openBlock.AcceptsLines() {
			// We're good.
		} else if indentation(line) >= 4 {
			code := &indentedCodeBlock{}
			openBlock.AppendChild(code)
			openBlocks = append(openBlocks, code)
			openBlock = code
			line = line[4:]
		} else if blank {
			continue
		} else {
			par := &paragraph{}
			openBlock.AppendChild(par)
			openBlocks = append(openBlocks, par)
			openBlock = par
		}
		assertf(openBlock.AcceptsLines(), "remaining types of block should all accept lines, but %T does not", openBlock)
		openBlock.AppendLine(line)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return doc, nil
}

// indentation returns the index of the first non-space. If the line consists
// entirely of spaces, it returns the index of the newline character.
func indentation(line []byte) int {
	for i, c := range line {
		if c != ' ' {
			return i
		}
	}
	assertf(false, "indentation() expects line %q to end in newline character", line)
	return 0
}

func processInlines(b Block) {
	switch t := b.(type) {
	case *paragraph:
		// "Final spaces are stripped before inline parsing, so a paragraph that
		// ends with two or more spaces will not end with a hard line break."
		t.inlineContent = parseInlines(bytes.TrimRight(t.content, " "))
	}

	for _, child := range b.Children() {
		processInlines(child)
	}
}

func assertf(condition bool, format string, args ...interface{}) {
	if !condition {
		log.Panicf(format, args...)
	}
}
<commit_msg>Close paragraphs upon encountering a blank line<commit_after>package commonmark

import (
	"bytes"
	"log"
)

// Block represents a node in the parse tree.
//
// "We can think of a document as a sequence of blocks—structural elements like
// paragraphs, block quotations, lists, headers, rules, and code blocks. Blocks
// can contain other blocks, or they can contain inline content: words, spaces,
// links, emphasized text, images, and inline code."
type Block interface {
	// Children returns the list of child blocks, in order.
	Children() []Block

	// AppendChild appends a child block to the list of children.
	AppendChild(Block)

	// AppendLine appends the given line to the list of lines.
	AppendLine([]byte)

	// AcceptsLines returns whether blocks of this type can contain lines.
	AcceptsLines() bool
}

// block implements the common part of the Block interface.
type block struct {
	children      []Block
	content       []byte
	inlineContent Inline
}

func (b *block) Children() []Block {
	return b.children
}

func (b *block) AppendChild(child Block) {
	b.children = append(b.children, child)
}

func (b *block) AppendLine(line []byte) {
	b.content = append(b.content, line...)
}

// document is the root node of the parse tree.
type document struct {
	block
}

func (d *document) AcceptsLines() bool {
	return false
}

// paragraph represents a paragraph of text.
//
// "A sequence of non-blank lines that cannot be interpreted as other kinds of
// blocks forms a paragraph."
type paragraph struct {
	block
}

func (p *paragraph) AppendLine(line []byte) {
	p.block.AppendLine(bytes.TrimLeft(line, " "))
}

func (p *paragraph) AcceptsLines() bool {
	return true
}

// indentedCodeBlock represents an indented code block.
//
// "An indented code block is composed of one or more indented chunks separated
// by blank lines. An indented chunk is a sequence of non-blank lines, each
// indented four or more spaces."
type indentedCodeBlock struct {
	block
}

func (c *indentedCodeBlock) AcceptsLines() bool {
	return true
}

func parseBlocks(data []byte) (*document, error) {
	scanner := newScanner(data)
	doc := &document{}
	openBlocks := []Block{doc}
	for scanner.Scan() {
		line := scanner.Bytes()
		line = tabsToSpaces(line)
		line = append(line, '\n')

		indent := indentation(line)
		blank := line[indent] == '\n'

		var openBlock Block
		var i int
		for i, openBlock = range openBlocks {
			allMatched := true
			switch openBlock.(type) {
			case *indentedCodeBlock:
				if indent >= 4 || blank {
					if len(line) > 4 {
						line = line[4:]
					} else {
						line = line[len(line)-1:]
					}
				} else {
					allMatched = false
				}
			case *paragraph:
				if blank {
					allMatched = false
				}
				break
			}
			if !allMatched {
				assertf(i > 0, "allMatched should not become false at the document root")
				i--
				break
			}
		}

		openBlocks = openBlocks[:i+1]
		openBlock = openBlocks[len(openBlocks)-1]

		if openBlock.AcceptsLines() {
			// We're good.
		} else if indentation(line) >= 4 {
			code := &indentedCodeBlock{}
			openBlock.AppendChild(code)
			openBlocks = append(openBlocks, code)
			openBlock = code
			line = line[4:]
		} else if blank {
			continue
		} else {
			par := &paragraph{}
			openBlock.AppendChild(par)
			openBlocks = append(openBlocks, par)
			openBlock = par
		}
		assertf(openBlock.AcceptsLines(), "remaining types of block should all accept lines, but %T does not", openBlock)
		openBlock.AppendLine(line)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return doc, nil
}

// indentation returns the index of the first non-space. If the line consists
// entirely of spaces, it returns the index of the newline character.
func indentation(line []byte) int {
	for i, c := range line {
		if c != ' ' {
			return i
		}
	}
	assertf(false, "indentation() expects line %q to end in newline character", line)
	return 0
}

func processInlines(b Block) {
	switch t := b.(type) {
	case *paragraph:
		// "Final spaces are stripped before inline parsing, so a paragraph that
		// ends with two or more spaces will not end with a hard line break."
		t.inlineContent = parseInlines(bytes.TrimRight(t.content, " "))
	}

	for _, child := range b.Children() {
		processInlines(child)
	}
}

func assertf(condition bool, format string, args ...interface{}) {
	if !condition {
		log.Panicf(format, args...)
	}
}
<|endoftext|>


<commit_before>package getter

import (
	"fmt"
	"path/filepath"
	"strings"
)

// SourceDirSubdir takes a source and returns the URL without
// the subdir and the subdir.
func SourceDirSubdir(src string) (url string, subdir string) {
	// Calcaulate an offset to avoid accidentally marking the scheme
	// as the dir.
	var offset int
	if idx := strings.Index(src, "://"); idx > -1 {
		offset = idx + 3
	}

	// URL might contains another url in query parameters
	stop := len(src)
	if idx := strings.Index(src, "?"); idx > -1 {
		stop = idx
	}

	// First see if we even have an explicit subdir
	idx := strings.Index(src[offset:stop], "//")
	if idx == -1 {
		return src, ""
	}

	idx += offset
	subdir = src[idx+2:]
	src = src[:idx]

	// Next, check if we have query parameters and push them onto the
	// URL.
	if idx = strings.Index(subdir, "?"); idx > -1 {
		query := subdir[idx:]
		subdir = subdir[:idx]
		src += query
	}

	return src, subdir
}

// SubdirGlob returns the actual subdir with globbing processed.
//
// dst should be a destination directory that is already populated (the
// download is complete) and subDir should be the set subDir. If subDir
// is an empty string, this returns an empty string.
//
// The returned path is the full absolute path.
func SubdirGlob(dst, subDir string) (string, error) {
	matches, err := filepath.Glob(filepath.Join(dst, subDir))
	if err != nil {
		return "", err
	}

	if len(matches) == 0 {
		return "", fmt.Errorf("subdir %q not found", subDir)
	}

	if len(matches) > 1 {
		return "", fmt.Errorf("subdir %q matches multiple paths", subDir)
	}

	return matches[0], nil
}
<commit_msg>remove named parameters of SourceDirSubdir and document it with examples instead<commit_after>package getter

import (
	"fmt"
	"path/filepath"
	"strings"
)

// SourceDirSubdir takes a source URL and returns a tuple of the URL without
// the subdir and the subdir.
//
// ex:
//   dom.com/path/?q=p               => dom.com/path/?q=p, ""
//   proto://dom.com/path//*?q=p     => proto://dom.com/path?q=p, "*"
//   proto://dom.com/path//path2?q=p => proto://dom.com/path?q=p, "path2"
//
func SourceDirSubdir(src string) (string, string) {

	// URL might contains another url in query parameters
	stop := len(src)
	if idx := strings.Index(src, "?"); idx > -1 {
		stop = idx
	}

	// Calculate an offset to avoid accidentally marking the scheme
	// as the dir.
	var offset int
	if idx := strings.Index(src[:stop], "://"); idx > -1 {
		offset = idx + 3
	}

	// First see if we even have an explicit subdir
	idx := strings.Index(src[offset:stop], "//")
	if idx == -1 {
		return src, ""
	}

	idx += offset
	subdir := src[idx+2:]
	src = src[:idx]

	// Next, check if we have query parameters and push them onto the
	// URL.
	if idx = strings.Index(subdir, "?"); idx > -1 {
		query := subdir[idx:]
		subdir = subdir[:idx]
		src += query
	}

	return src, subdir
}

// SubdirGlob returns the actual subdir with globbing processed.
//
// dst should be a destination directory that is already populated (the
// download is complete) and subDir should be the set subDir. If subDir
// is an empty string, this returns an empty string.
//
// The returned path is the full absolute path.
func SubdirGlob(dst, subDir string) (string, error) {
	matches, err := filepath.Glob(filepath.Join(dst, subDir))
	if err != nil {
		return "", err
	}

	if len(matches) == 0 {
		return "", fmt.Errorf("subdir %q not found", subDir)
	}

	if len(matches) > 1 {
		return "", fmt.Errorf("subdir %q matches multiple paths", subDir)
	}

	return matches[0], nil
}
<|endoftext|>