<commit_before>// Copyright 2015 Netflix, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"time"

	"github.com/netflix/rend/binprot"
	"github.com/netflix/rend/common"
	"github.com/netflix/rend/handlers"
	"github.com/netflix/rend/metrics"
	"github.com/netflix/rend/orcas"
	"github.com/netflix/rend/textprot"
)

func ListenAndServe(l ListenArgs, s ServerConst, o orcas.OrcaConst, h1, h2 handlers.HandlerConst) {
	var listener net.Listener
	var err error

	switch l.Type {
	case ListenTCP:
		listener, err = net.Listen("tcp", fmt.Sprintf(":%d", l.Port))
		if err != nil {
			log.Printf("Error binding to port %d\n", l.Port)
			return
		}

	case ListenUnix:
		err = os.Remove(l.Path)
		if err != nil && !os.IsNotExist(err) {
			log.Printf("Error removing previous unix socket file at %s\n", l.Path)
		}
		listener, err = net.Listen("unix", l.Path)
		if err != nil {
			log.Printf("Error binding to unix socket at %s\n", l.Path)
			return
		}

	default:
		log.Panicf("Unsupported server listen type: %v", l.Type)
	}

	for {
		remote, err := listener.Accept()
		if err != nil {
			log.Println("Error accepting connection from remote:", err.Error())
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedExt)

		if l.Type == ListenTCP {
			tcpRemote := remote.(*net.TCPConn)
			tcpRemote.SetKeepAlive(true)
			tcpRemote.SetKeepAlivePeriod(30 * time.Second)
		}

		// construct L1 handler using given constructor
		l1, err := h1()
		if err != nil {
			log.Println("Error opening connection to L1:", err.Error())
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedL1)

		// construct l2
		l2, err := h2()
		if err != nil {
			log.Println("Error opening connection to L2:", err.Error())
			l1.Close()
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedL2)

		// spin off a goroutine here to handle determining the protocol used for the connection.
		// The server loop can't be started until the protocol is known. Another goroutine is
		// necessary here because we don't want to block accepting new connections if the current
		// new connection doesn't send data immediately.
		go func(remoteConn net.Conn) {
			remoteReader := bufio.NewReader(remoteConn)
			remoteWriter := bufio.NewWriter(remoteConn)

			var reqParser common.RequestParser
			var responder common.Responder

			// A connection is either binary protocol or text. It cannot switch between the two.
			// This is the way memcached handles protocols, so it can be as strict here.
			binary, err := isBinaryRequest(remoteReader)
			if err != nil {
				// must be an IO error. Abort!
				abort([]io.Closer{remoteConn, l1, l2}, err)
				return
			}

			if binary {
				reqParser = binprot.NewBinaryParser(remoteReader)
				responder = binprot.NewBinaryResponder(remoteWriter)
			} else {
				reqParser = textprot.NewTextParser(remoteReader)
				responder = textprot.NewTextResponder(remoteWriter)
			}

			server := s([]io.Closer{remoteConn, l1, l2}, reqParser, o(l1, l2, responder))

			go server.Loop()
		}(remote)
	}
}
<commit_msg>Server listener now panics if it can't bind to the port.<commit_after>// Copyright 2015 Netflix, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"time"

	"github.com/netflix/rend/binprot"
	"github.com/netflix/rend/common"
	"github.com/netflix/rend/handlers"
	"github.com/netflix/rend/metrics"
	"github.com/netflix/rend/orcas"
	"github.com/netflix/rend/textprot"
)

func ListenAndServe(l ListenArgs, s ServerConst, o orcas.OrcaConst, h1, h2 handlers.HandlerConst) {
	var listener net.Listener
	var err error

	switch l.Type {
	case ListenTCP:
		listener, err = net.Listen("tcp", fmt.Sprintf(":%d", l.Port))
		if err != nil {
			log.Panicf("Error binding to port %d: %v\n", l.Port, err.Error())
		}

	case ListenUnix:
		err = os.Remove(l.Path)
		if err != nil && !os.IsNotExist(err) {
			log.Panicf("Error removing previous unix socket file at %s\n", l.Path)
		}
		listener, err = net.Listen("unix", l.Path)
		if err != nil {
			log.Panicf("Error binding to unix socket at %s: %v\n", l.Path, err.Error())
		}

	default:
		log.Panicf("Unsupported server listen type: %v", l.Type)
	}

	for {
		remote, err := listener.Accept()
		if err != nil {
			log.Println("Error accepting connection from remote:", err.Error())
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedExt)

		if l.Type == ListenTCP {
			tcpRemote := remote.(*net.TCPConn)
			tcpRemote.SetKeepAlive(true)
			tcpRemote.SetKeepAlivePeriod(30 * time.Second)
		}

		// construct L1 handler using given constructor
		l1, err := h1()
		if err != nil {
			log.Println("Error opening connection to L1:", err.Error())
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedL1)

		// construct l2
		l2, err := h2()
		if err != nil {
			log.Println("Error opening connection to L2:", err.Error())
			l1.Close()
			remote.Close()
			continue
		}
		metrics.IncCounter(MetricConnectionsEstablishedL2)

		// spin off a goroutine here to handle determining the protocol used for the connection.
		// The server loop can't be started until the protocol is known. Another goroutine is
		// necessary here because we don't want to block accepting new connections if the current
		// new connection doesn't send data immediately.
		go func(remoteConn net.Conn) {
			remoteReader := bufio.NewReader(remoteConn)
			remoteWriter := bufio.NewWriter(remoteConn)

			var reqParser common.RequestParser
			var responder common.Responder

			// A connection is either binary protocol or text. It cannot switch between the two.
			// This is the way memcached handles protocols, so it can be as strict here.
			binary, err := isBinaryRequest(remoteReader)
			if err != nil {
				// must be an IO error. Abort!
				abort([]io.Closer{remoteConn, l1, l2}, err)
				return
			}

			if binary {
				reqParser = binprot.NewBinaryParser(remoteReader)
				responder = binprot.NewBinaryResponder(remoteWriter)
			} else {
				reqParser = textprot.NewTextParser(remoteReader)
				responder = textprot.NewTextResponder(remoteWriter)
			}

			server := s([]io.Closer{remoteConn, l1, l2}, reqParser, o(l1, l2, responder))

			go server.Loop()
		}(remote)
	}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
)

type xywingTechnique struct {
	*basicSolveTechnique
}

func (self *xywingTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason about what the proper value should be for this.
	multiplier := 1.0
	if step != nil && step.TargetCells != nil && !step.TargetCells.SameBlock() {
		multiplier = 1.5
	}
	return self.difficultyHelper(60.0) * multiplier
}

func (self *xywingTechnique) Description(step *SolveStep) string {
	//TODO: make this description more clear
	return fmt.Sprintf("%s can only be two values, and cells %s have those two possibilities, plus one other, so if you put either of the main cell's two possibiltiies in, it forces the intersection of the other two cells to not have %s",
		step.PointerCells[0:1].Description(),
		step.PointerCells[1:3].Description(),
		step.TargetNums.Description(),
	)
}

func (self *xywingTechnique) normalizeStep(step *SolveStep) {
	//Do what the normal base technique does, but don't sort pointerCells, since their precise
	//order encodes which is the pivotCell.
	step.TargetCells.Sort()
	step.TargetNums.Sort()
	step.PointerNums.Sort()
}

func (self *xywingTechnique) Variants() []string {
	return []string{self.Name(), self.Name() + " (Same Block)"}
}

func (self *xywingTechnique) variant(step *SolveStep) string {
	if step.TargetCells.SameBlock() {
		return self.Name() + " (Same Block)"
	}
	return self.Name()
}

func (self *xywingTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := grid.queue().NewGetter()

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		pivot := getter.GetSmallerThan(3)

		if pivot == nil {
			break
		}

		pivotCell := pivot.(*Cell)

		possibilities := pivotCell.Possibilities()

		if len(possibilities) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		//OK, this is our pivot cell. Let's consider its two possibilities to
		//be X and Y.

		x := possibilities[0]
		y := possibilities[1]

		//We want to find two neighbors of the pivot cell that have two
		//possibilities that are XZ and YZ. So we're looking for neighbors
		//that have two possibilities, one of which is either x or y.

		xList := pivotCell.Neighbors().FilterByPossible(x).FilterByNumPossibilities(2)
		yList := pivotCell.Neighbors().FilterByPossible(y).FilterByNumPossibilities(2)

		//Now, we'll check for each possible value of Z
		for z := 1; z <= DIM; z++ {
			//z can't be either x or y, so don't do that work
			if z == x || z == y {
				continue
			}

			zFilteredXList := xList.FilterByPossible(z)
			zFilteredYList := yList.FilterByPossible(z)

			//If there area any items left, we've found candidates.
			if len(zFilteredXList) == 0 || len(zFilteredYList) == 0 {
				//TOOD: we can save a tiny bit of work by checking if
				//zFilteredXList is len() 0 before calculating zFilteredYList
				continue
			}

			//Okay, now the two-up combinations of all cells in the
			//two lists are all candidates. Often it will only be 1
			//cell in each list, but we'll general case this.
			for _, xCell := range zFilteredXList {
				for _, yCell := range zFilteredYList {
					//xCell, yCell is a possible pointing pair group.

					//find cells that are in both neighbor lists
					intersection := xCell.Neighbors().toCellSet().intersection(yCell.Neighbors().toCellSet()).toSlice(grid)

					//TODO: technically the filterByPossible(z) below will
					//also filter out cells that are already set, but this is
					//more semanitcally clear.
					intersection = intersection.FilterByHasPossibilities()

					//TODO: consider if we actually need to remove all of these cells;
					//it might never be able to be in the list anyway.
					//TODO: xCell, yCell can't be in there, because xCell is not in its
					//neighbors and yCell isn't in its neighbors. But it's semantically
					//clearer to do it here, maybe, to make clear the intention?
					intersection = intersection.RemoveCells(CellSlice{pivotCell, xCell, yCell})

					affectedCells := intersection.FilterByPossible(z)

					if len(affectedCells) == 0 {
						continue
					}

					if !affectedCells.SameBlock() {
						//The affected cells are not all in the same block,
						//so create chunked step variants.

						for _, block := range affectedCells.AllBlocks() {
							filter := func(cell *Cell) bool {
								return cell.Block() == block
							}
							chunkedAffectedCells := affectedCells.Filter(filter)

							step := &SolveStep{self,
								chunkedAffectedCells,
								IntSlice{z},
								CellSlice{pivotCell, xCell, yCell},
								IntSlice{x, y, z},
								nil,
							}

							if step.IsUseful(grid) {
								select {
								case results <- step:
								case <-done:
									return
								}
							}
						}

					}

					//Okay, we have a candidate step (unchunked). Is it useful?
					step := &SolveStep{self,
						affectedCells,
						IntSlice{z},
						CellSlice{pivotCell, xCell, yCell},
						IntSlice{x, y, z},
						nil,
					}

					if step.IsUseful(grid) {
						select {
						case results <- step:
						case <-done:
							return
						}
					}

				}
			}

		}

	}
}
<commit_msg>Visit possibilities in XYWing in random order. Fixes #114.<commit_after>package sudoku

import (
	"fmt"
	"math/rand"
)

type xywingTechnique struct {
	*basicSolveTechnique
}

func (self *xywingTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason about what the proper value should be for this.
	multiplier := 1.0
	if step != nil && step.TargetCells != nil && !step.TargetCells.SameBlock() {
		multiplier = 1.5
	}
	return self.difficultyHelper(60.0) * multiplier
}

func (self *xywingTechnique) Description(step *SolveStep) string {
	//TODO: make this description more clear
	return fmt.Sprintf("%s can only be two values, and cells %s have those two possibilities, plus one other, so if you put either of the main cell's two possibiltiies in, it forces the intersection of the other two cells to not have %s",
		step.PointerCells[0:1].Description(),
		step.PointerCells[1:3].Description(),
		step.TargetNums.Description(),
	)
}

func (self *xywingTechnique) normalizeStep(step *SolveStep) {
	//Do what the normal base technique does, but don't sort pointerCells, since their precise
	//order encodes which is the pivotCell.
	step.TargetCells.Sort()
	step.TargetNums.Sort()
	step.PointerNums.Sort()
}

func (self *xywingTechnique) Variants() []string {
	return []string{self.Name(), self.Name() + " (Same Block)"}
}

func (self *xywingTechnique) variant(step *SolveStep) string {
	if step.TargetCells.SameBlock() {
		return self.Name() + " (Same Block)"
	}
	return self.Name()
}

func (self *xywingTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := grid.queue().NewGetter()

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		pivot := getter.GetSmallerThan(3)

		if pivot == nil {
			break
		}

		pivotCell := pivot.(*Cell)

		possibilities := pivotCell.Possibilities()

		if len(possibilities) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		//OK, this is our pivot cell. Let's consider its two possibilities to
		//be X and Y.

		x := possibilities[0]
		y := possibilities[1]

		//We want to find two neighbors of the pivot cell that have two
		//possibilities that are XZ and YZ. So we're looking for neighbors
		//that have two possibilities, one of which is either x or y.

		xList := pivotCell.Neighbors().FilterByPossible(x).FilterByNumPossibilities(2)
		yList := pivotCell.Neighbors().FilterByPossible(y).FilterByNumPossibilities(2)

		//Now, we'll check for each possible value of Z
		for _, z := range rand.Perm(DIM + 1) {

			//z is 1-indexed, but perm returns a 0-indexed list
			if z == 0 {
				continue
			}

			//z can't be either x or y, so don't do that work
			if z == x || z == y {
				continue
			}

			zFilteredXList := xList.FilterByPossible(z)
			zFilteredYList := yList.FilterByPossible(z)

			//If there area any items left, we've found candidates.
			if len(zFilteredXList) == 0 || len(zFilteredYList) == 0 {
				//TOOD: we can save a tiny bit of work by checking if
				//zFilteredXList is len() 0 before calculating zFilteredYList
				continue
			}

			//Okay, now the two-up combinations of all cells in the
			//two lists are all candidates. Often it will only be 1
			//cell in each list, but we'll general case this.
			for _, xCell := range zFilteredXList {
				for _, yCell := range zFilteredYList {
					//xCell, yCell is a possible pointing pair group.

					//find cells that are in both neighbor lists
					intersection := xCell.Neighbors().toCellSet().intersection(yCell.Neighbors().toCellSet()).toSlice(grid)

					//TODO: technically the filterByPossible(z) below will
					//also filter out cells that are already set, but this is
					//more semanitcally clear.
					intersection = intersection.FilterByHasPossibilities()

					//TODO: consider if we actually need to remove all of these cells;
					//it might never be able to be in the list anyway.
					//TODO: xCell, yCell can't be in there, because xCell is not in its
					//neighbors and yCell isn't in its neighbors. But it's semantically
					//clearer to do it here, maybe, to make clear the intention?
					intersection = intersection.RemoveCells(CellSlice{pivotCell, xCell, yCell})

					affectedCells := intersection.FilterByPossible(z)

					if len(affectedCells) == 0 {
						continue
					}

					if !affectedCells.SameBlock() {
						//The affected cells are not all in the same block,
						//so create chunked step variants.

						for _, block := range affectedCells.AllBlocks() {
							filter := func(cell *Cell) bool {
								return cell.Block() == block
							}
							chunkedAffectedCells := affectedCells.Filter(filter)

							step := &SolveStep{self,
								chunkedAffectedCells,
								IntSlice{z},
								CellSlice{pivotCell, xCell, yCell},
								IntSlice{x, y, z},
								nil,
							}

							if step.IsUseful(grid) {
								select {
								case results <- step:
								case <-done:
									return
								}
							}
						}

					}

					//Okay, we have a candidate step (unchunked). Is it useful?
					step := &SolveStep{self,
						affectedCells,
						IntSlice{z},
						CellSlice{pivotCell, xCell, yCell},
						IntSlice{x, y, z},
						nil,
					}

					if step.IsUseful(grid) {
						select {
						case results <- step:
						case <-done:
							return
						}
					}

				}
			}

		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/http"
)

type hstsServer struct{}

func (server hstsServer) Handle(pattern string, handler http.Handler) {
	server.HandleFunc(pattern, handler.ServeHTTP)
}

func (hstsServer) HandleFunc(pattern string, handlerFunc http.HandlerFunc) {
	http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
		if hsts(w, r) {
			handlerFunc(w, r)
		}
	})
}

func isLocalhost(hostport string) bool {
	host, _, err := net.SplitHostPort(hostport)
	return err == nil && host == "localhost"
}

// `cont` indicates whether the callee should continue further processing.
func hsts(w http.ResponseWriter, r *http.Request) (cont bool) {
	if r.TLS != nil || maybeAppEngineHTTPS(r) {
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
	}

	switch {
	case (r.Host == "hstspreload.appspot.com"):
		u := fmt.Sprintf("https://hstspreload.org%s", r.URL.Path)
		http.Redirect(w, r, u, http.StatusFound)
		return false
	case (r.TLS != nil), isLocalhost(r.Host), maybeAppEngineHTTPS(r), maybeAppEngineCron(r):
		return true
	default:
		// The redirect below causes problems with Managed VMs/Flexible Environments.
		// In a standalone server we'd handle the redirect here, but we let app.yaml
		// handle it for now.

		// u := fmt.Sprintf("https://%s%s", r.Host, r.URL.Path)
		// http.Redirect(w, r, u, http.StatusMovedPermanently)
		return false
	}
}

// Note: This can be spoofed when not run on App Engine/Flexible Environment.
func maybeAppEngineCron(r *http.Request) bool {
	return r.Header.Get("X-Appengine-Cron") == "true"
}

// Note: This can be spoofed when not run on App Engine/Flexible Environment.
func maybeAppEngineHTTPS(r *http.Request) bool {
	return r.Header.Get("X-Appengine-Https") == "on"
}
<commit_msg>Change hstspreload.org redirect to a 301. #yolo<commit_after>package main

import (
	"fmt"
	"net"
	"net/http"
)

type hstsServer struct{}

func (server hstsServer) Handle(pattern string, handler http.Handler) {
	server.HandleFunc(pattern, handler.ServeHTTP)
}

func (hstsServer) HandleFunc(pattern string, handlerFunc http.HandlerFunc) {
	http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) {
		if hsts(w, r) {
			handlerFunc(w, r)
		}
	})
}

func isLocalhost(hostport string) bool {
	host, _, err := net.SplitHostPort(hostport)
	return err == nil && host == "localhost"
}

// `cont` indicates whether the callee should continue further processing.
func hsts(w http.ResponseWriter, r *http.Request) (cont bool) {
	if r.TLS != nil || maybeAppEngineHTTPS(r) {
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains; preload")
	}

	switch {
	case (r.Host == "hstspreload.appspot.com"):
		u := fmt.Sprintf("https://hstspreload.org%s", r.URL.Path)
		http.Redirect(w, r, u, http.StatusMovedPermanently)
		return false
	case (r.TLS != nil), isLocalhost(r.Host), maybeAppEngineHTTPS(r), maybeAppEngineCron(r):
		return true
	default:
		// The redirect below causes problems with Managed VMs/Flexible Environments.
		// In a standalone server we'd handle the redirect here, but we let app.yaml
		// handle it for now.

		// u := fmt.Sprintf("https://%s%s", r.Host, r.URL.Path)
		// http.Redirect(w, r, u, http.StatusMovedPermanently)
		return false
	}
}

// Note: This can be spoofed when not run on App Engine/Flexible Environment.
func maybeAppEngineCron(r *http.Request) bool {
	return r.Header.Get("X-Appengine-Cron") == "true"
}

// Note: This can be spoofed when not run on App Engine/Flexible Environment.
func maybeAppEngineHTTPS(r *http.Request) bool {
	return r.Header.Get("X-Appengine-Https") == "on"
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

// Error defines a standard application error.
type Error struct {
	// Machine-readable error code.
	Code string `json:"code,omitempty"`

	// Human-readable message.
	Message string `json:"message"`
}

// Error returns the string representation of the error message.
func (e *Error) Error() string {
	return e.Message
}
<commit_msg>Unexport http.Error.Code<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

// Error defines a standard application error.
type Error struct {
	// Machine-readable error code.
	code string `json:"code,omitempty"`

	// Human-readable message.
	Message string `json:"message"`
}

// Error returns the string representation of the error message.
func (e *Error) Error() string {
	return e.Message
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"hash"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
	"github.com/kardianos/govendor/vendorfile"

	"golang.org/x/tools/go/vcs"
)

func skipperTree(name string, dir bool) bool {
	return false
}
func skipperPackage(name string, dir bool) bool {
	return dir
}

func (ctx *Context) VerifyVendor() (outOfDate []*vendorfile.Package, err error) {
	vf := ctx.VendorFile
	root := filepath.Join(ctx.RootDir, ctx.VendorFolder)
	add := func(vp *vendorfile.Package) {
		outOfDate = append(outOfDate, vp)
	}
	for _, vp := range vf.Package {
		if vp.Remove {
			continue
		}
		if len(vp.Path) == 0 {
			continue
		}
		if len(vp.ChecksumSHA1) == 0 {
			add(vp)
			continue
		}
		fp := filepath.Join(root, pathos.SlashToFilepath(vp.Path))
		h := sha1.New()
		sk := skipperPackage
		if vp.Tree {
			sk = skipperTree
		}
		err = getHash(root, fp, h, sk)
		if err != nil {
			return
		}
		checksum := base64.StdEncoding.EncodeToString(h.Sum(nil))
		if vp.ChecksumSHA1 != checksum {
			add(vp)
		}
	}
	return
}

func getHash(root, fp string, h hash.Hash, skipper func(name string, isDir bool) bool) error {
	rel := pathos.FileTrimPrefix(fp, root)
	rel = pathos.SlashToImportPath(rel)
	rel = strings.Trim(rel, "/")

	h.Write([]byte(rel))

	dir, err := os.Open(fp)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return fmt.Errorf("Failed to open dir %q: %v", fp, err)
	}
	filelist, err := dir.Readdir(-1)
	dir.Close()
	if err != nil {
		return fmt.Errorf("Failed to read dir %q: %v", fp, err)
	}
	sort.Sort(fileInfoSort(filelist))
	for _, fi := range filelist {
		if skipper(fi.Name(), fi.IsDir()) {
			continue
		}
		p := filepath.Join(fp, fi.Name())
		if fi.IsDir() {
			err = getHash(root, p, h, skipper)
			if err != nil {
				return err
			}
			continue
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		h.Write([]byte(fi.Name()))
		_, err = io.Copy(h, f)
		f.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

// similarSegments compares two paths and determines if they have
// similar prefix segments. For example github.com/kardianos/rdb and
// github.com/kardianos/govendor have 2 similar segments.
func similarSegments(p1, p2 string) (string, int) {
	seg1 := strings.Split(p1, "/")
	seg2 := strings.Split(p2, "/")

	ct := len(seg1)
	if len(seg2) < ct {
		ct = len(seg2)
	}

	similar := &bytes.Buffer{}
	for i := 0; i < ct; i++ {
		if seg1[i] != seg2[i] {
			return similar.String(), i
		}
		if i != 0 {
			similar.WriteRune('/')
		}
		similar.WriteString(seg1[i])
	}
	return similar.String(), ct
}

type remoteFailure struct {
	Path string
	Msg  string
	Err  error
}

func (fail remoteFailure) Error() string {
	return fmt.Sprintf("Failed for %q (%s): %v", fail.Path, fail.Msg, fail.Err)
}

type remoteFailureList []remoteFailure

func (list remoteFailureList) Error() string {
	if len(list) == 0 {
		return "(no remote failure)"
	}
	buf := &bytes.Buffer{}
	buf.WriteString("Remotes failed for:\n")
	for _, item := range list {
		buf.WriteString("\t")
		buf.WriteString(item.Error())
		buf.WriteString("\n")
	}
	return buf.String()
}

type VCSCmd struct {
	*vcs.Cmd
}

func (vcsCmd *VCSCmd) RevisionSync(dir, revision string) error {
	return vcsCmd.run(dir, vcsCmd.TagSyncCmd, "tag", revision)
}

func (v *VCSCmd) run(dir string, cmd string, keyval ...string) error {
	_, err := v.run1(dir, cmd, keyval, true)
	return err
}

// run1 is the generalized implementation of run and runOutput.
func (vcsCmd *VCSCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) {
	v := vcsCmd.Cmd
	m := make(map[string]string)
	for i := 0; i < len(keyval); i += 2 {
		m[keyval[i]] = keyval[i+1]
	}
	args := strings.Fields(cmdline)
	for i, arg := range args {
		args[i] = expand(m, arg)
	}

	_, err := exec.LookPath(v.Cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr,
			"go: missing %s command. See http://golang.org/s/gogetcmd\n",
			v.Name)
		return nil, err
	}

	cmd := exec.Command(v.Cmd, args...)
	cmd.Dir = dir
	cmd.Env = envForDir(cmd.Dir)
	if vcs.ShowCmd {
		fmt.Printf("cd %s\n", dir)
		fmt.Printf("%s %s\n", v.Cmd, strings.Join(args, " "))
	}
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	err = cmd.Run()
	out := buf.Bytes()
	if err != nil {
		if verbose {
			fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " "))
			os.Stderr.Write(out)
		}
		return nil, err
	}
	return out, nil
}

// expand rewrites s to replace {k} with match[k] for each key k in match.
func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

// envForDir returns a copy of the environment
// suitable for running in the given directory.
// The environment is the current process's environment
// but with an updated $PWD, so that an os.Getwd in the
// child will be faster.
func envForDir(dir string) []string {
	env := os.Environ()
	// Internally we only use rooted paths, so dir is rooted.
	// Even if dir is not rooted, no harm done.
	return mergeEnvLists([]string{"PWD=" + dir}, env)
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
func mergeEnvLists(in, out []string) []string {
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}

func updateVcsCmd(cmd *vcs.Cmd) *VCSCmd {
	switch cmd.Name {
	case "Git":
		cmd.TagSyncCmd = "reset --hard {tag}"
		cmd.TagSyncDefault = "reset --hard origin/master"
		cmd.DownloadCmd = "fetch"
	case "Mercurial":
	case "Bazaar":
	case "Subversion":
	}
	return &VCSCmd{Cmd: cmd}
}

var isSecureScheme = map[string]bool{
	"https":   true,
	"git+ssh": true,
	"bzr+ssh": true,
	"svn+ssh": true,
	"ssh":     true,
}

func vcsIsSecure(repo string) bool {
	u, err := url.Parse(repo)
	if err != nil {
		// If repo is not a URL, it's not secure.
		return false
	}
	return isSecureScheme[u.Scheme]
}

// Sync checks for outdated packages in the vendor folder and fetches the
// correct revision from the remote.
func (ctx *Context) Sync(dryrun bool) (err error) {
	// vcs.ShowCmd = true
	outOfDate, err := ctx.VerifyVendor()
	if err != nil {
		return fmt.Errorf("Failed to verify checksums: %v", err)
	}
	// GOPATH includes the src dir, move up a level.
	cacheRoot := filepath.Join(ctx.RootGopath, "..", ".cache", "govendor")
	err = os.MkdirAll(cacheRoot, 0700)
	if err != nil {
		return err
	}

	// collect errors and proceed where you can.
	rem := remoteFailureList{}

	h := sha1.New()
	updatedVendorFile := false

	for _, vp := range outOfDate {
		// Bundle packages together that have the same revision and share at least one root segment.
		if len(vp.Revision) == 0 {
			continue
		}
		from := vp.Path
		if len(vp.Origin) > 0 {
			from = vp.Origin
		}
		if from != vp.Path {
			fmt.Fprintf(ctx, "fetch %q from %q\n", vp.Path, from)
		} else {
			fmt.Fprintf(ctx, "fetch %q\n", vp.Path)
		}
		if dryrun {
			continue
		}
		pkgDir := filepath.Join(cacheRoot, from)

		// See if repo exists.
		sysVcsCmd, repoRoot, err := vcs.FromDir(pkgDir, cacheRoot)
		var vcsCmd *VCSCmd
		repoRootDir := filepath.Join(cacheRoot, repoRoot)
		if err != nil {
			rr, err := vcs.RepoRootForImportPath(from, false)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to ping remote repo", Path: vp.Path, Err: err})
				continue
			}
			if !ctx.Insecure && !vcsIsSecure(rr.Repo) {
				rem = append(rem, remoteFailure{Msg: "repo remote not secure", Path: vp.Path, Err: nil})
				continue
			}

			vcsCmd = updateVcsCmd(rr.VCS)

			repoRoot = rr.Root
			repoRootDir = filepath.Join(cacheRoot, repoRoot)
			err = os.MkdirAll(repoRootDir, 0700)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to make repo root dir", Path: vp.Path, Err: err})
				continue
			}

			err = vcsCmd.CreateAtRev(repoRootDir, rr.Repo, vp.Revision)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to clone repo", Path: vp.Path, Err: err})
				continue
			}
		} else {
			vcsCmd = updateVcsCmd(sysVcsCmd)
			err = vcsCmd.Download(repoRootDir)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to download repo", Path: vp.Path, Err: err})
				continue
			}

			err = vcsCmd.RevisionSync(repoRootDir, vp.Revision)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to sync repo to " + vp.Revision, Path: vp.Path, Err: err})
				continue
			}
		}
		dest := filepath.Join(ctx.RootDir, ctx.VendorFolder, pathos.SlashToFilepath(vp.Path))
		// Path handling with single sub-packages and differing origins need to be properly handled.
		src := pkgDir

		// Scan go files for files that should be ignored based on tags and filenames.
		ignoreFiles, _, err := ctx.getIngoreFiles(src)
		if err != nil {
			rem = append(rem, remoteFailure{Msg: "failed to get ignore files", Path: vp.Path, Err: err})
			continue
		}

		root, _ := pathos.TrimCommonSuffix(src, vp.Path)

		// Need to ensure we copy files from "b.Root/<import-path>" for the following command.
		err = ctx.CopyPackage(dest, src, root, vp.Path, ignoreFiles, vp.Tree, h, nil)
		if err != nil {
			fmt.Fprintf(ctx, "failed to copy package from %q to %q: %+v", src, dest, err)
		}
		checksum := h.Sum(nil)
		h.Reset()
		vp.ChecksumSHA1 = base64.StdEncoding.EncodeToString(checksum)
		updatedVendorFile = true
	}

	// Only write a vendor file if something changes.
	if updatedVendorFile {
		err = ctx.WriteVendorFile()
		if err != nil {
			return err
		}
	}

	// Return network errors here.
	if len(rem) > 0 {
		return rem
	}

	return nil
}
<commit_msg>Check if revision exists before downloading<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package context

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"hash"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/kardianos/govendor/internal/pathos"
	"github.com/kardianos/govendor/vendorfile"

	"golang.org/x/tools/go/vcs"
)

func skipperTree(name string, dir bool) bool {
	return false
}
func skipperPackage(name string, dir bool) bool {
	return dir
}

func (ctx *Context) VerifyVendor() (outOfDate []*vendorfile.Package, err error) {
	vf := ctx.VendorFile
	root := filepath.Join(ctx.RootDir, ctx.VendorFolder)
	add := func(vp *vendorfile.Package) {
		outOfDate = append(outOfDate, vp)
	}
	for _, vp := range vf.Package {
		if vp.Remove {
			continue
		}
		if len(vp.Path) == 0 {
			continue
		}
		if len(vp.ChecksumSHA1) == 0 {
			add(vp)
			continue
		}
		fp := filepath.Join(root, pathos.SlashToFilepath(vp.Path))
		h := sha1.New()
		sk := skipperPackage
		if vp.Tree {
			sk = skipperTree
		}
		err = getHash(root, fp, h, sk)
		if err != nil {
			return
		}
		checksum := base64.StdEncoding.EncodeToString(h.Sum(nil))
		if vp.ChecksumSHA1 != checksum {
			add(vp)
		}
	}
	return
}

func getHash(root, fp string, h hash.Hash, skipper func(name string, isDir bool) bool) error {
	rel := pathos.FileTrimPrefix(fp, root)
	rel = pathos.SlashToImportPath(rel)
	rel = strings.Trim(rel, "/")

	h.Write([]byte(rel))

	dir, err := os.Open(fp)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return fmt.Errorf("Failed to open dir %q: %v", fp, err)
	}
	filelist, err := dir.Readdir(-1)
	dir.Close()
	if err != nil {
		return fmt.Errorf("Failed to read dir %q: %v", fp, err)
	}
	sort.Sort(fileInfoSort(filelist))
	for _, fi := range filelist {
		if skipper(fi.Name(), fi.IsDir()) {
			continue
		}
		p := filepath.Join(fp, fi.Name())
		if fi.IsDir() {
			err = getHash(root, p, h, skipper)
			if err != nil {
				return err
			}
			continue
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		h.Write([]byte(fi.Name()))
		_, err = io.Copy(h, f)
		f.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

// similarSegments compares two paths and determines if they have
// similar prefix segments. For example github.com/kardianos/rdb and
// github.com/kardianos/govendor have 2 similar segments.
func similarSegments(p1, p2 string) (string, int) {
	seg1 := strings.Split(p1, "/")
	seg2 := strings.Split(p2, "/")

	ct := len(seg1)
	if len(seg2) < ct {
		ct = len(seg2)
	}

	similar := &bytes.Buffer{}
	for i := 0; i < ct; i++ {
		if seg1[i] != seg2[i] {
			return similar.String(), i
		}
		if i != 0 {
			similar.WriteRune('/')
		}
		similar.WriteString(seg1[i])
	}
	return similar.String(), ct
}

type remoteFailure struct {
	Path string
	Msg  string
	Err  error
}

func (fail remoteFailure) Error() string {
	return fmt.Sprintf("Failed for %q (%s): %v", fail.Path, fail.Msg, fail.Err)
}

type remoteFailureList []remoteFailure

func (list remoteFailureList) Error() string {
	if len(list) == 0 {
		return "(no remote failure)"
	}
	buf := &bytes.Buffer{}
	buf.WriteString("Remotes failed for:\n")
	for _, item := range list {
		buf.WriteString("\t")
		buf.WriteString(item.Error())
		buf.WriteString("\n")
	}
	return buf.String()
}

type VCSCmd struct {
	*vcs.Cmd
}

func (vcsCmd *VCSCmd) RevisionSync(dir, revision string) error {
	return vcsCmd.run(dir, vcsCmd.TagSyncCmd, "tag", revision)
}

func (v *VCSCmd) run(dir string, cmd string, keyval ...string) error {
	_, err := v.run1(dir, cmd, keyval, true)
	return err
}

// run1 is the generalized implementation of run and runOutput.
func (vcsCmd *VCSCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) {
	v := vcsCmd.Cmd
	m := make(map[string]string)
	for i := 0; i < len(keyval); i += 2 {
		m[keyval[i]] = keyval[i+1]
	}
	args := strings.Fields(cmdline)
	for i, arg := range args {
		args[i] = expand(m, arg)
	}

	_, err := exec.LookPath(v.Cmd)
	if err != nil {
		fmt.Fprintf(os.Stderr,
			"go: missing %s command. See http://golang.org/s/gogetcmd\n",
			v.Name)
		return nil, err
	}

	cmd := exec.Command(v.Cmd, args...)
	cmd.Dir = dir
	cmd.Env = envForDir(cmd.Dir)
	if vcs.ShowCmd {
		fmt.Printf("cd %s\n", dir)
		fmt.Printf("%s %s\n", v.Cmd, strings.Join(args, " "))
	}
	var buf bytes.Buffer
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	err = cmd.Run()
	out := buf.Bytes()
	if err != nil {
		if verbose {
			fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " "))
			os.Stderr.Write(out)
		}
		return nil, err
	}
	return out, nil
}

// expand rewrites s to replace {k} with match[k] for each key k in match.
func expand(match map[string]string, s string) string {
	for k, v := range match {
		s = strings.Replace(s, "{"+k+"}", v, -1)
	}
	return s
}

// envForDir returns a copy of the environment
// suitable for running in the given directory.
// The environment is the current process's environment
// but with an updated $PWD, so that an os.Getwd in the
// child will be faster.
func envForDir(dir string) []string {
	env := os.Environ()
	// Internally we only use rooted paths, so dir is rooted.
	// Even if dir is not rooted, no harm done.
	return mergeEnvLists([]string{"PWD=" + dir}, env)
}

// mergeEnvLists merges the two environment lists such that
// variables with the same name in "in" replace those in "out".
func mergeEnvLists(in, out []string) []string {
NextVar:
	for _, inkv := range in {
		k := strings.SplitAfterN(inkv, "=", 2)[0]
		for i, outkv := range out {
			if strings.HasPrefix(outkv, k) {
				out[i] = inkv
				continue NextVar
			}
		}
		out = append(out, inkv)
	}
	return out
}

func updateVcsCmd(cmd *vcs.Cmd) *VCSCmd {
	switch cmd.Name {
	case "Git":
		cmd.TagSyncCmd = "reset --hard {tag}"
		cmd.TagSyncDefault = "reset --hard origin/master"
		cmd.DownloadCmd = "fetch"
	case "Mercurial":
	case "Bazaar":
	case "Subversion":
	}
	return &VCSCmd{Cmd: cmd}
}

var isSecureScheme = map[string]bool{
	"https":   true,
	"git+ssh": true,
	"bzr+ssh": true,
	"svn+ssh": true,
	"ssh":     true,
}

func vcsIsSecure(repo string) bool {
	u, err := url.Parse(repo)
	if err != nil {
		// If repo is not a URL, it's not secure.
		return false
	}
	return isSecureScheme[u.Scheme]
}

// Sync checks for outdated packages in the vendor folder and fetches the
// correct revision from the remote.
func (ctx *Context) Sync(dryrun bool) (err error) {
	// vcs.ShowCmd = true
	outOfDate, err := ctx.VerifyVendor()
	if err != nil {
		return fmt.Errorf("Failed to verify checksums: %v", err)
	}
	// GOPATH includes the src dir, move up a level.
	cacheRoot := filepath.Join(ctx.RootGopath, "..", ".cache", "govendor")
	err = os.MkdirAll(cacheRoot, 0700)
	if err != nil {
		return err
	}

	// collect errors and proceed where you can.
	rem := remoteFailureList{}

	h := sha1.New()
	updatedVendorFile := false

	for _, vp := range outOfDate {
		// Bundle packages together that have the same revision and share at least one root segment.
		if len(vp.Revision) == 0 {
			continue
		}
		from := vp.Path
		if len(vp.Origin) > 0 {
			from = vp.Origin
		}
		if from != vp.Path {
			fmt.Fprintf(ctx, "fetch %q from %q\n", vp.Path, from)
		} else {
			fmt.Fprintf(ctx, "fetch %q\n", vp.Path)
		}
		if dryrun {
			continue
		}
		pkgDir := filepath.Join(cacheRoot, from)

		// See if repo exists.
		sysVcsCmd, repoRoot, err := vcs.FromDir(pkgDir, cacheRoot)
		var vcsCmd *VCSCmd
		repoRootDir := filepath.Join(cacheRoot, repoRoot)
		if err != nil {
			rr, err := vcs.RepoRootForImportPath(from, false)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to ping remote repo", Path: vp.Path, Err: err})
				continue
			}
			if !ctx.Insecure && !vcsIsSecure(rr.Repo) {
				rem = append(rem, remoteFailure{Msg: "repo remote not secure", Path: vp.Path, Err: nil})
				continue
			}

			vcsCmd = updateVcsCmd(rr.VCS)

			repoRoot = rr.Root
			repoRootDir = filepath.Join(cacheRoot, repoRoot)
			err = os.MkdirAll(repoRootDir, 0700)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to make repo root dir", Path: vp.Path, Err: err})
				continue
			}

			err = vcsCmd.CreateAtRev(repoRootDir, rr.Repo, vp.Revision)
			if err != nil {
				rem = append(rem, remoteFailure{Msg: "failed to clone repo", Path: vp.Path, Err: err})
				continue
			}
		} else {
			// Use cache.
			vcsCmd = updateVcsCmd(sysVcsCmd)

			err = vcsCmd.RevisionSync(repoRootDir, vp.Revision)
			// If revision was not found in the cache, download and try again.
			if err != nil {
				err = vcsCmd.Download(repoRootDir)
				if err != nil {
					rem = append(rem, remoteFailure{Msg: "failed to download repo", Path: vp.Path, Err: err})
					continue
				}
				err = vcsCmd.RevisionSync(repoRootDir, vp.Revision)
				if err != nil {
					rem = append(rem, remoteFailure{Msg: "failed to sync repo to " + vp.Revision, Path: vp.Path, Err: err})
					continue
				}
			}
		}
		dest := filepath.Join(ctx.RootDir, ctx.VendorFolder, pathos.SlashToFilepath(vp.Path))
		// Path handling with single sub-packages and differing origins need to be properly handled.
		src := pkgDir

		// Scan go files for files that should be ignored based on tags and filenames.
		ignoreFiles, _, err := ctx.getIngoreFiles(src)
		if err != nil {
			rem = append(rem, remoteFailure{Msg: "failed to get ignore files", Path: vp.Path, Err: err})
			continue
		}

		root, _ := pathos.TrimCommonSuffix(src, vp.Path)

		// Need to ensure we copy files from "b.Root/<import-path>" for the following command.
		err = ctx.CopyPackage(dest, src, root, vp.Path, ignoreFiles, vp.Tree, h, nil)
		if err != nil {
			fmt.Fprintf(ctx, "failed to copy package from %q to %q: %+v", src, dest, err)
		}
		checksum := h.Sum(nil)
		h.Reset()
		vp.ChecksumSHA1 = base64.StdEncoding.EncodeToString(checksum)
		updatedVendorFile = true
	}

	// Only write a vendor file if something changes.
	if updatedVendorFile {
		err = ctx.WriteVendorFile()
		if err != nil {
			return err
		}
	}

	// Return network errors here.
	if len(rem) > 0 {
		return rem
	}

	return nil
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"net/url"
	"net/http"
	"io/ioutil"
	"time"
	"bytes"
	"strings"
	"crypto/tls"
	"labix.org/v2/mgo/bson"
	"encoding/json"
	"github.com/mreiferson/go-httpclient"
)

const (
	DefaultSocketTimeout = 40000
	HttpPostMethod = "POST"
	HttpGetMethod = "GET"

	ContentTypeHeader = "Content-Type"

	NoStatusCode = -1 // This is the value used if an error is generated before the status code is available

	ContentTypeTextPlain = "text/plain; charset=utf-8"
	ContentTypePostForm = "application/x-www-form-urlencoded"
	ContentTypeJson = "application/json"
)

// The http request client allows you to configure global values for timeout/connect etc. It also
// handles closing the transport and provides convenience methods for posting some types of data.
type HttpRequestClient struct {
	DisableKeepAlives bool
	DisableCompression bool
	SkipSslVerify bool
	MaxIdleConnsPerHost int
	ConnectTimeout time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout time.Duration
	ReadWriteTimeout time.Duration
}

func NewDefaultHttpRequestClient() *HttpRequestClient {
	return NewHttpRequestClient(true, false, false, 0, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout, DefaultSocketTimeout)
}

func NewHttpRequestClient(	disableKeepAlives,
							disableCompression,
							skipSslVerify bool,
							maxIdleConnsPerHost,
							connectTimeoutInMs,
							responseHeaderTimeoutInMs,
							requestTimeoutInMs,
							readWriteTimeoutInMs int) *HttpRequestClient {

	return &HttpRequestClient{ 	DisableKeepAlives: disableKeepAlives,
								DisableCompression: disableCompression,
								SkipSslVerify: skipSslVerify,
								MaxIdleConnsPerHost: maxIdleConnsPerHost,
								ConnectTimeout: time.Duration(connectTimeoutInMs) * time.Millisecond,
								ResponseHeaderTimeout: time.Duration(responseHeaderTimeoutInMs) * time.Millisecond,
								RequestTimeout: time.Duration(requestTimeoutInMs) * time.Millisecond,
								ReadWriteTimeout: time.Duration(readWriteTimeoutInMs) * time.Millisecond,
	}
}

// Post the values to the url.
func (self *HttpRequestClient) Post(url string, values url.Values, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url,  strings.NewReader(values.Encode()))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the raw string to the url.
func (self *HttpRequestClient) PostStr(url string, value string, headers map[string]string) (int, []byte, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader([]byte(value)))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the json struct to the url.
func (self *HttpRequestClient) PostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) {

	rawJson, err := json.Marshal(value)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawJson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypeJson)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Post the bson doc to the url.
func (self *HttpRequestClient) PostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) {

	rawBson, err := bson.Marshal(bsonDoc)
	if err != nil { return NoStatusCode, nil, err }

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, err := http.NewRequest(HttpPostMethod, url, bytes.NewReader(rawBson))
	if err != nil { return NoStatusCode, nil, err }

	request.Header.Set(ContentTypeHeader, ContentTypePostForm)
	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	// We do not return the response so don't report if there is an error.
	if data, err := ioutil.ReadAll(response.Body); err != nil { return statusCode, nil, err
	} else { return statusCode, data, nil }
}

// Issue a GET to retrieve a bson doc.
func (self *HttpRequestClient) GetBson(url string, headers map[string]string) (int, bson.M, error) {

	client, transport := self.getClientAndTransport()
	defer transport.Close()

	request, requestErr := http.NewRequest(HttpGetMethod, url, nil)
	if requestErr != nil { return NoStatusCode, nil, requestErr }

	for key, value := range headers { request.Header.Set(key, value) }

	response, err := client.Do(request)
	if err != nil { return NoStatusCode, nil, err }

	statusCode := response.StatusCode

	defer response.Body.Close()

	rawBson, err := ioutil.ReadAll(response.Body)
	if err != nil { return statusCode, nil, err }

	var bsonDoc bson.M
	if err := bson.Unmarshal(rawBson, &bsonDoc); err != nil { return statusCode, nil, err }

	return statusCode, bsonDoc, nil
}

// Use the clone method if you need to override some/all of the configured values.
func (self *HttpRequestClient) Clone() *HttpRequestClient {
	val := *self
	return &val
}

func (self *HttpRequestClient) getClientAndTransport() (*http.Client, *httpclient.Transport) {
	transport := self.getTransport()
	return &http.Client{ Transport: transport }, transport
}

func (self *HttpRequestClient) getTransport() *httpclient.Transport {
	return &httpclient.Transport {

		DisableKeepAlives: self.DisableKeepAlives,
		DisableCompression: self.DisableCompression,

		MaxIdleConnsPerHost: self.MaxIdleConnsPerHost,

		ConnectTimeout: self.ConnectTimeout,
		ResponseHeaderTimeout: self.ResponseHeaderTimeout,
		RequestTimeout: self.RequestTimeout,
		ReadWriteTimeout: self.ReadWriteTimeout,

		TLSClientConfig: &tls.Config{ InsecureSkipVerify: self.SkipSslVerify },
	}
}

func HttpPost(url string, values url.Values, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().Post(url, values, headers) }

func HttpPostStr(url string, value string, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostStr(url, value, headers) }

func HttpPostJson(url string, value interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostJson(url, value, headers) }

func HttpPostBson(url string, bsonDoc interface{}, headers map[string]string) (int, []byte, error) { return NewDefaultHttpRequestClient().PostBson(url, bsonDoc, headers) }

func HttpGetBson(url string, headers map[string]string) (int, bson.M, error) { return NewDefaultHttpRequestClient().GetBson(url, headers) }

<commit_msg>changed file name.<commit_after><|endoftext|>


<commit_before>// Copyright 2013 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File convert_test.go tests the conversion
// to and from go data structures of a variety of types.

package zoom

import (
	"github.com/garyburd/redigo/redis"
	"reflect"
	"testing"
)

func TestPrimativeTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newPrimativeTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(primativeTypesModel{}), construct, t)
}

func TestPointerToPrimativeTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newPointersToPrimativeTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(pointersToPrimativeTypesModel{}), construct, t)
}

func TestInconvertibleTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newInconvertibleTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(inconvertibleTypesModel{}), construct, t)
}

func TestModelWithList(t *testing.T) {
	// use the generic tester to make sure what we get out is the same
	// as what we put in
	construct := func() (interface{}, error) {
		return &modelWithList{
			List: []string{"one", "two", "three"},
		}, nil
	}
	testConvertType(reflect.TypeOf(modelWithList{}), construct, t)

	// test to make sure the field is saved as a redis list type
	testingSetUp()
	defer testingTearDown()

	m := &modelWithList{
		List: []string{"one", "two", "three"},
	}
	if err := Save(m); err != nil {
		t.Error(err)
	}

	conn := GetConn()
	defer conn.Close()

	listKey := "modelWithList:" + m.Id + ":List"
	result, err := redis.Strings(conn.Do("LRANGE", listKey, "0", "-1"))
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(m.List, result) {
		t.Errorf("List was not saved correctly.\nExpected: %+v\nGot: %+v\n", m.List, result)
	}
}

// func TestModelWithSet(t *testing.T) {
// 	// test to make sure the field is saved as a redis set type
// 	testingSetUp()
// 	defer testingTearDown()

// 	m := &modelWithSet{
// 		Set: []string{"one", "two", "three"},
// 	}
// 	if err := Save(m); err != nil {
// 		t.Error(err)
// 	}
// 	fmt.Println(m.Set)

// 	conn := GetConn()
// 	defer conn.Close()

// 	setKey := "modelWithSet:" + m.Id + ":Set"
// 	result, err := redis.Strings(conn.Do("SMEMBERS", setKey))
// 	if err != nil {
// 		t.Error(err)
// 	}
// 	fmt.Println(m.Set)
// 	if equal, msg := compareAsStringSet(m.Set, result); !equal {
// 		t.Errorf("Set was not saved correctly.\nExpected: %+v\nGot: %+v\n%s\n", m.Set, result, msg)
// 	}
// 	fmt.Println(m.Set)

// 	// test to make sure what we put in is what we get out
// 	mCopy := new(modelWithSet)
// 	if err := ScanById(m.Id, mCopy); err != nil {
// 		t.Error(err)
// 	}
// 	if equal, msg := compareAsStringSet(m.Set, mCopy.Set); !equal {
// 		t.Errorf("Set was not retrieved correctly.\nExpected: %+v\nGot: %+v\n%s\n", m.Set, mCopy.Set, msg)
// 	}
// }

// TODO:
//	- ModelWithHash

func TestEmbeddedStruct(t *testing.T) {
	construct := func() (interface{}, error) {
		return &embeddedStructModel{
			embed: embed{
				Int: 42,
			},
		}, nil
	}
	testConvertType(reflect.TypeOf(embeddedStructModel{}), construct, t)
}

func TestEmbeddedPointerToStruct(t *testing.T) {
	construct := func() (interface{}, error) {
		return &embeddedPointerToStructModel{
			embed: &embed{
				Int: 42,
			},
		}, nil
	}
	testConvertType(reflect.TypeOf(embeddedPointerToStructModel{}), construct, t)
}

// a general test that uses reflection
func testConvertType(typ reflect.Type, construct func() (in interface{}, err error), t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	// construct a model using the construct function
	modelInterface, err := construct()
	if err != nil {
		t.Error(err)
	}
	model, ok := modelInterface.(Model)
	if !ok {
		t.Errorf("couldn't convert type %T to Model", modelInterface)
	}
	if err := Save(model); err != nil {
		t.Error(err)
	}

	// create a copy of the same type and use ScanById
	modelCopyInterface := reflect.New(typ).Interface()
	modelCopy, ok := modelCopyInterface.(Model)
	id := model.getId()
	if !ok {
		t.Errorf("couldn't convert type %T to Model", modelCopyInterface)
	}
	if err := ScanById(id, modelCopy); err != nil {
		t.Error(err)
	}

	// make sure the copy equals the original
	equal, err := looseEquals(model, modelCopy)
	if err != nil {
		t.Error(err)
	}
	if !equal {
		t.Errorf("model was not saved/retrieved correctly.\nExpected: %+v\nGot %+v\n", model, modelCopy)
	}
}
<commit_msg>uncomment TestModelWithSet. Since the bug in util.compareAsStringSet was fixed, it passes now.<commit_after>// Copyright 2013 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// File convert_test.go tests the conversion
// to and from go data structures of a variety of types.

package zoom

import (
	"github.com/garyburd/redigo/redis"
	"reflect"
	"testing"
)

func TestPrimativeTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newPrimativeTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(primativeTypesModel{}), construct, t)
}

func TestPointerToPrimativeTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newPointersToPrimativeTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(pointersToPrimativeTypesModel{}), construct, t)
}

func TestInconvertibleTypes(t *testing.T) {
	construct := func() (interface{}, error) {
		ms, err := newInconvertibleTypesModels(1)
		return ms[0], err
	}
	testConvertType(reflect.TypeOf(inconvertibleTypesModel{}), construct, t)
}

func TestModelWithList(t *testing.T) {
	// use the generic tester to make sure what we get out is the same
	// as what we put in
	construct := func() (interface{}, error) {
		return &modelWithList{
			List: []string{"one", "two", "three"},
		}, nil
	}
	testConvertType(reflect.TypeOf(modelWithList{}), construct, t)

	// test to make sure the field is saved as a redis list type
	testingSetUp()
	defer testingTearDown()

	m := &modelWithList{
		List: []string{"one", "two", "three"},
	}
	if err := Save(m); err != nil {
		t.Error(err)
	}

	conn := GetConn()
	defer conn.Close()

	listKey := "modelWithList:" + m.Id + ":List"
	result, err := redis.Strings(conn.Do("LRANGE", listKey, "0", "-1"))
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(m.List, result) {
		t.Errorf("List was not saved correctly.\nExpected: %+v\nGot: %+v\n", m.List, result)
	}
}

func TestModelWithSet(t *testing.T) {
	// test to make sure the field is saved as a redis set type
	testingSetUp()
	defer testingTearDown()

	m := &modelWithSet{
		Set: []string{"one", "two", "three"},
	}
	if err := Save(m); err != nil {
		t.Error(err)
	}

	conn := GetConn()
	defer conn.Close()

	setKey := "modelWithSet:" + m.Id + ":Set"
	result, err := redis.Strings(conn.Do("SMEMBERS", setKey))
	if err != nil {
		t.Error(err)
	}
	if equal, msg := compareAsStringSet(m.Set, result); !equal {
		t.Errorf("Set was not saved correctly.\nExpected: %+v\nGot: %+v\n%s\n", m.Set, result, msg)
	}

	// test to make sure what we put in is what we get out
	mCopy := new(modelWithSet)
	if err := ScanById(m.Id, mCopy); err != nil {
		t.Error(err)
	}
	if equal, msg := compareAsStringSet(m.Set, mCopy.Set); !equal {
		t.Errorf("Set was not retrieved correctly.\nExpected: %+v\nGot: %+v\n%s\n", m.Set, mCopy.Set, msg)
	}
}

// TODO:
//	- ModelWithHash

func TestEmbeddedStruct(t *testing.T) {
	construct := func() (interface{}, error) {
		return &embeddedStructModel{
			embed: embed{
				Int: 42,
			},
		}, nil
	}
	testConvertType(reflect.TypeOf(embeddedStructModel{}), construct, t)
}

func TestEmbeddedPointerToStruct(t *testing.T) {
	construct := func() (interface{}, error) {
		return &embeddedPointerToStructModel{
			embed: &embed{
				Int: 42,
			},
		}, nil
	}
	testConvertType(reflect.TypeOf(embeddedPointerToStructModel{}), construct, t)
}

// a general test that uses reflection
func testConvertType(typ reflect.Type, construct func() (in interface{}, err error), t *testing.T) {
	testingSetUp()
	defer testingTearDown()

	// construct a model using the construct function
	modelInterface, err := construct()
	if err != nil {
		t.Error(err)
	}
	model, ok := modelInterface.(Model)
	if !ok {
		t.Errorf("couldn't convert type %T to Model", modelInterface)
	}
	if err := Save(model); err != nil {
		t.Error(err)
	}

	// create a copy of the same type and use ScanById
	modelCopyInterface := reflect.New(typ).Interface()
	modelCopy, ok := modelCopyInterface.(Model)
	id := model.getId()
	if !ok {
		t.Errorf("couldn't convert type %T to Model", modelCopyInterface)
	}
	if err := ScanById(id, modelCopy); err != nil {
		t.Error(err)
	}

	// make sure the copy equals the original
	equal, err := looseEquals(model, modelCopy)
	if err != nil {
		t.Error(err)
	}
	if !equal {
		t.Errorf("model was not saved/retrieved correctly.\nExpected: %+v\nGot %+v\n", model, modelCopy)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/lambda"
)

func addPermissionToLambdaFromCloudWatchEvents(lc *lambda.Lambda, rules []Rule) error {
	for _, rule := range rules {
		for _, target := range rule.Targets {
			if !IsLambdaFunction(target.Arn) {
				continue
			}

			if result, err := isAlreadyAddPermission(lc, rule, target); err != nil {
				return err
			} else if result {
				// do nothing (already granted permission)
				continue
			} else {
				lc.AddPermission(&lambda.AddPermissionInput{
					Action:       aws.String("lambda:InvokeFunction"),
					FunctionName: aws.String(LambdaFunctionNameFromArn(target.Arn)),
					Principal:    aws.String("events.amazonaws.com"),
					SourceArn:    rule.ActualRule.Arn,
					StatementId:  aws.String(target.Id),
				})
			}
		}
	}
	return nil
}

func isAlreadyAddPermission(lc *lambda.Lambda, rule Rule, target Target) (bool, error) {
	var policy LambdaPolicy

	policyOutput, err := lc.GetPolicy(&lambda.GetPolicyInput{
		FunctionName: &target.Arn,
	})

	if err != nil {
		return false, err
	}

	errJ := json.Unmarshal([]byte(*policyOutput.Policy), &policy)
	if errJ != nil {
		return false, errJ
	}

	for _, statement := range *policy.Statement {
		if (statement.Resource == target.Arn &&
			strings.HasSuffix(statement.Condition.ArnLike.AwsSourceArn, rule.Name) &&
			statement.Effect == "Allow" &&
			statement.Principal.Service == "events.amazonaws.com" &&
			statement.Action == "lambda:InvokeFunction") ||
			statement.StatementId == target.Id {
			return true, nil
		}
	}

	return false, nil
}

func IsLambdaFunction(arn string) bool {
	return strings.HasPrefix(arn, "arn:aws:lambda")
}

func LambdaFunctionNameFromArn(arn string) string {
	return strings.Split(arn, ":")[6]
}
<commit_msg>Return error information<commit_after>package main

import (
	"encoding/json"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/lambda"
)

func addPermissionToLambdaFromCloudWatchEvents(lc *lambda.Lambda, rules []Rule) error {
	for _, rule := range rules {
		for _, target := range rule.Targets {
			if !IsLambdaFunction(target.Arn) {
				continue
			}

			if result, err := isAlreadyAddPermission(lc, rule, target); err != nil {
				return err
			} else if result {
				// do nothing (already granted permission)
				continue
			} else {
				_, errL := lc.AddPermission(&lambda.AddPermissionInput{
					Action:       aws.String("lambda:InvokeFunction"),
					FunctionName: aws.String(LambdaFunctionNameFromArn(target.Arn)),
					Principal:    aws.String("events.amazonaws.com"),
					SourceArn:    rule.ActualRule.Arn,
					StatementId:  aws.String(target.Id),
				})

				if errL != nil {
					return errL
				}
			}
		}
	}
	return nil
}

func isAlreadyAddPermission(lc *lambda.Lambda, rule Rule, target Target) (bool, error) {
	var policy LambdaPolicy

	policyOutput, err := lc.GetPolicy(&lambda.GetPolicyInput{
		FunctionName: &target.Arn,
	})

	if err != nil {
		return false, err
	}

	errJ := json.Unmarshal([]byte(*policyOutput.Policy), &policy)
	if errJ != nil {
		return false, errJ
	}

	for _, statement := range *policy.Statement {
		if (statement.Resource == target.Arn &&
			strings.HasSuffix(statement.Condition.ArnLike.AwsSourceArn, rule.Name) &&
			statement.Effect == "Allow" &&
			statement.Principal.Service == "events.amazonaws.com" &&
			statement.Action == "lambda:InvokeFunction") ||
			statement.StatementId == target.Id {
			return true, nil
		}
	}

	return false, nil
}

func IsLambdaFunction(arn string) bool {
	return strings.HasPrefix(arn, "arn:aws:lambda")
}

func LambdaFunctionNameFromArn(arn string) string {
	return strings.Split(arn, ":")[6]
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"github.com/GrappigPanda/notorious/bencode"
	"gopkg.in/redis.v3"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	STARTED = iota
	COMPLETED
	STOPPED
)

type TorrentEvent struct {
	started   int
	completed int
	stopped   int
}

type TorrentRequestData struct {
	info_hash  string //20 byte sha1 hash
	peer_id    string //max len 20
	ip         string //optional
	port       string // port number the peer is listening on
	uploaded   int    // base10 ascii amount uploaded so far
	downloaded int    // base10 ascii amount downloaded so far
	left       int    // # of bytes left to download (base 10 ascii)
	event      int    // TorrentEvent
	numwant    int    // Number of peers requested by client.
	compact    bool   // Bep23 peer list compression decision: True -> compress bep23
}

var ANNOUNCE_URL = "/announce"

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func parseTorrentGetRequestURI(request string, ip string) map[string]interface{} {
	torrentdata := make(map[string]interface{})
	querydata := decodeQueryURL(request)

	lookupvalues := []string{
		"info_hash",
		"peer_id",
		// ip intentially left out
		"port",
		"uploaded",
		"downloaded",
		"left",
		"event",
		"numwant",
		"compact",
	}

	lookupints := []string{
		"uploaded",
		"downloaded",
		"left",
		"event",
		"numwant",
	}

	for i := range lookupvalues {
		index := lookupvalues[i]

		// TODO(ian): Delegate responsibility to another function.
		x := querydata[index]
		if len(x) <= 0 {
			continue
		}

		if index == "info_hash" {
			torrentdata[index] = ParseInfoHash(x[0])
		} else if index == "compact" {
			if querydata[index][0] == "1" {
				torrentdata[index] = true
			} else {
				torrentdata[index] = false
			}
		} else {
			torrentdata[index] = querydata[index][0]
		}

		// TODO(ian): Delegate responsibility here, as well.
		for j := range lookupints {
			if lookupints[j] == index {
				val, _ := strconv.Atoi(index)
				torrentdata[index] = val
			}
		}
	}

	// TODO(ian): Add a generic size lookup function and assert > 0
	x := strings.Split(ip, ":")
	if len(x) > 0 {
		torrentdata["ip"] = x[0]
	}

	fmt.Println(torrentdata)
	return torrentdata
}

func ParseInfoHash(s string) string {
	return fmt.Sprintf("%x", s)
}

func fillEmptyMapValues(torrentMap map[string]interface{}) *TorrentRequestData {
	_, ok := torrentMap["uploaded"]
	if !ok {
		torrentMap["uploaded"] = 0
	}

	_, ok = torrentMap["downloaded"]
	if !ok {
		torrentMap["downloaded"] = 0
	}

	_, ok = torrentMap["left"]
	if !ok {
		torrentMap["left"] = 0
	}

	_, ok = torrentMap["event"]
	if !ok {
		torrentMap["event"] = STOPPED
	}

	_, ok = torrentMap["compact"]
	if !ok {
		torrentMap["compact"] = true
	}

	_, ok = torrentMap["numwant"]
	if !ok {
		torrentMap["numwant"] = 30
	}

	x := TorrentRequestData{
		torrentMap["info_hash"].(string),
		torrentMap["peer_id"].(string),
		torrentMap["ip"].(string),
		torrentMap["port"].(string),
		torrentMap["uploaded"].(int),
		torrentMap["downloaded"].(int),
		torrentMap["left"].(int),
		torrentMap["event"].(int),
		torrentMap["numwant"].(int),
		torrentMap["compact"].(bool),
	}

	return &x
}

func worker(client *redis.Client, torrdata *TorrentRequestData) []string {
	if RedisGetBoolKeyVal(client, torrdata.info_hash, torrdata) {
		x := RedisGetKeyVal(client, torrdata.info_hash, torrdata)

		RedisSetKeyVal(client,
			concatenateKeyMember(torrdata.info_hash, "ip"),
			createIpPortPair(torrdata))

		return x

	} else {
		CreateNewTorrentKey(client, torrdata.info_hash, torrdata)
		return worker(client, torrdata)
	}
}

func compactifyIpPort(ip string) string {
	sz := strings.Split(ip, ":")
	newip := net.IPMask(net.ParseIP(sz[0])).String()
	x, err := strconv.Atoi(sz[1])
	if err != nil {
		panic("Invalid IP!")
	}
	port := fmt.Sprintf("%x", x)

	// TODO(ian): Sometime in the future, support ivp6
	newip = newip[len(newip)-8 : len(newip)]
	port = port[len(port)-4 : len(port)]

	ret := ""

	for i := 0; i < 8; i += 2 {
		ret += fmt.Sprintf("\\x%s", newip[i:i+2])
	}

	for i := 0; i < 4; i += 2 {
		ret += fmt.Sprintf("\\x%s", port[i:i+2])
	}

	return ret
}

func formatResponseData(ips []string, torrentdata *TorrentRequestData) string {
	for i := range ips {
		ips[i] = compactifyIpPort(ips[i])
	}

	if torrentdata.compact {
		// TODO(ian): Support bep-23
		return EncodeResponse(ips, torrentdata)
	} else {
		return EncodeResponse(ips, torrentdata)
	}
}

func decodeQueryURL(s string) url.Values {
	u, err := url.Parse(s)
	if err != nil {
		panic(err)
	}

	m, _ := url.ParseQuery(u.RawQuery)
	return m
}

func encodeKV(key string, value string) string {
	return fmt.Sprintf("%s%s", bencode.EncodeByteString(key), bencode.EncodeByteString(value))
}

func EncodeResponse(ipport []string, torrentdata *TorrentRequestData) string {
	ret := ""
	ret += encodeKV("interval", "30")
	ret += encodeKV("complete", "1")
	ipstr := ""

	if torrentdata.compact {
		count := 0

		for i := range ipport {
			ipstr += fmt.Sprintf("%s", ipport[i])
			count += 1
		}

		ret += encodeKV("incomplete", fmt.Sprintf("%d", count))
		ret += bencode.EncodeDictionary("peers", ipstr)
	} else {

		for i := range ipport {
			data := strings.Split(ipport[i], ":")

			ipstr += encodeKV("peer_id", "1")
			ipstr += encodeKV("ip", data[0])
			ipstr += encodeKV("port", data[1])
		}

		ret += bencode.EncodeDictionary("peers", ipstr)
	}

	return bencode.EncodeDictionary("", ret)
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	client := OpenClient()

	torrentdata := parseTorrentGetRequestURI(req.RequestURI, req.RemoteAddr)
	if torrentdata != nil {
		data := fillEmptyMapValues(torrentdata)

		worker(client, data)
		x := RedisGetKeyVal(client, data.info_hash, data)
		fmt.Println(x)

		response := formatResponseData(x, data)
		fmt.Println(response)

		w.Write([]byte(response))
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}

func OpenClient() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	return client
}

func CreateNewTorrentKey(client *redis.Client, key string, value *TorrentRequestData) {
	// CreateNewTorrentKey creates a new key. By default, it adds a member
	// ":ip". I don't think this ought to ever be generalized, as I just want
	// Redis to function in one specific way in notorious.

	// TODO(ian): You might want to set this explicitly in parameters
	// value := *TorrentRequestData
	client.SAdd(key, "ip")
}

func createIpPortPair(value *TorrentRequestData) string {
	// createIpPortPair creates a string formatted ("%s:%s", value.ip,
	// value.port) looking like so: "127.0.0.1:6886" and returns this value.
	return fmt.Sprintf("%s:%s", value.ip, value.port)
}

func RedisSetKeyVal(client *redis.Client, keymember string, value string) {
	// RedisSetKeyVal sets a key:member's value to value. Returns nothing as of
	// yet.
	client.SAdd(keymember, value)
}

func RedisGetKeyVal(client *redis.Client, key string, value *TorrentRequestData) []string {
	// RedisGetKeyVal retrieves a value from the Redis store by looking up the
	// provided key. If the key does not yet exist, we create the key in the KV
	// storage or if the value is empty, we add the current requester to the
	// list.
	keymember := concatenateKeyMember(key, "ip")

	val, err := client.SMembers(keymember).Result()
	if err != nil {
		// Fail because the key doesn't exist in the KV storage.
		CreateNewTorrentKey(client, keymember, value)
	}

	// If no keys yet exist in the KV storage.
	if len(val) == 0 {
		RedisSetKeyVal(client, keymember, createIpPortPair(value))
	}

	return val
}

func RedisGetBoolKeyVal(client *redis.Client, key string, value interface{}) bool {
	_, err := client.Get(key).Result()

	return err != nil
}

func concatenateKeyMember(key string, member string) string {
	var buffer bytes.Buffer

	buffer.WriteString(key)
	buffer.WriteString(":")
	buffer.WriteString(member)

	return buffer.String()
}
<commit_msg>Update server.go<commit_after>package server

import (
	"bytes"
	"fmt"
	"github.com/GrappigPanda/notorious/bencode"
	"gopkg.in/redis.v3"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	STARTED = iota
	COMPLETED
	STOPPED
)

type TorrentEvent struct {
	started   int
	completed int
	stopped   int
}

type TorrentRequestData struct {
	info_hash  string //20 byte sha1 hash
	peer_id    string //max len 20
	ip         string //optional
	port       string // port number the peer is listening on
	uploaded   int    // base10 ascii amount uploaded so far
	downloaded int    // base10 ascii amount downloaded so far
	left       int    // # of bytes left to download (base 10 ascii)
	event      int    // TorrentEvent
	numwant    int    // Number of peers requested by client.
	compact    bool   // Bep23 peer list compression decision: True -> compress bep23
}

var ANNOUNCE_URL = "/announce"

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func parseTorrentGetRequestURI(request string, ip string) map[string]interface{} {
	torrentdata := make(map[string]interface{})
	querydata := decodeQueryURL(request)

	lookupvalues := []string{
		"info_hash",
		"peer_id",
		// ip intentially left out
		"port",
		"uploaded",
		"downloaded",
		"left",
		"event",
		"numwant",
		"compact",
	}

	lookupints := []string{
		"uploaded",
		"downloaded",
		"left",
		"event",
		"numwant",
	}

	for i := range lookupvalues {
		index := lookupvalues[i]

		// TODO(ian): Delegate responsibility to another function.
		x := querydata[index]
		if len(x) <= 0 {
			continue
		}

		if index == "info_hash" {
			torrentdata[index] = ParseInfoHash(x[0])
		} else if index == "compact" {
			if querydata[index][0] == "1" {
				torrentdata[index] = true
			} else {
				torrentdata[index] = false
			}
		} else {
			torrentdata[index] = querydata[index][0]
		}

		// TODO(ian): Delegate responsibility here, as well.
		for j := range lookupints {
			if lookupints[j] == index {
				val, _ := strconv.Atoi(index)
				torrentdata[index] = val
			}
		}
	}

	// TODO(ian): Add a generic size lookup function and assert > 0
	x := strings.Split(ip, ":")
	if len(x) > 0 {
		torrentdata["ip"] = x[0]
	}

	fmt.Println(torrentdata)
	return torrentdata
}

func ParseInfoHash(s string) string {
	return fmt.Sprintf("%x", s)
}

func fillEmptyMapValues(torrentMap map[string]interface{}) *TorrentRequestData {
	_, ok := torrentMap["uploaded"]
	if !ok {
		torrentMap["uploaded"] = 0
	}

	_, ok = torrentMap["downloaded"]
	if !ok {
		torrentMap["downloaded"] = 0
	}

	_, ok = torrentMap["left"]
	if !ok {
		torrentMap["left"] = 0
	}

	_, ok = torrentMap["event"]
	if !ok {
		torrentMap["event"] = STOPPED
	}

	_, ok = torrentMap["compact"]
	if !ok {
		torrentMap["compact"] = true
	}

	_, ok = torrentMap["numwant"]
	if !ok {
		torrentMap["numwant"] = 30
	}

	x := TorrentRequestData{
		torrentMap["info_hash"].(string),
		torrentMap["peer_id"].(string),
		torrentMap["ip"].(string),
		torrentMap["port"].(string),
		torrentMap["uploaded"].(int),
		torrentMap["downloaded"].(int),
		torrentMap["left"].(int),
		torrentMap["event"].(int),
		torrentMap["numwant"].(int),
		torrentMap["compact"].(bool),
	}

	return &x
}

func worker(client *redis.Client, torrdata *TorrentRequestData) []string {
	if RedisGetBoolKeyVal(client, torrdata.info_hash, torrdata) {
		x := RedisGetKeyVal(client, torrdata.info_hash, torrdata)

		RedisSetKeyVal(client,
			concatenateKeyMember(torrdata.info_hash, "ip"),
			createIpPortPair(torrdata))

		return x

	} else {
		CreateNewTorrentKey(client, torrdata.info_hash, torrdata)
		return worker(client, torrdata)
	}
}
func compactIPPort(ip string, port string) []byte {
    res := bytes.NewBuffer(make([]byte, 0))

    intPort, err := strconv.Atoi(port)
    if err != nil {
        panic("failure1")
    }

    if err := binary.Write(res, binary.BigEndian, binary.BigEndian.Uint32(net.ParseIP(ip).To4())); err != nil {
        panic("failure0")
    }

    err = binary.Write(res, binary.BigEndian, uint16(intPort));
        if err != nil {
        panic("failure2")
    }

    return res.Bytes()
}

func CompactAllPeers(ipport []string) []byte {
    ret := bytes.NewBuffer(make([]byte, 0))
    for i := range ipport {
        sz := strings.Split(ipport[i], ":")
        ip := sz[0]
        port := sz[1]

        ret += compactIPPort(ip, port)
    }

    return ret.Bytes()
}

func formatResponseData(ips []string, torrentdata *TorrentRequestData) string {
	for i := range ips {
		ips[i] = compactifyIpPort(ips[i])
	}

	if torrentdata.compact {
		// TODO(ian): Support bep-23
		return EncodeResponse(ips, torrentdata)
	} else {
		return EncodeResponse(ips, torrentdata)
	}
}

func decodeQueryURL(s string) url.Values {
	u, err := url.Parse(s)
	if err != nil {
		panic(err)
	}

	m, _ := url.ParseQuery(u.RawQuery)
	return m
}

func encodeKV(key string, value string) string {
	return fmt.Sprintf("%s%s", bencode.EncodeByteString(key), bencode.EncodeByteString(value))
}

func EncodeResponse(ipport []string, torrentdata *TorrentRequestData) string {
	ret := ""
	ret += encodeKV("interval", "30")
	ret += encodeKV("complete", "1")
	ipstr := ""

	if torrentdata.compact {
		count := 0

		for i := range ipport {
			ipstr += fmt.Sprintf("%s", ipport[i])
			count += 1
		}

		ret += encodeKV("incomplete", fmt.Sprintf("%d", count))
		ret += bencode.EncodeDictionary("peers", ipstr)
	} else {

		for i := range ipport {
			data := strings.Split(ipport[i], ":")

			ipstr += encodeKV("peer_id", "1")
			ipstr += encodeKV("ip", data[0])
			ipstr += encodeKV("port", data[1])
		}

		ret += bencode.EncodeDictionary("peers", ipstr)
	}

	return bencode.EncodeDictionary("", ret)
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	client := OpenClient()

	torrentdata := parseTorrentGetRequestURI(req.RequestURI, req.RemoteAddr)
	if torrentdata != nil {
		data := fillEmptyMapValues(torrentdata)

		worker(client, data)
		x := RedisGetKeyVal(client, data.info_hash, data)
		fmt.Println(x)

		response := formatResponseData(x, data)
		fmt.Println(response)

		w.Write([]byte(response))
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}

func OpenClient() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	return client
}

func CreateNewTorrentKey(client *redis.Client, key string, value *TorrentRequestData) {
	// CreateNewTorrentKey creates a new key. By default, it adds a member
	// ":ip". I don't think this ought to ever be generalized, as I just want
	// Redis to function in one specific way in notorious.

	// TODO(ian): You might want to set this explicitly in parameters
	// value := *TorrentRequestData
	client.SAdd(key, "ip")
}

func createIpPortPair(value *TorrentRequestData) string {
	// createIpPortPair creates a string formatted ("%s:%s", value.ip,
	// value.port) looking like so: "127.0.0.1:6886" and returns this value.
	return fmt.Sprintf("%s:%s", value.ip, value.port)
}

func RedisSetKeyVal(client *redis.Client, keymember string, value string) {
	// RedisSetKeyVal sets a key:member's value to value. Returns nothing as of
	// yet.
	client.SAdd(keymember, value)
}

func RedisGetKeyVal(client *redis.Client, key string, value *TorrentRequestData) []string {
	// RedisGetKeyVal retrieves a value from the Redis store by looking up the
	// provided key. If the key does not yet exist, we create the key in the KV
	// storage or if the value is empty, we add the current requester to the
	// list.
	keymember := concatenateKeyMember(key, "ip")

	val, err := client.SMembers(keymember).Result()
	if err != nil {
		// Fail because the key doesn't exist in the KV storage.
		CreateNewTorrentKey(client, keymember, value)
	}

	// If no keys yet exist in the KV storage.
	if len(val) == 0 {
		RedisSetKeyVal(client, keymember, createIpPortPair(value))
	}

	return val
}

func RedisGetBoolKeyVal(client *redis.Client, key string, value interface{}) bool {
	_, err := client.Get(key).Result()

	return err != nil
}

func concatenateKeyMember(key string, member string) string {
	var buffer bytes.Buffer

	buffer.WriteString(key)
	buffer.WriteString(":")
	buffer.WriteString(member)

	return buffer.String()
}
<|endoftext|>